diff --git a/devlog/_plan/261102_request_owned_affinity/000_plan.md b/devlog/_plan/261102_request_owned_affinity/000_plan.md new file mode 100644 index 00000000000..63eea7a8d92 --- /dev/null +++ b/devlog/_plan/261102_request_owned_affinity/000_plan.md @@ -0,0 +1,43 @@ +# 261102 — Request-owned bearer conversation affinity (#8080) + +**Reader summary.** When Codex CLI signs in with ChatGPT and talks to a Pool-mode proxy, its own +bearer makes `main` one more round-robin candidate (#5019), but the request had no affinity key, so +every turn re-ran the strategy and a conversation hopped between main and the stored accounts. The +request log labelled each of those picks `new_bind/healthy` although nothing was stored. This unit +makes every affinity label describe what selection actually stored, and adds the opt-in +`codexPool.requestOwnedAffinity` (default off) that keeps one account per conversation for a +request-owned bearer. Research: `010_research.md`. Contract: `structure/providers/codex-request-owned-affinity.md`. + +## Scope + +- In: `src/codex/request-owned-affinity.ts` (key, label check, issuer helpers), + `src/codex/request-owned-main-pin.ts` (byte-for-byte move), wiring in `src/codex/auth-context.ts`, + request previews (`src/server/responses/request-prepare.ts`), issuer and retry/compact fallbacks, + request log and usage normalization, config field, manual-selection issuer retention. +- Out: affinity persistence (#8091), round-robin eligibility (#8092), Claude identity lanes. + +## Decisions + +1. Labels are checked when selection returns: `new_bind`/`rebound` stay only if the key's entry names + the selected account; otherwise `unbound` (`request_owned`, `no_conversation`, or the original + reason) or `cleared`. `unbound` is not a spend move reason. +2. The opt-in key is an HMAC under a process-random secret over the exact Authorization value and the + conversation key. Account-id claims are not used as principal. +3. Final auth and both previews (first and recovery, per candidate model) share one derivation; fixed + selectors and Reserve get no key; request-owned traffic never records lineage. +4. A separate bearer-scoped conversation-state key (never passed to routing) carries issuer records + for request-owned Pool and caller-owned main contexts, with or without the opt-in. +5. Manual account selection keeps issuer records while clearing routing bindings. +6. A refusal on a caller-owned main binding releases that binding without touching shared main health. + +## Acceptance + +Regression tests in `tests/codex-integration/codex-request-owned-affinity.test.ts`, +`tests/codex-integration/reserve-auth-context.test.ts`, `tests/responses/compact-transient-alternate.test.ts`, +`tests/responses/responses-alternate-main-cancellation.test.ts` and +`tests/responses/responses-preview-main-read-fence.test.ts` fail on the pre-change code and pass after it. + +## Outcome + +PR #8096. Plan audited over five rounds; implementation reviewed by a dedicated security review +and an independent code review before publication. diff --git a/devlog/_plan/261102_request_owned_affinity/010_research.md b/devlog/_plan/261102_request_owned_affinity/010_research.md new file mode 100644 index 00000000000..c03e9bb25b1 --- /dev/null +++ b/devlog/_plan/261102_request_owned_affinity/010_research.md @@ -0,0 +1,20 @@ +# 010 — Research (#8080) + +Source: Terra research task, 2026-11-02, read-only, at 6f1d3e65c. + +- Ownership flag: `codexRouteCredentialOwnership` (src/server/responses/core-auth.ts) marks a forwardable + caller bearer request-scoped. +- `poolStateEligible` (src/codex/auth-context.ts) rejects request-scoped credentials, so `affinityKey` and + lineage are undefined and `resolveCodexAccountForThreadDetailed` receives a null key. +- Since #5019/#5024 an unpinned caller bearer makes main an ordinary round-robin candidate; a stored + account selection materializes that stored credential. Intentional and documented in + structure/providers/openai-tiers.md. +- Round-robin commits its pick with a null key but binds only when `commitAffinity && threadId` + (src/codex/routing/selection.ts). With main + one stored account and sticky limit 1 the ring + alternates; with two stored accounts it cycles main, A, B. +- Every unbound success returns `affinityAfterRelease(null, undefined)` = `new_bind/healthy`. +- Log path: `noteCodexSelectionAffinity` (pool/main-pool only) → request log → `persistedAffinityFields` + → usage.jsonl via `KNOWN_AFFINITY_MOVES/REASONS` (src/usage/log.ts). No exhaustive Record over the + unions; GUI does not render these values. +- Options weighed: opt-in process-local namespace (chosen); always serve main (contradicts #5019); + raise sticky limit (shared counter, not affinity); omit label only (honest but uninformative). diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index ee6d1f4bb91..5e656662799 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -1314,6 +1314,23 @@ When an account leaves pool selection, the reason travels with the decision inst A main-account refresh that does not complete still answers `503` with `Retry-After`, because a retry may still succeed. The message now adds that a failure which persists means the main account needs reauthentication, rather than only asking for another attempt. +### Request-owned conversation affinity + +`codexPool.requestOwnedAffinity` is an optional boolean, defaulting to `false`. Set it to `true` +to keep a conversation carrying its own ChatGPT bearer on one eligible account within each quota +scope. Releases and model detours can still move it. The binding is process-local; restarting the +proxy or rotating the bearer starts a new binding. Session-less multi-generation subagent chains +may bind per thread. + +Affinity labels describe the binding when selection returns. `unbound` means no binding was stored; +`request_owned` means the caller-owned credential did not opt in, and `no_conversation` means no +conversation key was available. A later eligibility change can release a binding after selection. + +With or without the option, a request that carries its own bearer records which account served each +turn under a key scoped to that bearer. When a later turn is served by a different account, including +after you switch accounts manually, OpenCodex removes `previous_response_id` before sending and +rejects carried uploaded `file_id` references with HTTP 400, because the new account cannot read them. + ### Optional idle-window steering `codexPool.startIdleWindows` is an optional boolean and defaults to `false`. When enabled, a new diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index ffdb46d1801..80b5b0185ef 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -675,7 +675,7 @@ "codex-plugins-doctor.test.ts": "codex-integration", "codex-pool-plan-exclusion.test.ts": "codex-integration", "codex-pool-refresh-backoff.test.ts": "codex-integration", - "codex-pool-request-owned-main.test.ts": "codex-integration", + "codex-pool-request-owned-main.test.ts": "codex-integration", "codex-request-owned-affinity.test.ts": "codex-integration", "codex-pool-rotation.test.ts": "codex-integration", "codex-auth-release.test.ts": "codex-integration", "codex-priority-failback.test.ts": "codex-integration", "codex-round-robin-headroom.test.ts": "codex-integration", "codex-idle-window.test.ts": "codex-integration", diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index dd35923d2fa..7a17e2144ed 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1,8 +1,10 @@ +import { requestOwnedAffinityKey, honestAffinityDecision, callerOwnedFallbackAffinityDecision, captureRequestOwnedAffinityBindings, type RequestOwnedBindingSnapshot } from "./request-owned-affinity"; +import { requestOwnedMainPinState, mainCreditsHoldResetAt, callerMatchesObservedMain } from "./request-owned-main-pin"; +export { requestOwnedMainPinState } from "./request-owned-main-pin"; import type { OcxRuntimeProviderConfig } from "./auth-context-provider"; export type { OcxRuntimeProviderConfig } from "./auth-context-provider"; export { stripCodexRuntimeProviderFields } from "./auth-context-provider"; import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt, poolCreditRecoveryNeeded, poolContextCreditRecoveryNeeded, poolCreditRecoveryRetryAt } from "./pool-credit-policy"; -import { hasSpendableCodexCredits } from "./quota-types"; import { nextQuotaQueryDelay } from "./quota-query-backoff"; import { noteMainAccountActivity } from "./main-account-external-usage"; import { codexAccountPriorityFailbackEnabled } from "./account-priority"; @@ -40,9 +42,7 @@ import type { NativeMainStartupBlockReason } from "./native-profile-startup"; import { waitForMainAccountPolicyBinding } from "./main-account-policy-wait"; import { codexQuotaScopeForModel, - computeCodexUsageScore, getCodexQuotaHealthSnapshot, - isEffectiveCodexAccountPinned, releaseCodexQuotaProbeLease, releaseCodexQuotaScopeProbeLease, tryAcquireCodexQuotaProbeLease, @@ -74,8 +74,8 @@ import type { CodexCooldownSource, CodexQuotaScope } from "./routing"; import { maskAccountId } from "../lib/privacy"; import { formatErrorResponse, formatUsageLimitReachedResponse } from "../bridge"; import { nextCodexPoolRecoveryAt } from "./routing/pool-recovery"; -import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota, getMainPolicyQuota, isValidWhamHistoryObservation, parseUsageQuota, parseMainPolicyUsageQuota, setAccountQuotaFromParsed } from "./quota"; -import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt } from "./account-credit-use"; +import { getAccountQuota, isValidWhamHistoryObservation, parseUsageQuota, parseMainPolicyUsageQuota, setAccountQuotaFromParsed } from "./quota"; +import { codexAccountUsesCreditsAfterLimit } from "./account-credit-use"; import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types"; import { FORWARD_HEADERS } from "../adapters/openai-responses"; import { isolateCodexPoolAccountHeaders } from "./forward-transport-headers"; @@ -85,7 +85,6 @@ import { resolveMainAccountHardLockThresholds, getMainAccountHardLockStatus, isMainAccountHardLockEnabled, - isMainAccountHardLocked, } from "./main-account-hard-lock"; import { captureMainAccountIdentityGeneration, @@ -102,67 +101,9 @@ import { CODEX_RESERVE_HELPER_UNSUPPORTED_MESSAGE, isCodexReserveHelperUnsupport import type { DataPlaneAdmission } from "../server/auth-cors"; import { getMainReserveAuthorization, isMainReserveAuthorizationLive, nativeUserIdClaims, type MainReserveAuthorization } from "./reserve-availability"; import { UpstreamRetryEvidenceError } from "../lib/upstream-retry"; -import { getEffectiveCodexAutoSwitchThreshold } from "./account-auto-switch"; import { CODEX_MAIN_DUPLICATE_ACTION, configuredCodexMainDuplicate } from "./collision-identity"; import { AgentAssertionUnsupportedError, authorizationCredential, hasAgentAssertionAuthorization } from "../lib/authorization-scheme"; -/** - * A request-owned bearer cannot inspect the physical main credential for its plan, but cached - * WHAM usage is still valid routing evidence for the same logical main account. Score it with - * the conservative unknown-plan rule: an unobserved governing window preserves the pin, while - * any known weekly/monthly/short value at the threshold releases it through the ordinary Pool - * path. This keeps the keyring boundary intact instead of reading auth.json just to classify a - * request that already brought its own credential (#3157). - */ -function requestOwnedMainPinHasQuotaHeadroom(config: OcxConfig): boolean { - const threshold = getEffectiveCodexAutoSwitchThreshold(config, MAIN_CODEX_ACCOUNT_ID); - if (threshold <= 0) return true; - const usage = computeCodexUsageScore(getAccountQuota(MAIN_CODEX_ACCOUNT_ID), undefined, Date.now(), codexAccountUsesCreditsAfterLimit(config, MAIN_CODEX_ACCOUNT_ID)); - return usage >= CODEX_UNKNOWN_USAGE_SCORE || usage < threshold; -} - -/** - * Whether a request carrying its OWN main credential still serves on main because the operator - * manually pinned it (#3166), split from the surrounding resolution so request preview can ask - * the identical question (#4850). - * - * Exported for exactly one reason: two copies of this fence is how #4850 happened. Final - * authentication honoured the ownership boundary while request preview, computing its fence from - * recovery and drain state alone, still handed pool eligibility the default liveness probe and - * opened the physical `auth.json` twice per spawn. A predicate one caller can forget is a - * predicate the other caller will eventually disagree with. - * - * Read-free by construction, which is what makes it usable on the fenced side. Every input is - * config, policy, or in-memory runtime state: the pin fields, the paused list, the cached quota - * score, the main policy quota the hard lock and the credits switch read, and `callerMatchesObservedMain`, which compares HMAC digests against the observed - * credential record in `main-account-cache.ts`. Nothing here opens a file. - * - * `candidate` is the pin before the hard-lock question, because the caller still owes the - * pending-binding check that only final authentication can fail closed on. - */ -export function requestOwnedMainPinState( - headers: Headers, - config: OcxConfig, - policy: CodexAuthPolicyConfig, - requestScopedMainCredential: boolean, - fixedAccountId: string | undefined, - quotaScope?: CodexQuotaScope, -): { candidate: boolean; preserve: boolean } { - const candidate = requestScopedMainCredential - && fixedAccountId === undefined - && config.activeCodexAccountPinned === MAIN_CODEX_ACCOUNT_ID - && isEffectiveCodexAccountPinned(config) - && !policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID) - && requestOwnedMainPinHasQuotaHeadroom(config); - return { - candidate, - preserve: candidate && !(callerMatchesObservedMain(headers) - && (isMainAccountHardLocked(policy) - || mainCreditsHoldResetAt(policy) !== undefined - || getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope)?.cooldownUntil)), - }; -} - /** * One conversation tree keys as ONE cohort (#4780). * @@ -247,12 +188,12 @@ export function requestOwnedMainCredentialIsLive(inputs: { } /** - * May this request own Pool affinity state at all? + * May this request use stored-credential Pool lineage? * - * Two credentials authenticate outside the Pool: an exact account selector (including the - * Reserve pin) and a request-owned main bearer, which exists for one request and must never - * fold into durable account state. Neither may read or write a binding, so neither may read - * or write LINEAGE either. + * Two credentials authenticate outside stored Pool state: an exact account selector (including + * Reserve) and a request-owned main bearer, which must never fold into durable account state. + * Neither may read or write a stored-credential binding or lineage. The optional request-owned + * affinity namespace is separate from this predicate. * * Exported so that a preview asks the question with the code that answers it, instead of a * restatement that can drift. It drifted once already: preview read a family relation from raw @@ -272,6 +213,15 @@ export function codexPoolStateEligible( ); } +/** Shared affinity derivation for final authentication and both request previews. */ +export function codexRequestAffinityKey(headers: Headers, config: OcxConfig, policy: CodexAuthPolicyConfig | undefined, options: CodexPoolStateEligibility = {}): string | undefined { + if (options.accountId !== undefined || requiresReserveAuthorization(policy, options.modelId, options.admission)) return undefined; + if (options.requestScopedMainCredential === true && hasCallerCodexBearer(headers)) { + return config.codexPool?.requestOwnedAffinity === true ? requestOwnedAffinityKey(headers) : undefined; + } + return codexPoolAffinityKey(headers); +} + /** * The lineage a PREVIEW is allowed to see: read-only, and only for a request that may hold Pool * state. Recording is left to the resolution that actually binds, so a preview can never leave a @@ -289,7 +239,7 @@ export function previewCodexPoolLineage( export type CodexAuthContext = | { - kind: "main"; accountId: null; reserveAuthorization?: MainReserveAuthorization; + kind: "main"; accountId: null; reserveAuthorization?: MainReserveAuthorization; affinityKey?: string; conversationStateKey?: string; affinityDecision?: CodexAffinityDecision; affinityBindingSnapshot?: RequestOwnedBindingSnapshot; /** Captured for the observed main credential actually selected for upstream. */ mainQuotaDispatch?: MainQuotaDispatch; } @@ -304,7 +254,7 @@ export type CodexAuthContext = /** Bypass Pool selection and suppress quota/transient failover for an exact selector. */ fixedAccount?: boolean; /** Pool binding key; the Desktop fallback is an opaque process-local HMAC. */ - affinityKey?: string; + affinityKey?: string; conversationStateKey?: string; /** * Set when this request was admitted through an active quota cooldown as * the account's single probe. Must be echoed into the upstream outcome so @@ -338,7 +288,7 @@ export type CodexAuthContext = chatgptAccountId: string; /** Bypass Pool selection and suppress quota/transient failover for an exact selector. */ fixedAccount?: boolean; - affinityKey?: string; // See `pool.affinityKey`. + affinityKey?: string; conversationStateKey?: string; // See `pool.affinityKey`. /** See `pool.probeLeaseId`. */ probeLeaseId?: string; quotaScope?: CodexQuotaScope; @@ -786,12 +736,6 @@ export function createCodexAuthDispatchGuard( }; } -function mainCreditsHoldResetAt(config: Pick): number | undefined { - if (codexAccountUsesCreditsAfterLimit(config, MAIN_CODEX_ACCOUNT_ID) - && hasSpendableCodexCredits(getMainPolicyQuota())) return undefined; - return codexUsageLimitResetAt(getMainPolicyQuota(), undefined, Date.now()); -} - function assertMainAccountPolicy( config: Pick | undefined, ): void { @@ -807,15 +751,6 @@ function assertMainAccountPolicy( noteMainAccountActivity(); } -/** No auth-file I/O: an unsigned claim alone never identifies a caller as stored main. */ -function callerMatchesObservedMain(headers: Headers): boolean { - const bearer = authorizationCredential(headers); - if (!bearer) return false; - const effectiveAccountId = headers.get("chatgpt-account-id") - ?? extractAccountId(undefined, bearer); - return matchesMainQuotaCredential(bearer, effectiveAccountId); -} - /** Constant-time digest comparison so bearer bytes never drive branch timing. */ function sameCredentialMaterial(a: string, b: string): boolean { return timingSafeEqual(createHash("sha256").update(a).digest(), createHash("sha256").update(b).digest()); @@ -1282,6 +1217,8 @@ async function resolveCodexAuthContextUnannotated( directSelectionAdmission.release(); } }; + const affinityKey = codexRequestAffinityKey(headers, config, policy, { accountId: fixedAccountId, modelId: options.modelId, admission: options.admission, requestScopedMainCredential }); + const conversationStateKey = affinityKey ?? (requestScopedMainCredential && fixedAccountId === undefined && !reserve ? requestOwnedAffinityKey(headers) : undefined); // Pool discovery excludes request-owned main credentials by design: they must never be folded // into stored-account entitlement, affinity, or persistence state. An effective manual main pin // is the one exception where that exclusion is selection evidence in the opposite direction. @@ -1294,7 +1231,7 @@ async function resolveCodexAuthContextUnannotated( options.isDirectCallerEntitledToCodexModel ?? isDirectCallerEntitledToCodexModel )(headers, options.modelId); if (callerEntitled && preserveRequestOwnedMainPin()) { - return { kind: "main", accountId: null }; + return { kind: "main", accountId: null, ...(mode === "pool" && conversationStateKey ? { conversationStateKey } : {}) }; } } // An explicit namespace binding is stronger than the provider's default mode. It must use the @@ -1309,19 +1246,24 @@ async function resolveCodexAuthContextUnannotated( } // A caller bearer can still accompany a request that selects a configured Pool account. Do not // let that request read, delete, or create a file-main affinity binding while deciding whether a - // stored account is available; only the stored credential selected below may own Pool state. - const affinityKey = poolStateEligible(fixedAccountId, requestScopedMainCredential) - ? codexPoolAffinityKey(headers) - : undefined; - // The thread's family relation, recorded under the same condition as the key itself. A - // first-placing child consults it; a request-owned or fixed credential never enters Pool - // state, so it never enters lineage either. - const lineage = affinityKey !== undefined + // stored account is available. Optional caller-owned bindings use a separate namespace; fixed + // selectors never own a binding. + // A first-placing child consults its thread's family relation. Fixed and request-owned + // credentials never enter stored Pool lineage, even when request-owned affinity is enabled. + const lineage = affinityKey !== undefined && poolStateEligible(fixedAccountId, requestScopedMainCredential) ? recordCodexThreadLineage(headers) : undefined; // Why this request is on this account, carried to the request log so a move reads as an event // instead of something inferred from account labels across lines (#4546). let affinityDecision: CodexAffinityDecision | undefined; + const resolveKeyedCallerMain = async (fallbackReason?: "cooldown" | "unusable"): Promise => { + const withheld = requestScopedMainCredential && config.codexPool?.requestOwnedAffinity !== true; + const decision = fallbackReason ? callerOwnedFallbackAffinityDecision(affinityDecision, affinityKey, quotaScope, options.modelId, withheld, fallbackReason) + : honestAffinityDecision(affinityDecision, affinityKey, MAIN_CODEX_ACCOUNT_ID, quotaScope, options.modelId, withheld); + const affinityBindingSnapshot = captureRequestOwnedAffinityBindings(affinityKey, quotaScope, options.modelId); + const context = await resolveCallerOwnedMainContext(); + return context.kind === "main" ? { ...context, ...(affinityBindingSnapshot ? { affinityBindingSnapshot } : {}), ...(affinityKey ? { affinityKey } : {}), ...(conversationStateKey ? { conversationStateKey } : {}), ...(decision ? { affinityDecision: decision } : {}) } : context; + }; // The half-open trial this request was granted, if it is the one allowed to test a held // account. Declared out here because the release paths below and the returned context are on // opposite sides of several throws (#4701). @@ -1470,17 +1412,17 @@ async function resolveCodexAuthContextUnannotated( throw new CodexRecoveryWithheldError(resolution.accountId, resolution.retryAt, resolution.detourAccountId); } const selected = resolution.status === "selected" ? resolution.accountId : null; - affinityDecision = resolution.affinity; + affinityDecision = selected ? honestAffinityDecision(resolution.affinity, affinityKey, selected, quotaScope, options.modelId, requestScopedMainCredential && config.codexPool?.requestOwnedAffinity !== true) : resolution.affinity; transientProbe = resolution.status === "selected" ? resolution.transientProbe : undefined; // Main now takes part in ordering when the request carries its own main bearer (#5019), so // selection can name it outright rather than only through the no-candidate fallback below. // Serve that selection from the credential the request arrived with: the read fence above // forbids this request from claiming, reading or reconciling the stored main profile, and a - // request-owned credential owns no Pool state, so there is nothing here to claim, prime or - // cool down. Hand back any trial first -- no Pool account is being sent to. + // request-owned credential owns no shared health state, so there is nothing here to claim, + // prime or cool down. Hand back any trial first -- no Pool account is being sent to. if (selected === MAIN_CODEX_ACCOUNT_ID && requestScopedMainCredential) { releaseTransientProbeGrant(); - const context = await resolveCallerOwnedMainContext(); + const context = await resolveKeyedCallerMain(); console.info("[codex] Pool selected caller-owned main credential."); return context; } @@ -1500,7 +1442,7 @@ async function resolveCodexAuthContextUnannotated( && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID ) { - const context = await resolveCallerOwnedMainContext(); + const context = await resolveKeyedCallerMain("unusable"); console.info("[codex] Pool used caller-owned main credential: no selectable stored account."); return context; } @@ -1667,7 +1609,7 @@ async function resolveCodexAuthContextUnannotated( && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID && !callerIsCooledPoolAccount(headers, config, accountId)) { - const context = await resolveCallerOwnedMainContext(); + const context = await resolveKeyedCallerMain("cooldown"); console.info("[codex] Pool used caller-owned main credential: stored account cooldown."); return context; } @@ -1727,6 +1669,7 @@ async function resolveCodexAuthContextUnannotated( chatgptAccountId: token.chatgptAccountId, ...(fixedAccountId !== undefined ? { fixedAccount: true } : {}), ...(affinityKey ? { affinityKey } : {}), + ...(conversationStateKey ? { conversationStateKey } : {}), ...(quotaScope ? { quotaScope } : {}), ...(probeLeaseId ? { probeLeaseId } : {}), ...(probeQuotaScope ? { probeQuotaScope } : {}), @@ -1750,6 +1693,7 @@ async function resolveCodexAuthContextUnannotated( chatgptAccountId: token.chatgptAccountId, ...(fixedAccountId !== undefined ? { fixedAccount: true } : {}), ...(affinityKey ? { affinityKey } : {}), + ...(conversationStateKey ? { conversationStateKey } : {}), ...(quotaScope ? { quotaScope } : {}), ...(probeLeaseId ? { probeLeaseId } : {}), ...(probeQuotaScope ? { probeQuotaScope } : {}), diff --git a/src/codex/request-owned-affinity.ts b/src/codex/request-owned-affinity.ts new file mode 100644 index 00000000000..514dc300c0b --- /dev/null +++ b/src/codex/request-owned-affinity.ts @@ -0,0 +1,121 @@ +/** + * Caller-owned affinity is scoped to the exact Authorization value and conversation key under a + * process-random HMAC key. It is process-local and must never be anchored or persisted, as required + * by the structure contract. Decisions describe the binding at selection time. + */ +import { createHmac, randomBytes } from "node:crypto"; +import { MAIN_CODEX_ACCOUNT_ID } from "./account-id"; +import { codexConversationIdentity } from "./lineage"; +import { getThreadAffinity, getModelDetourAffinity, deleteThreadAffinity, deleteModelDetourAffinity, notePendingReleaseReason, type CodexAffinityDecision, type CodexAffinityReason, type ThreadAffinityEntry } from "./routing/thread-affinity"; +import type { CodexQuotaScope } from "./routing/health-store"; +import { classifyCodexUpstreamOutcome } from "./routing/cooldown-math"; +import type { CodexAuthContext } from "./auth-context"; + +const key = randomBytes(32); +const prefix = "request-owned:v1:"; + +/** Derive the opaque key for this caller and conversation, if both exist. */ +export function requestOwnedAffinityKey(headers: Headers): string | undefined { + const authorization = headers.get("authorization"); + const conversation = codexConversationIdentity(headers)?.conversationKey; + if (!authorization || !conversation) return undefined; + return prefix + createHmac("sha256", key) + .update("opencodex-request-owned-affinity-v1\0").update(authorization).update("\0").update(conversation) + .digest("base64url"); +} + +/** Identify namespace keys that must remain process-local. */ +export function isRequestOwnedAffinityKey(value: string): boolean { + return value.startsWith(prefix); +} + +/** Report a bind only when the selection-time entry names the selected account. */ +export function honestAffinityDecision( + decision: CodexAffinityDecision | undefined, + affinityKey: string | undefined, + accountId: string, + quotaScope?: CodexQuotaScope, + modelId?: string, + requestOwnedKeyWithheld = false, +): CodexAffinityDecision | undefined { + if (!decision || (decision.move !== "new_bind" && decision.move !== "rebound")) return decision; + // This is a selection-time snapshot, immediately after the synchronous selector returns. + if (affinityKey && (getThreadAffinity(affinityKey, quotaScope)?.accountId === accountId + || getModelDetourAffinity(affinityKey, modelId, quotaScope)?.accountId === accountId)) return decision; + if (decision.move === "rebound") return { move: "cleared", reason: decision.reason }; + return { move: "unbound", reason: affinityKey ? decision.reason : requestOwnedKeyWithheld ? "request_owned" : "no_conversation" }; +} + +/** Identify the issuer for a Pool or keyed caller-main context. */ +export function conversationStateAccountId(ctx: { kind: "main" | "pool" | "main-pool"; accountId: string | null; affinityKey?: string; conversationStateKey?: string }): string | undefined { + return ctx.kind === "main" ? (ctx.affinityKey || ctx.conversationStateKey ? MAIN_CODEX_ACCOUNT_ID : undefined) : ctx.accountId ?? undefined; +} + +/** Describe the account actually serving a caller-owned fallback. */ +export function callerOwnedFallbackAffinityDecision( + decision: CodexAffinityDecision | undefined, affinityKey: string | undefined, quotaScope: CodexQuotaScope | undefined, + modelId: string | undefined, requestOwnedKeyWithheld: boolean, reason: "cooldown" | "unusable", +): CodexAffinityDecision { + const ordinary = affinityKey ? getThreadAffinity(affinityKey, quotaScope) : undefined; + const detour = affinityKey ? getModelDetourAffinity(affinityKey, modelId, quotaScope) : undefined; + if (ordinary?.accountId === MAIN_CODEX_ACCOUNT_ID || detour?.accountId === MAIN_CODEX_ACCOUNT_ID) + return honestAffinityDecision(decision ?? { move: "reused", reason: "healthy" }, affinityKey, MAIN_CODEX_ACCOUNT_ID, quotaScope, modelId, requestOwnedKeyWithheld)!; + if (ordinary || detour) return { move: "detour", reason }; + return { move: "unbound", reason: affinityKey ? decision?.reason ?? "healthy" : requestOwnedKeyWithheld ? "request_owned" : "no_conversation" }; +} + +/** Opaque process-local identity for the bindings present when main was selected. */ +export type RequestOwnedBindingSnapshot = object; +type EntryIdentity = { entry: ThreadAffinityEntry; accountId: string; generation: number; createdAt: number }; +const bindingSnapshots = new WeakMap(); + +function identity(entry: ThreadAffinityEntry | undefined): EntryIdentity | undefined { + return entry && { entry, accountId: entry.accountId, generation: entry.generation, createdAt: entry.createdAt }; +} +function matches(captured: EntryIdentity | undefined, current: ThreadAffinityEntry | undefined): boolean { + return !!captured && captured.entry === current && captured.accountId === current.accountId + && captured.generation === current.generation && captured.createdAt === current.createdAt; +} + +/** Capture the exact entries that a request is authorized to release after selection. */ +export function captureRequestOwnedAffinityBindings(key: string | undefined, scope?: CodexQuotaScope, modelId?: string): RequestOwnedBindingSnapshot | undefined { + if (!key || !isRequestOwnedAffinityKey(key)) return undefined; + const snapshot = {}; + bindingSnapshots.set(snapshot, { key, scope, modelId, ordinary: identity(getThreadAffinity(key, scope)), detour: identity(getModelDetourAffinity(key, modelId, scope)) }); + return snapshot; +} + +/** Whether a context carries a caller-owned routing key. */ +export function isKeyedRequestOwnedMain(ctx: CodexAuthContext): ctx is Extract & { affinityKey: string } { + return ctx.kind === "main" && !!ctx.affinityKey && isRequestOwnedAffinityKey(ctx.affinityKey); +} + +/** Release this caller’s scoped binding after its own main credential is refused. */ +export function releaseRequestOwnedMainOnOutcome(ctx: CodexAuthContext, outcome: number, _modelId?: string): void { + if (!isKeyedRequestOwnedMain(ctx) || !ctx.affinityBindingSnapshot) return; + const snapshot = bindingSnapshots.get(ctx.affinityBindingSnapshot); + if (!snapshot || snapshot.key !== ctx.affinityKey) return; + const outcomeClass = classifyCodexUpstreamOutcome(outcome); + const reason: CodexAffinityReason | undefined = outcomeClass === "quota" ? "quota_refusal" + : outcomeClass === "credential" ? "unusable" : outcomeClass === "transient" ? "transient" : undefined; + if (!reason) return; + const key = ctx.affinityKey; + const { scope, modelId: selectedModelId } = snapshot; + let released = false; + const ordinary = getThreadAffinity(key, scope); + const detour = getModelDetourAffinity(key, selectedModelId, scope); + const mainDetour = matches(snapshot.detour, detour) && detour?.accountId === MAIN_CODEX_ACCOUNT_ID; + // A caller-main fallback or model detour can leave an ordinary binding on a different + // account. Retire it too so the next placement reports the release rather than reuse. If + // main was itself bound and another turn has since rebound elsewhere, keep that newer entry. + const callerMainDetour = ctx.affinityDecision?.move === "detour"; + if (matches(snapshot.ordinary, ordinary) && (ordinary?.accountId === MAIN_CODEX_ACCOUNT_ID || mainDetour || callerMainDetour)) { + deleteThreadAffinity(key, scope); + released = true; + } + if (mainDetour) { + deleteModelDetourAffinity(key, selectedModelId, scope); + released = true; + } + if (released) notePendingReleaseReason(key, reason); +} diff --git a/src/codex/request-owned-main-pin.ts b/src/codex/request-owned-main-pin.ts new file mode 100644 index 00000000000..dd9917b264b --- /dev/null +++ b/src/codex/request-owned-main-pin.ts @@ -0,0 +1,86 @@ +import type { CodexAuthPolicyConfig } from "./auth-context"; +import type { OcxConfig } from "../types"; +import type { CodexQuotaScope } from "./routing"; +import { MAIN_CODEX_ACCOUNT_ID } from "./account-id"; +import { computeCodexUsageScore, getCodexQuotaHealthSnapshot, isEffectiveCodexAccountPinned } from "./routing"; +import { getEffectiveCodexAutoSwitchThreshold } from "./account-auto-switch"; +import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt } from "./account-credit-use"; +import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota, getMainPolicyQuota } from "./quota"; +import { hasSpendableCodexCredits } from "./quota-types"; +import { isMainAccountHardLocked } from "./main-account-hard-lock"; +import { authorizationCredential } from "../lib/authorization-scheme"; +import { extractAccountId } from "../oauth/chatgpt"; +import { matchesMainQuotaCredential } from "./main-account-cache"; + +/** + * A request-owned bearer cannot inspect the physical main credential for its plan, but cached + * WHAM usage is still valid routing evidence for the same logical main account. Score it with + * the conservative unknown-plan rule: an unobserved governing window preserves the pin, while + * any known weekly/monthly/short value at the threshold releases it through the ordinary Pool + * path. This keeps the keyring boundary intact instead of reading auth.json just to classify a + * request that already brought its own credential (#3157). + */ +function requestOwnedMainPinHasQuotaHeadroom(config: OcxConfig): boolean { + const threshold = getEffectiveCodexAutoSwitchThreshold(config, MAIN_CODEX_ACCOUNT_ID); + if (threshold <= 0) return true; + const usage = computeCodexUsageScore(getAccountQuota(MAIN_CODEX_ACCOUNT_ID), undefined, Date.now(), codexAccountUsesCreditsAfterLimit(config, MAIN_CODEX_ACCOUNT_ID)); + return usage >= CODEX_UNKNOWN_USAGE_SCORE || usage < threshold; +} + +/** + * Whether a request carrying its OWN main credential still serves on main because the operator + * manually pinned it (#3166), split from the surrounding resolution so request preview can ask + * the identical question (#4850). + * + * Exported for exactly one reason: two copies of this fence is how #4850 happened. Final + * authentication honoured the ownership boundary while request preview, computing its fence from + * recovery and drain state alone, still handed pool eligibility the default liveness probe and + * opened the physical `auth.json` twice per spawn. A predicate one caller can forget is a + * predicate the other caller will eventually disagree with. + * + * Read-free by construction, which is what makes it usable on the fenced side. Every input is + * config, policy, or in-memory runtime state: the pin fields, the paused list, the cached quota + * score, the main policy quota the hard lock and the credits switch read, and `callerMatchesObservedMain`, which compares HMAC digests against the observed + * credential record in `main-account-cache.ts`. Nothing here opens a file. + * + * `candidate` is the pin before the hard-lock question, because the caller still owes the + * pending-binding check that only final authentication can fail closed on. + */ +export function requestOwnedMainPinState( + headers: Headers, + config: OcxConfig, + policy: CodexAuthPolicyConfig, + requestScopedMainCredential: boolean, + fixedAccountId: string | undefined, + quotaScope?: CodexQuotaScope, +): { candidate: boolean; preserve: boolean } { + const candidate = requestScopedMainCredential + && fixedAccountId === undefined + && config.activeCodexAccountPinned === MAIN_CODEX_ACCOUNT_ID + && isEffectiveCodexAccountPinned(config) + && !policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID) + && requestOwnedMainPinHasQuotaHeadroom(config); + return { + candidate, + preserve: candidate && !(callerMatchesObservedMain(headers) + && (isMainAccountHardLocked(policy) + || mainCreditsHoldResetAt(policy) !== undefined + || getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope)?.cooldownUntil)), + }; +} + +export function mainCreditsHoldResetAt(config: Pick): number | undefined { + if (codexAccountUsesCreditsAfterLimit(config, MAIN_CODEX_ACCOUNT_ID) + && hasSpendableCodexCredits(getMainPolicyQuota())) return undefined; + return codexUsageLimitResetAt(getMainPolicyQuota(), undefined, Date.now()); +} + + +/** No auth-file I/O: an unsigned claim alone never identifies a caller as stored main. */ +export function callerMatchesObservedMain(headers: Headers): boolean { + const bearer = authorizationCredential(headers); + if (!bearer) return false; + const effectiveAccountId = headers.get("chatgpt-account-id") + ?? extractAccountId(undefined, bearer); + return matchesMainQuotaCredential(bearer, effectiveAccountId); +} diff --git a/src/codex/routing/active-account.ts b/src/codex/routing/active-account.ts index 811bea390d5..7055199d491 100644 --- a/src/codex/routing/active-account.ts +++ b/src/codex/routing/active-account.ts @@ -32,7 +32,7 @@ let runtimeActiveCodexAccountId: string | undefined; /** Manual selection resets transient routing evidence without bypassing a real 429 cooldown. */ export function resetCodexRoutingForManualSelection(accountId: string): void { - clearThreadAccountMap(); + clearThreadAccountMap({ preserveConversationStateIssuers: true }); // Manual selection is the operator source of truth — drop any automatic runtime cursor. runtimeActiveCodexAccountId = undefined; // Record the pick as an unspent one-shot on the shared scope and on each independent scope diff --git a/src/codex/routing/thread-affinity.ts b/src/codex/routing/thread-affinity.ts index 1a44437177f..643d64fc5c4 100644 --- a/src/codex/routing/thread-affinity.ts +++ b/src/codex/routing/thread-affinity.ts @@ -86,7 +86,9 @@ export type CodexAffinityMove = /** There was no live binding; this request established one. */ | "new_bind" /** The binding was released without a replacement on this request. */ - | "cleared"; + | "cleared" + /** No binding was established for this selection. */ + | "unbound"; /** * Why. A move is the expensive event -- it discards the prompt-cache prefix warmed on the old @@ -95,6 +97,10 @@ export type CodexAffinityMove = */ export type CodexAffinityReason = | "healthy" + /** The request had no usable conversation key. */ + | "no_conversation" + /** A caller-owned credential did not opt into affinity. */ + | "request_owned" | "quota_headroom" | "quota_refusal" | "transient" @@ -202,7 +208,7 @@ type ConversationStateIssuerEntry = { }; const conversationStateIssuerMap = new Map(); -export function clearThreadAccountMap(): void { +export function clearThreadAccountMap(options: { preserveConversationStateIssuers?: boolean } = {}): void { threadAccountMap.clear(); threadAffinityEntryTotal = 0; // A refresh cooldown is per-account runtime state learned alongside these bindings. Leaving it @@ -213,7 +219,9 @@ export function clearThreadAccountMap(): void { // going away. A held account nobody may probe because of a lease issued against the previous // roster is a recovery that never starts. clearPoolRecoveryState(); - conversationStateIssuerMap.clear(); + // A manual account switch clears routing affinity, not the issuer of carried state. The next + // account needs that record to scrub or refuse account-bound continuation. + if (!options.preserveConversationStateIssuers) conversationStateIssuerMap.clear(); } export function clearConversationStateIssuerMap(): void { @@ -294,7 +302,7 @@ export function peekConversationStateIssuer( const pendingReleaseReasons = new Map(); const MAX_PENDING_RELEASE_REASONS = 4096; -function notePendingReleaseReason(threadId: string | null, reason: CodexAffinityReason): void { +export function notePendingReleaseReason(threadId: string | null, reason: CodexAffinityReason): void { if (threadId === null) return; if (!pendingReleaseReasons.has(threadId) && pendingReleaseReasons.size >= MAX_PENDING_RELEASE_REASONS) { const oldest = pendingReleaseReasons.keys().next(); diff --git a/src/config/schema/leaf-validators.ts b/src/config/schema/leaf-validators.ts index 3eacf076d56..5df3213e410 100644 --- a/src/config/schema/leaf-validators.ts +++ b/src/config/schema/leaf-validators.ts @@ -1045,6 +1045,7 @@ export const clientConnectionSchema = z.object({ export const codexPoolSchema = z.object({ excludedPlans: z.array(z.string().trim().min(1)).optional(), startIdleWindows: z.boolean().optional(), + requestOwnedAffinity: z.boolean().optional(), lowQuotaProtection: z.object({ enabled: z.boolean(), threshold: z.number().finite().min(1).max(100), diff --git a/src/server/request-log.ts b/src/server/request-log.ts index 925bd5f2c54..be3dde123cc 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -1235,13 +1235,13 @@ export function httpStatusForRequestLogTerminal( * `unresolved` rather than quietly inflating `settled`. */ export function requestSpendRecord( - logCtx: Pick, + logCtx: Pick, attempts: readonly PersistedUsageAttempt[] | undefined, ): PersistedRequestSpend | undefined { const rows = attempts ?? []; const budget = logCtx.executionBudget; const reasons = [...new Set( - (logCtx.affinityMoveReasons ?? (logCtx.affinityReason ? [logCtx.affinityReason] : [])) + (logCtx.affinityMoveReasons ?? (logCtx.affinity !== "unbound" && logCtx.affinityReason ? [logCtx.affinityReason] : [])) .filter(isKnownAffinityReason), )]; if (rows.length === 0 && !budget && reasons.length === 0) return undefined; @@ -1277,11 +1277,11 @@ export function noteAffinityMove( /** * Copy the selection's own affinity decision onto the request log. Both Pool-served kinds carry - * one: a stored Pool account and the main login rotating as `main-pool` (#4546, #7799). A move - * then reads as an event with its cause instead of something inferred from account labels. + * one: a stored Pool account and the main login rotating as `main-pool` (#4546, #7799). A keyed + * caller-owned main can also carry one. A move reads as an event with its cause. */ export function noteCodexSelectionAffinity(logCtx: RequestLogContext, authCtx: CodexAuthContext): void { - const decision = authCtx.kind === "pool" || authCtx.kind === "main-pool" ? authCtx.affinityDecision : undefined; + const decision = authCtx.affinityDecision; if (!decision) return; logCtx.affinity = decision.move; logCtx.affinityReason = decision.reason; diff --git a/src/server/responses/account-change-state.ts b/src/server/responses/account-change-state.ts index 95889b91db4..c76130dfb21 100644 --- a/src/server/responses/account-change-state.ts +++ b/src/server/responses/account-change-state.ts @@ -10,6 +10,7 @@ * The issuer association lives next to thread affinity in `src/codex/routing.ts`. */ +import { conversationStateAccountId } from "../../codex/request-owned-affinity"; import type { CodexAuthContext } from "../../codex/auth-context"; import { peekConversationStateIssuer, @@ -216,14 +217,20 @@ export function scrubUnportableConversationStateInPlace(body: unknown): boolean return changed; } +/** Preserve a Pool context's absent key instead of borrowing the raw request key. */ +export function compactAlternateAffinityKey(authCtx: CodexAuthContext, rawKey: () => string | undefined): string | undefined { + return authCtx.kind === "main" ? (authCtx.affinityKey ?? authCtx.conversationStateKey ?? rawKey()) : authCtx.affinityKey ?? authCtx.conversationStateKey; +} + export function conversationStateBindingFromAuth( authCtx: CodexAuthContext, fallbackAffinityKey?: string | null, ): { accountId: string; bindingKey: string } | null { - if (authCtx.kind !== "pool" && authCtx.kind !== "main-pool") return null; - const bindingKey = authCtx.affinityKey ?? fallbackAffinityKey ?? undefined; - if (!bindingKey || !authCtx.accountId) return null; - return { accountId: authCtx.accountId, bindingKey }; + const accountId = conversationStateAccountId(authCtx); + if (!accountId) return null; + const bindingKey = authCtx.affinityKey ?? authCtx.conversationStateKey ?? fallbackAffinityKey ?? undefined; + if (!bindingKey) return null; + return { accountId, bindingKey }; } export function rememberServingConversationStateIssuer( diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 9484250829f..01f90d542be 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -1,3 +1,4 @@ +import { conversationStateAccountId, releaseRequestOwnedMainOnOutcome, isKeyedRequestOwnedMain } from "../../codex/request-owned-affinity"; import { CodexInstallationIdUnavailableError } from "../../codex/installation-id"; import { isolateCodexPoolClientMetadata } from "../../codex/forward-transport-headers"; import { captureAccountAutoPauseAttempt, wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; @@ -97,6 +98,7 @@ import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-windo import { applyAccountChangeConversationStateScrub, conversationStateBindingFromAuth, + compactAlternateAffinityKey, accountChangeFileReferenceRefusal, rememberServingConversationStateIssuer, conversationCarriesUploadedFiles, @@ -1084,8 +1086,12 @@ export async function handleResponsesCompact( } = {}, ) => { // A gateway response is not evidence about any Codex account. - if (gatewayTransport || !usesCodexForwardPoolAuth(ctx, route.provider)) return; + if (gatewayTransport) return; if ((outcome === 401 || outcome === 403) && upstream && wasPermanentAccountRefusal(upstream)) return; + if (!usesCodexForwardPoolAuth(ctx, route.provider)) { + if (isKeyedRequestOwnedMain(ctx) && typeof outcome === "number" && (!upstream || !isNonReplayableResponse(upstream)) && isCanonicalOpenAiForwardProvider(route.provider)) releaseRequestOwnedMainOnOutcome(ctx, outcome, selectedModelId); + return; + } recordCodexUpstreamOutcome(config, ctx.accountId, outcome, { ...meta, ...codexFailureWindowOutcomeMeta(ctx), @@ -1428,9 +1434,7 @@ export async function handleResponsesCompact( const scrubCompactStateForAlternate = (alternateAuthCtx: CodexAuthContext): void => { const binding = conversationStateBindingFromAuth( alternateAuthCtx, - (authCtx.kind === "pool" || authCtx.kind === "main-pool") - ? authCtx.affinityKey - : codexPoolAffinityKey(req.headers), + compactAlternateAffinityKey(authCtx, () => codexPoolAffinityKey(req.headers)), ); if (!binding) return; for (const body of [raw, compactBody]) { @@ -1438,7 +1442,7 @@ export async function handleResponsesCompact( body, bindingKey: binding.bindingKey, servingAccountId: binding.accountId, - priorAccountId: authCtx.accountId, + priorAccountId: conversationStateAccountId(authCtx), logCtx, }); } diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 3edcc235521..f48f5cf7b82 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -1,3 +1,4 @@ +import { conversationStateAccountId, releaseRequestOwnedMainOnOutcome, isKeyedRequestOwnedMain } from "../../codex/request-owned-affinity"; import { CodexInstallationIdUnavailableError } from "../../codex/installation-id"; import { captureAccountAutoPauseAttempt, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; import type { OcxConfig, OcxProviderConfig, OcxParsedRequest } from "../../types"; @@ -912,9 +913,7 @@ export async function retryCodexPoolOnAlternateAccount( { const binding = conversationStateBindingFromAuth( retryAuthCtx, - firstAuthCtx.kind === "pool" || firstAuthCtx.kind === "main-pool" - ? firstAuthCtx.affinityKey - : undefined, + firstAuthCtx.affinityKey ?? firstAuthCtx.conversationStateKey, ); if (binding) { applyAccountChangeConversationStateScrub({ @@ -925,7 +924,7 @@ export async function retryCodexPoolOnAlternateAccount( && parsed._nativeReasoningRetention?.accountSwitch === true, bindingKey: binding.bindingKey, servingAccountId: binding.accountId, - priorAccountId: firstAuthCtx.accountId, + priorAccountId: conversationStateAccountId(firstAuthCtx), logCtx, }); } @@ -1164,7 +1163,14 @@ export function codexForwardTerminalOutcomeRecorder( modelId?: string, logCtx?: RequestLogContext, ): ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined { - if (!usesCodexForwardPoolAuth(authCtx, provider)) return undefined; + if (!usesCodexForwardPoolAuth(authCtx, provider)) { + if (!isKeyedRequestOwnedMain(authCtx) || !isCanonicalOpenAiForwardProvider(provider)) return undefined; + return (status, httpStatusOverride) => { + const quota = [httpStatusOverride, logCtx?.terminalHttpStatus].find(value => value === 429 || value === 402); + const outcome = status === "completed" || (status === "incomplete" && quota === undefined) ? 200 : (quota ?? httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502); + releaseRequestOwnedMainOnOutcome(authCtx, outcome, modelId); + }; + } const windowMeta = codexFailureWindowOutcomeMeta(authCtx); return (status, httpStatusOverride) => { const quotaStatus = [httpStatusOverride, logCtx?.terminalHttpStatus] diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 279d6ed03c7..8b7bba1ecc5 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -1,3 +1,4 @@ +import { releaseRequestOwnedMainOnOutcome, isKeyedRequestOwnedMain } from "../../codex/request-owned-affinity"; import { wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; import { hasResponseMediaType } from "../../lib/http-response-semantics"; import { isNativeControlResponse } from "./native-response-control"; @@ -34,6 +35,7 @@ import { codexDenialOutcomeMeta, isFixedCodexAccount, shouldDeferCodexResetDerivedCooldown, + shouldRetryCodexPoolAccountQuota, } from "./core-codex-account"; import { isMainQuotaDispatchLive, isMainQuotaDispatchWsClaimed } from "../../codex/main-account-cache"; import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/account-id"; @@ -571,6 +573,18 @@ export async function deliverPassthroughResponse( }); } } else { + if (terminalBodyWillRecord) { + options.setTerminalOutcomeRecorder?.((status, httpStatusOverride) => { + terminalRecorder?.(status, httpStatusOverride); + options.onNativePassthroughTerminal?.(status); + }); + } + if (isKeyedRequestOwnedMain(admissionState.authCtx) && !terminalBodyWillRecord && !upstreamResponse.ok && !isNonReplayableResponse(upstreamResponse) + && !isReplayRefusalResponse(upstreamResponse) && !wasPermanentAccountRefusal(upstreamResponse) && isCanonicalOpenAiForwardProvider(route.provider)) { + const outcome = upstreamResponse.status >= 500 && await shouldRetryCodexPoolAccountQuota(upstreamResponse, options.abortSignal) + ? 429 : upstreamResponse.status; + releaseRequestOwnedMainOnOutcome(admissionState.authCtx, outcome, route.modelId); + } // The WS observer is the only plain-main publisher for a WebSocket exchange; // a prelude projection carries the prelude snapshot, not fresh evidence. // The dispatch claim is authoritative because downstream wrappers can replace the Response. diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index 8a7c561933a..065c3f66357 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -53,6 +53,7 @@ import { } from "./encrypted-payload"; import { codexPoolAffinityKey, + codexRequestAffinityKey, previewCodexPoolLineage, applyCodexAuthContextToProvider, releaseCodexAuthContextProbeLease, @@ -958,10 +959,10 @@ export async function prepareResponsesRequest( // // "The same" means both halves of the question the final resolution asks. The Authorization // it will be given, because the lineage scope is an HMAC of exactly that header; and its own - // Pool-state predicate, because a fixed account selector and a request-owned credential - // deliberately create no affinity at all -- previewing a family binding for one of those would - // hand model fallback an account this request can never authenticate as. Read-only: the record - // is written by the resolution that binds, never by a preview that may own no Pool state. + // Pool-state predicate: fixed selectors and request-owned credentials never use stored-credential + // lineage. An opt-in request-owned key has its own namespace; previewing a stored family binding + // would hand model fallback an account this request will not use. Preview is read-only: only + // eligible stored-credential resolutions record lineage. const poolLineage = previewCodexPoolLineage(previewAuthHeaders, options.codexAuthPolicy ?? config, { accountId: route.codexAccountId, modelId: route.modelId, @@ -1000,7 +1001,7 @@ export async function prepareResponsesRequest( }); const fallbackNow = Date.now(); subagentFallbackAccountPreview = (modelId, previewNow, modelEligibleAccountIds) => previewCodexAccountForRequest( - poolAffinityKey, + codexRequestAffinityKey(previewAuthHeaders, config, options.codexAuthPolicy ?? config, { accountId: route.codexAccountId, modelId, admission: options.admission, requestScopedMainCredential: previewRequestScopedMainCredential }) ?? null, config, previewNow, codexQuotaScopeForModel(modelId), @@ -1218,7 +1219,7 @@ export async function prepareResponsesRequest( // and then fail closed at final auth — the same class of stale-selection bug as // the quota scope, one layer over. subagentFallbackAccountPreview = (modelId, previewNow, modelEligibleAccountIds) => previewCodexAccountForRequest( - poolAffinityKey, + codexRequestAffinityKey(recoveryAuthHeaders, config, options.codexAuthPolicy ?? config, { accountId: route.codexAccountId, modelId, admission: options.admission, requestScopedMainCredential: recoveryRequestScopedMainCredential }) ?? null, config, previewNow, codexQuotaScopeForModel(modelId), @@ -1238,7 +1239,7 @@ export async function prepareResponsesRequest( }), }, modelId, - poolLineage, + previewCodexPoolLineage(recoveryAuthHeaders, options.codexAuthPolicy ?? config, { accountId: route.codexAccountId, modelId, admission: options.admission, requestScopedMainCredential: recoveryRequestScopedMainCredential }), ); const recoveryPreviewAccountId = subagentFallbackAccountPreview( parsed.modelId, diff --git a/src/types/config.ts b/src/types/config.ts index 10fcdcff171..1bc480ef5c4 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1851,6 +1851,8 @@ export interface OcxWebSearchSidecarConfig { export interface OcxCodexPoolConfig { /** Start idle Codex windows when the pool is initialized. */ startIdleWindows?: boolean; + /** Enable process-local affinity for caller-owned credentials; defaults off. */ + requestOwnedAffinity?: boolean; /** * Plan keys ordinary rotation skips, matched case-insensitively against the plan stored on each * account. Absent or empty means no policy. diff --git a/src/usage/log.ts b/src/usage/log.ts index 51da4e7ee68..8a0f9ca5e1c 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -507,10 +507,10 @@ export function isKnownTerminalSource(value: unknown): value is NonNullable>([ - "reused", "held", "detour", "rebound", "new_bind", "cleared", + "reused", "held", "detour", "rebound", "new_bind", "cleared", "unbound", ]); const KNOWN_AFFINITY_REASONS = new Set>([ - "healthy", "quota_headroom", "quota_refusal", "transient", "transient_hold_expired", + "healthy", "no_conversation", "request_owned", "quota_headroom", "quota_refusal", "transient", "transient_hold_expired", "unusable", "paused", "plan_excluded", "credits_off", "cooldown", "quota_avoided", "generation", "expired", "model_lane", ]); diff --git a/structure/INDEX.md b/structure/INDEX.md index 985ab19c7ec..5e9f69f973e 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -70,6 +70,7 @@ The wire surfaces a client actually talks to. | [`data-planes/chat-reasoning-details.md`](data-planes/chat-reasoning-details.md) | The reasoning_details carrier that lets Chat Completions clients round-trip signed Anthropic thinking across tool steps (#7429). | | [`data-planes/protocol-paths.md`](data-planes/protocol-paths.md) | Shared protocol vocabulary, declared feature dispositions, the ingress-by-upstream baseline, plan/trace shapes, and protocol settings. | | [`providers/codex-installation-identity.md`](providers/codex-installation-identity.md) | Per-account Pool wire installation identity, private secret, existence gates and fail-closed delivery. | +| [`providers/codex-request-owned-affinity.md`](providers/codex-request-owned-affinity.md) | Opt-in conversation affinity for a caller's own ChatGPT bearer in Pool mode, honest selection-time affinity labels, and the persistence exclusion contract. | | [`fallback-policy.md`](fallback-policy.md) | OCX-11 fallback off: what is a model or provider fallback (and what intra-pool rotation is not), scope precedence, the guarded hop sites, the fallback_refused and home_unreachable contracts, and the ocx.fallback/1 state. | | [`remote-workspace.md`](remote-workspace.md) | Opt-in workspace identity, executor grants, runtime adapters, management and dashboard integration. | | [`remote-link.md`](remote-link.md) | SSH machine-link building blocks: OpenSSH argument policy, ssh_config candidates, tunnel lifecycle reducer, the private link store, and standalone pairing-grant delivery. | @@ -260,7 +261,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/codex/auth-api/account-preservation.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | | `src/codex/auth-api/pool-mode-gate.ts` | [`providers/pool-credit-renewal.md`](providers/pool-credit-renewal.md) | | `src/codex/auth-context-provider.ts` | [`providers/codex-installation-identity.md`](providers/codex-installation-identity.md) | -| `src/codex/auth-context.ts` | [`providers/codex-installation-identity.md`](providers/codex-installation-identity.md) | +| `src/codex/auth-context.ts` | [`providers/codex-installation-identity.md`](providers/codex-installation-identity.md)
[`providers/codex-request-owned-affinity.md`](providers/codex-request-owned-affinity.md) | | `src/codex/catalog/effort.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md)
[`catalog-effort-clamp.md`](catalog-effort-clamp.md) | | `src/codex/catalog/model-hints.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md) | | `src/codex/catalog/ownership.ts` | [`catalog-routed-ownership.md`](catalog-routed-ownership.md) | @@ -290,6 +291,8 @@ A source area can be described by more than one doc, because these docs are orga | `src/codex/installation-id.ts` | [`providers/codex-installation-identity.md`](providers/codex-installation-identity.md) | | `src/codex/provider-routing.ts` | [`codex-provider-routing.md`](codex-provider-routing.md) | | `src/codex/quota.ts` | [`quota-snapshots.md`](quota-snapshots.md) | +| `src/codex/request-owned-affinity.ts` | [`providers/codex-request-owned-affinity.md`](providers/codex-request-owned-affinity.md) | +| `src/codex/request-owned-main-pin.ts` | [`providers/codex-request-owned-affinity.md`](providers/codex-request-owned-affinity.md) | | `src/codex/routing/selection.ts` | [`providers/codex-round-robin-placement.md`](providers/codex-round-robin-placement.md) | | `src/codex/runtime.ts` | [`catalog-effort-clamp.md`](catalog-effort-clamp.md) | | `src/codex/shim.ts` | [`codex-cli-update.md`](codex-cli-update.md) | diff --git a/structure/manifest.json b/structure/manifest.json index 5c88ec2f184..494781fd41e 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -711,6 +711,13 @@ "scope": "Per-account Pool wire installation identity, private secret, existence gates and fail-closed delivery.", "documents": ["src/codex/installation-id.ts", "src/codex/auth-context.ts", "src/codex/auth-context-provider.ts", "src/codex/forward-transport-headers.ts", "src/adapters/openai-responses/transport.ts", "src/adapters/openai-responses/passthrough.ts", "src/server/ws-bridge.ts", "src/server/responses/codex-ws-request.ts", "src/server/responses/codex-ws-pool.ts", "src/server/responses/compact.ts", "src/server/responses/passthrough-build-error.ts", "src/server/context-history.ts", "src/server/search.ts", "src/server/images.ts", "src/server/live.ts", "src/server/audio-live.ts", "src/server/audio-dictation.ts", "src/server/index/serve-options.ts", "src/server/index/websocket-handler.ts", "src/server/index/live-sideband.ts", "src/server/claude-messages.ts", "src/codex/affinity-debug.ts"] }, + { + "path": "providers/codex-request-owned-affinity.md", + "tier": 3, + "title": "Codex Request-Owned Affinity", + "scope": "Opt-in conversation affinity for a caller's own ChatGPT bearer in Pool mode, honest selection-time affinity labels, and the persistence exclusion contract.", + "documents": ["src/codex/request-owned-affinity.ts", "src/codex/request-owned-main-pin.ts", "src/codex/auth-context.ts"] + }, { "path": "providers/openai-accounts.md", "tier": 4, diff --git a/structure/providers/codex-request-owned-affinity.md b/structure/providers/codex-request-owned-affinity.md new file mode 100644 index 00000000000..d9945fff821 --- /dev/null +++ b/structure/providers/codex-request-owned-affinity.md @@ -0,0 +1,74 @@ +# Codex Request-Owned Affinity + +A Codex request that carries the caller's own forwardable ChatGPT bearer in Pool mode is +request-owned (`codexRouteCredentialOwnership` in `src/server/responses/core-auth.ts`). Since #5019 that +bearer makes main an ordinary Pool candidate next to the stored accounts, while a stored-account pick is +served from that account's stored credential. This document owns how such a request may keep one account +per conversation, and how every affinity decision reports whether a binding was actually stored (#8080). + +## Ownership invariant + +By default a request-owned bearer owns no Pool affinity, lineage, persistence, entitlement cache, health +state or shared-active state: `codexRequestAffinityKey` in `src/codex/auth-context.ts` returns no key, so +every turn is placed by the strategy again. + +With `codexPool.requestOwnedAffinity: true` the request may own a process-local binding from the pair +(exact Authorization value, conversation key) to the account that served it. The key is +`request-owned:v1:` followed by an HMAC-SHA256 under a process-random secret +(`src/codex/request-owned-affinity.ts`), so: + +- two requests share a binding only when they present the byte-identical Authorization value and the same + conversation key; a stored-credential request's raw `app:` key cannot collide with it; +- a binding to `__main__` is always served from the presenting request's own bearer, so no caller is ever + served with another caller's credential; a binding to a stored account names an account ordinary + selection could already give this caller, and every reuse re-checks eligibility; +- a ChatGPT account-id claim is never the principal: it is header-suppliable and shared by workspace + members, so a rotated bearer simply starts a new binding; +- a restart makes every key unreproducible. + +Fixed account selectors and Reserve get no key in final auth or in either preview. Request-owned traffic +never records lineage; key derivation may read only the caller's own lineage scope, which that traffic +never writes, so session-less multi-generation subagent chains key per thread. + +The manual-pin exception for a request-owned bearer (`requestOwnedMainPinState`, #3166, #4850) lives in +`src/codex/request-owned-main-pin.ts`, read-free by construction and shared by final auth and both previews. + +## Labels + +`honestAffinityDecision` runs synchronously when selection returns. A `new_bind` or `rebound` decision +stays only when the ordinary or model-detour entry for the key names the selected account; otherwise +`new_bind` becomes `unbound` and `rebound` becomes `cleared`. The `unbound` reason is `request_owned` when +the key was withheld for a request-owned bearer, `no_conversation` when no key exists for another reason, +and the original reason when a key existed but nothing was stored. The label describes selection time, not +dispatch. `requestSpendRecord` does not count `unbound` as a move reason. A caller-main refusal +releases only the exact ordinary and model-detour entries captured at selection, so a delayed outcome +cannot delete a later binding. The snapshot is opaque and process-local; only an actual release records +a pending reason. + +## Caller-owned main and carried state + +Request-owned Pool contexts and caller-owned main contexts carry a process-local `conversationStateKey` +separate from the optional routing `affinityKey`. The issuer key is derived even when affinity is off +and on the manual-pin path; it is never passed to the routing selector. For conversation-state issuer +purposes a main context with either key is `__main__` (`conversationStateAccountId`). Pool to main and +main to Pool moves, including regular retry and compact alternate paths, scrub `previous_response_id` +when the issuer differs. This also applies with affinity off: those requests now record a bearer-scoped +issuer instead of leaving main untracked or using the stored-credential raw conversation key. Fixed +account selectors and Reserve do not receive a request-owned issuer key. Manual account selection +clears routing bindings while retaining issuer records, so the selected account can detect carried +state from the previous account: `clearThreadAccountMap({ preserveConversationStateIssuers: true })` +in `src/codex/routing/active-account.ts` retains issuers; a full reset clears both. Fixed-account and +other stored-credential traffic keeps the shared stored-credential issuer namespace; its `app:` key +is not caller-scoped by design and is unchanged by this feature. + +## Persistence contract + +Persistence of thread affinity must, for every key where `isRequestOwnedAffinityKey` is true: never register +an anchor for it, never serialize its thread rows (any scope, including model detours) or issuer rows, and +never adopt a restored row into it. The conversation-state key carries the same prefix, so the predicate +covers issuer records too. + +## Tests + +`tests/codex-integration/codex-request-owned-affinity.test.ts`, `tests/codex-integration/reserve-auth-context.test.ts`, +`tests/responses/compact-transient-alternate.test.ts`, `tests/responses/responses-alternate-main-cancellation.test.ts`. diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 8f11ab67151..0eafaec4b20 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -561,7 +561,7 @@ A keyring-backed Codex request can carry its own forwardable ChatGPT bearer whil in Pool mode. When the effective manual pin is `__main__`, main is not paused, and its cached quota still has headroom, auth resolution validates the caller bearer's own gated-model roster and uses that request-owned credential before stored-Pool selection. The credential never enters Pool persistence, -affinity, entitlement cache, or health state, and this decision never reads the physical main credential. +entitlement cache, or health state, nor by default affinity ([request-owned affinity](codex-request-owned-affinity.md)); this never reads the physical main credential. If the caller lacks the requested model, a stored-account model detour may serve the request without clearing the healthy shared main pin. With quota-strategy cache affinity, the same detour preserves an ordinary added-account binding and shared selection beyond the proactive-switch threshold until genuine @@ -574,7 +574,7 @@ A forwardable request-owned bearer IS main's live credential for that request sent if selection named main — so `requestOwnedMainCredentialIsLive` answers the pool-liveness question yes, and main is compared against the stored accounts on the operator's own usage, priority and reset ordering. Selecting main then serves it from the caller's credential without claiming, -reading, reconciling or priming the stored profile, and without owning affinity or health state. +reading, reconciling or priming the stored profile, and without owning health state or (by default) affinity. A main that wins only through that bearer is also never recorded as the shared active account: the request resolves to main, while the persisted and runtime shared active account remain unchanged. Answering that question with the pin predicate instead scored main `main_credential_unavailable` on diff --git a/tests/codex-integration/codex-main-rotation.test.ts b/tests/codex-integration/codex-main-rotation.test.ts index 4c9c86665e3..0f5e59702cd 100644 --- a/tests/codex-integration/codex-main-rotation.test.ts +++ b/tests/codex-integration/codex-main-rotation.test.ts @@ -170,7 +170,7 @@ describe("main account rotation (Option A)", () => { identityKey: expect.stringMatching(/^[a-f0-9]{64}$/), identityGeneration: expect.any(Number), }, - affinityDecision: { move: "new_bind", reason: "healthy" }, + affinityDecision: { move: "unbound", reason: "no_conversation" }, }); expect(isCodexAuthContextUsable(ctx, config)).toBe(true); const headers = headersForCodexAuthContext(new Headers(), ctx); @@ -203,7 +203,7 @@ describe("main account rotation (Option A)", () => { identityKey: expect.stringMatching(/^[a-f0-9]{64}$/), identityGeneration: expect.any(Number), }, - affinityDecision: { move: "new_bind", reason: "healthy" }, + affinityDecision: { move: "unbound", reason: "no_conversation" }, }); expect(isCodexAccountInCooldown(MAIN_CODEX_ACCOUNT_ID)).toBe(false); expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); diff --git a/tests/codex-integration/codex-request-owned-affinity.test.ts b/tests/codex-integration/codex-request-owned-affinity.test.ts new file mode 100644 index 00000000000..dbc658d59aa --- /dev/null +++ b/tests/codex-integration/codex-request-owned-affinity.test.ts @@ -0,0 +1,579 @@ +import { describe, expect, test } from "bun:test"; +import { bindThreadAffinity, clearThreadAccountMap } from "../../src/codex/routing/thread-affinity"; +import { requestSpendRecord } from "../../src/server/request-log"; + +describe("request-owned affinity", () => { + test("namespaces bindings by exact bearer and conversation", async () => { + const { requestOwnedAffinityKey, isRequestOwnedAffinityKey } = await import("../../src/codex/request-owned-affinity"); + const a = new Headers({ authorization: "Bearer alpha", "session-id": "session", "thread-id": "thread" }); + const b = new Headers({ authorization: "Bearer beta", "session-id": "session", "thread-id": "thread" }); + const key = requestOwnedAffinityKey(a)!; + expect(isRequestOwnedAffinityKey(key)).toBe(true); + expect(key).toBe(requestOwnedAffinityKey(a)); + expect(key).not.toBe(requestOwnedAffinityKey(b)); + expect(key).not.toContain("alpha"); + expect(key).not.toContain("session"); + }); + test("selection labels claim a bind only when the chosen account is bound", async () => { + const { honestAffinityDecision } = await import("../../src/codex/request-owned-affinity"); + clearThreadAccountMap(); + const decision = { move: "new_bind", reason: "healthy" } as const; + expect(honestAffinityDecision(decision, undefined, "account", undefined, undefined, true)).toEqual({ move: "unbound", reason: "request_owned" }); + expect(honestAffinityDecision(decision, undefined, "account", undefined, undefined, false)).toEqual({ move: "unbound", reason: "no_conversation" }); + expect(honestAffinityDecision(decision, "key", "account")).toEqual({ move: "unbound", reason: "healthy" }); + bindThreadAffinity("key", "__main__", Date.now()); + expect(honestAffinityDecision(decision, "key", "__main__")).toEqual(decision); + clearThreadAccountMap(); + }); + test("unbound reasons do not become spend move reasons", () => { + expect(requestSpendRecord({ affinity: "unbound", affinityReason: "healthy" }, [])).toBeUndefined(); + expect(requestSpendRecord({ affinity: "unbound", affinityReason: "request_owned" }, [])).toBeUndefined(); + expect(requestSpendRecord({ affinity: "unbound", affinityReason: "request_owned", affinityMoveReasons: ["quota_refusal"] }, [])?.moveReasons).toEqual(["quota_refusal"]); + }); +}); + +test("opt-in main selection binds and reuses a caller-scoped key", async () => { + const { resolveCodexAuthContext, codexRequestAffinityKey, codexPoolAffinityKey } = await import("../../src/codex/auth-context"); + const { getThreadAffinity, clearThreadAccountMap } = await import("../../src/codex/routing/thread-affinity"); + const { conversationStateBindingFromAuth } = await import("../../src/server/responses/account-change-state"); + const { isRequestOwnedAffinityKey } = await import("../../src/codex/request-owned-affinity"); + const config = { providers: {}, codexAccounts: [], accountPoolStrategy: "round-robin", codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer alpha", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true, primeCodexPoolQuotas: async () => {} }; + clearThreadAccountMap(); + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision).toEqual({ move: "new_bind", reason: "healthy" }); + expect(second.affinityDecision).toEqual({ move: "reused", reason: "healthy" }); + expect(first.affinityKey).toBe(codexRequestAffinityKey(headers, config, config, options)); + expect(isRequestOwnedAffinityKey(first.affinityKey!)).toBe(true); + expect(first.affinityKey).not.toBe(codexPoolAffinityKey(headers)); + expect(getThreadAffinity(codexPoolAffinityKey(headers)!)).toBeUndefined(); + expect(conversationStateBindingFromAuth(first)).toEqual({ accountId: "__main__", bindingKey: first.affinityKey }); + clearThreadAccountMap(); +}); + +test("preview derivation excludes fixed selectors and preserves parent fallback", async () => { + const { codexRequestAffinityKey, codexPoolAffinityKey } = await import("../../src/codex/auth-context"); + const config = { providers: {}, codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer alpha", "session-id": "s", "thread-id": "t" }); + const owned = { requestScopedMainCredential: true }; + expect(codexRequestAffinityKey(headers, config, config, { ...owned, accountId: "__main__" })).toBeUndefined(); + expect(codexRequestAffinityKey(headers, config, config, { ...owned, accountId: "pool" })).toBeUndefined(); + const reserveConfig = { ...config, codexDesktopAuthless: true }; + expect(codexRequestAffinityKey(headers, reserveConfig, reserveConfig, { ...owned, modelId: "gpt-reserve", admission: { source: "loopback" } })).toBeUndefined(); + expect(codexRequestAffinityKey(headers, config, config, {})).toBe(codexPoolAffinityKey(headers)); + expect(codexRequestAffinityKey(new Headers({ authorization: "Bearer alpha" }), config, config, owned)).toBeUndefined(); + const long = "x".repeat(513); + expect(codexRequestAffinityKey(new Headers({ authorization: "Bearer alpha", "session-id": long, "thread-id": "t" }), config, config, owned)).toBeUndefined(); + expect(codexRequestAffinityKey(new Headers({ authorization: "Bearer alpha", "session-id": long, "x-codex-parent-thread-id": "parent" }), config, config, owned)).toBeDefined(); +}); + +test("bound callers remain isolated and never record lineage", async () => { + const { resolveCodexAuthContext, codexPoolAffinityKey } = await import("../../src/codex/auth-context"); + const { codexLineageScopeKey, codexThreadLineageLookup, clearCodexThreadLineageForTests } = await import("../../src/codex/lineage"); + const { getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const config = { providers: {}, codexAccounts: [], accountPoolStrategy: "round-robin", codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const a = new Headers({ authorization: "Bearer one", "session-id": "s", "thread-id": "t" }); + const b = new Headers({ authorization: "Bearer two", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true }; + clearThreadAccountMap(); clearCodexThreadLineageForTests(); + const first = await resolveCodexAuthContext(a, config, "pool", options); + const second = await resolveCodexAuthContext(a, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const other = await resolveCodexAuthContext(b, config, "pool", options); + expect(other.affinityDecision?.move).toBe("new_bind"); + expect(other.affinityKey).not.toBe(first.affinityKey); + expect((await resolveCodexAuthContext(a, config, "pool", options)).affinityDecision?.move).toBe("reused"); + expect(getThreadAffinity(codexPoolAffinityKey(a)!)).toBeUndefined(); + expect(codexThreadLineageLookup(codexPoolAffinityKey(a)!, codexLineageScopeKey(a))).toBeUndefined(); + clearThreadAccountMap(); clearCodexThreadLineageForTests(); +}); + +test("schema validates the default-off flag", async () => { + const { codexPoolSchema } = await import("../../src/config/schema/leaf-validators"); + expect(codexPoolSchema.safeParse({ requestOwnedAffinity: true }).success).toBe(true); + expect(codexPoolSchema.safeParse({ requestOwnedAffinity: "true" }).success).toBe(false); +}); + +test("usage log round trips the new affinity values", async () => { + const { mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { removeTreeWithRetry } = await import("../helpers/remove-tree"); + const { addRequestLog, clearRequestLogsForTests } = await import("../../src/server/request-log"); + const { readUsageEntries, resetUsageReadCacheForTests } = await import("../../src/usage/log"); + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-affinity-")); + process.env.OPENCODEX_HOME = home; + clearRequestLogsForTests(); resetUsageReadCacheForTests(); + try { + for (const reason of ["request_owned", "no_conversation"] as const) addRequestLog({ + requestId: reason, timestamp: Date.now(), provider: "openai", model: "test", status: 200, + durationMs: 1, affinity: "unbound", affinityReason: reason, + }); + expect(readUsageEntries().map(row => [row.affinity, row.affinityReason])).toEqual([ + ["unbound", "request_owned"], ["unbound", "no_conversation"], + ]); + } finally { + clearRequestLogsForTests(); resetUsageReadCacheForTests(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); + +test("stored selection without a key labels the missing binding honestly", async () => { + const { mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { removeTreeWithRetry } = await import("../helpers/remove-tree"); + const { saveCodexAccountCredential } = await import("../../src/codex/account-store"); + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const home = mkdtempSync(join(tmpdir(), "ocx-owned-selection-")); + const oldHome = process.env.OPENCODEX_HOME; + const oldCodex = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + try { + saveCodexAccountCredential("a", { accessToken: "test-access", refreshToken: "test-refresh", expiresAt: Date.now() + 3600000, chatgptAccountId: "test-account" }); + const config = { providers: {}, codexAccounts: [{ id: "a", email: "a@example.test", isMain: false }], activeCodexAccountId: "a", pausedCodexAccountIds: ["__main__"] } as import("../../src/types").OcxConfig; + const options = { primeCodexPoolQuotas: async () => {} }; + clearThreadAccountMap(); + const noConversation = await resolveCodexAuthContext(new Headers(), config, "pool", options); + expect(noConversation.affinityDecision).toEqual({ move: "unbound", reason: "no_conversation" }); + const caller = new Headers({ authorization: "Bearer test", "session-id": "s", "thread-id": "t" }); + const off = await resolveCodexAuthContext(caller, config, "pool", { ...options, requestScopedMainCredential: true }); + expect(off.affinityDecision).toEqual({ move: "unbound", reason: "request_owned" }); + expect(off.affinityKey).toBeUndefined(); + const { codexPoolAffinityKey } = await import("../../src/codex/auth-context"); + const { rememberServingConversationStateIssuer, conversationStateBindingFromAuth } = await import("../../src/server/responses/account-change-state"); + const { peekConversationStateIssuer } = await import("../../src/codex/routing/thread-affinity"); + expect(off.conversationStateKey).toStartWith("request-owned:v1:"); + expect(off.conversationStateKey).not.toBe(codexPoolAffinityKey(caller)); + rememberServingConversationStateIssuer(off, codexPoolAffinityKey(caller)); + expect(peekConversationStateIssuer(off.conversationStateKey!)).toBe("a"); + expect(peekConversationStateIssuer(codexPoolAffinityKey(caller)!)).toBeUndefined(); + const stored = { kind: "pool", accountId: "other", generation: 0, writerGeneration: 0, accessToken: "test", chatgptAccountId: "test" } as const; + rememberServingConversationStateIssuer(stored, codexPoolAffinityKey(caller)); + expect(conversationStateBindingFromAuth(stored, codexPoolAffinityKey(caller))?.bindingKey).toBe(codexPoolAffinityKey(caller)); + expect(peekConversationStateIssuer(off.conversationStateKey!)).toBe("a"); + config.codexPool = { requestOwnedAffinity: true }; + const bound = await resolveCodexAuthContext(caller, config, "pool", { ...options, requestScopedMainCredential: true }); + const reused = await resolveCodexAuthContext(caller, config, "pool", { ...options, requestScopedMainCredential: true }); + expect(bound.affinityDecision?.move).toBe("new_bind"); + expect(reused.affinityDecision?.move).toBe("reused"); + const fixedPool = await resolveCodexAuthContext(caller, config, "pool", { ...options, requestScopedMainCredential: true, accountId: "a" }); + const fixedMain = await resolveCodexAuthContext(caller, config, "pool", { ...options, requestScopedMainCredential: true, accountId: "__main__" }); + expect(fixedPool.affinityKey).toBeUndefined(); + expect(fixedPool.conversationStateKey).toBeUndefined(); + expect(conversationStateBindingFromAuth(fixedPool, codexPoolAffinityKey(caller))?.bindingKey).toBe(codexPoolAffinityKey(caller)); + expect(fixedMain.affinityKey).toBeUndefined(); + expect(fixedMain.conversationStateKey).toBeUndefined(); + const { bindModelDetourAffinity, getThreadAffinity, getModelDetourAffinity } = await import("../../src/codex/routing/thread-affinity"); + const { honestAffinityDecision } = await import("../../src/codex/request-owned-affinity"); + bindModelDetourAffinity(bound.affinityKey!, "__main__", Date.now(), "detour-model"); + expect(getThreadAffinity(bound.affinityKey!)?.accountId).toBe("a"); + expect(getModelDetourAffinity(bound.affinityKey!, "detour-model")?.accountId).toBe("__main__"); + expect(honestAffinityDecision({ move: "new_bind", reason: "model_lane" }, bound.affinityKey, "__main__", undefined, "detour-model")).toEqual({ move: "new_bind", reason: "model_lane" }); + config.pausedCodexAccountIds = ["a"]; + const moved = await resolveCodexAuthContext(caller, config, "pool", { ...options, requestScopedMainCredential: true }); + expect(moved.kind).toBe("main"); + expect(moved.affinityDecision?.move).toBe("rebound"); + } finally { + clearThreadAccountMap(); + if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; + if (oldCodex === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = oldCodex; + removeTreeWithRetry(home); + } +}); + +test("keyed main issuer participates in account-change scrub and file refusal", async () => { + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { conversationStateBindingFromAuth, rememberServingConversationStateIssuer, applyAccountChangeConversationStateScrub, accountChangeFileReferenceRefusal } = await import("../../src/server/responses/account-change-state"); + const config = { providers: {}, codexAccounts: [], codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer issuer", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true }; + clearThreadAccountMap(); + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + rememberServingConversationStateIssuer(second); + const binding = conversationStateBindingFromAuth(second)!; + const body = { previous_response_id: "opaque" }; + expect(applyAccountChangeConversationStateScrub({ body, bindingKey: binding.bindingKey, servingAccountId: "other" })).toBe(true); + expect(body).not.toHaveProperty("previous_response_id"); + expect(accountChangeFileReferenceRefusal({ body: { input: [{ file_id: "opaque" }] }, bindingKey: binding.bindingKey, servingAccountId: "other" })?.status).toBe(400); + clearThreadAccountMap(); +}); + +test("a namespaced binding remains independent across quota scopes", async () => { + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const config = { providers: {}, codexAccounts: [], codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer scopes", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true }; + clearThreadAccountMap(); + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const shared = await resolveCodexAuthContext(headers, config, "pool", { ...options, modelId: "test-model" }); + expect(shared.affinityDecision?.move).toBe("new_bind"); + expect(getThreadAffinity(first.affinityKey!)?.accountId).toBe("__main__"); + expect(getThreadAffinity(first.affinityKey!, "shared")?.accountId).toBe("__main__"); + clearThreadAccountMap(); +}); + +test("substituted admission and injected main credentials use the ordinary key", async () => { + const { codexRouteCredentialOwnership } = await import("../../src/server/responses/core-auth"); + const { resolveCodexAuthContext, codexRequestAffinityKey, codexPoolAffinityKey } = await import("../../src/codex/auth-context"); + const { fakeChatGptJwt } = await import("../helpers/agent-task-recovery"); + const provider = { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", authMode: "forward" } as import("../../src/types").OcxProviderConfig; + const route = { provider, codexAccountMode: "pool" as const }; + const config = { providers: {}, codexAccounts: [], codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: `Bearer ${fakeChatGptJwt("test-account")}`, "session-id": "s", "thread-id": "t" }); + clearThreadAccountMap(); + const first = await resolveCodexAuthContext(headers, config, "pool", { requestScopedMainCredential: true }); + const second = await resolveCodexAuthContext(headers, config, "pool", { requestScopedMainCredential: true }); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const admission = { admission: { source: "bearer" } } as Parameters[3]; + const injected = { stripClaudeMainAuthForNoncanonicalForward: true, trustedClaudeMainAuth: { authorization: headers.get("authorization")! } } as Parameters[3]; + for (const options of [admission, injected]) { + const ownership = codexRouteCredentialOwnership(headers, config, route, options); + expect(ownership.requestScopedMainCredential).toBe(false); + expect(codexRequestAffinityKey(headers, config, config, ownership)).toBe(codexPoolAffinityKey(headers)); + } + clearThreadAccountMap(); +}); + +test("reusing a main binding never reads physical main auth", async () => { + const fs = await import("node:fs"); + const { spyOn } = await import("bun:test"); + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const config = { providers: {}, codexAccounts: [], codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer fenced", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true }; + clearThreadAccountMap(); + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const original = fs.readFileSync as (...args: unknown[]) => unknown; + let mainReads = 0; + const spy = spyOn(fs, "readFileSync").mockImplementation(((...args: unknown[]) => { + if (typeof args[0] === "string" && args[0].endsWith("auth.json")) mainReads++; + return original(...args); + }) as typeof fs.readFileSync); + try { + expect((await resolveCodexAuthContext(headers, config, "pool", options)).affinityDecision?.move).toBe("reused"); + expect(mainReads).toBe(0); + } finally { spy.mockRestore(); clearThreadAccountMap(); } +}); + + +test("compact fixed-account Pool contexts never fall back to the raw key", async () => { + const { compactAlternateAffinityKey } = await import("../../src/server/responses/account-change-state"); + const pool = { kind: "pool", accountId: "a", fixedAccount: true, writerGeneration: 0, generation: 0, accessToken: "test", chatgptAccountId: "test" } as const; + let reads = 0; + const rawKey = () => { reads++; return "raw-key"; }; + expect(compactAlternateAffinityKey(pool, rawKey)).toBeUndefined(); + expect(compactAlternateAffinityKey({ kind: "main", accountId: null, affinityKey: "owned-key" }, rawKey)).toBe("owned-key"); + expect(reads).toBe(0); + expect(compactAlternateAffinityKey({ kind: "main", accountId: null }, rawKey)).toBe("raw-key"); + expect(reads).toBe(1); +}); + +test("six request-owned round-robin turns rotate when off and bind when on", async () => { + const { mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { removeTreeWithRetry } = await import("../helpers/remove-tree"); + const { saveCodexAccountCredential } = await import("../../src/codex/account-store"); + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { clearPoolRotationState } = await import("../../src/codex/pool-rotation"); + const { clearCodexUpstreamHealth } = await import("../../src/codex/routing"); + const home = mkdtempSync(join(tmpdir(), "ocx-owned-rotation-")); + const oldHome = process.env.OPENCODEX_HOME, oldCodex = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + try { + for (const id of ["a", "b"]) saveCodexAccountCredential(id, { accessToken: `access-${id}`, refreshToken: `refresh-${id}`, expiresAt: Date.now() + 3600000, chatgptAccountId: `test-${id}` }); + const config = { providers: {}, codexAccounts: ["a", "b"].map(id => ({ id, email: `${id}@example.test`, isMain: false })), accountPoolStrategy: "round-robin", accountPoolStickyLimit: 1, autoSwitchThreshold: 0 } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer rotation", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true, primeCodexPoolQuotas: async () => {} }; + clearThreadAccountMap(); clearPoolRotationState(); clearCodexUpstreamHealth(); + const off = []; + for (let i = 0; i < 6; i++) off.push(await resolveCodexAuthContext(headers, config, "pool", options)); + expect(new Set(off.map(ctx => ctx.kind === "main" ? "__main__" : ctx.accountId)).size).toBe(3); + for (const ctx of off.filter(ctx => ctx.kind === "pool")) expect(ctx.affinityDecision).toEqual({ move: "unbound", reason: "request_owned" }); + clearThreadAccountMap(); clearPoolRotationState(); + config.codexPool = { requestOwnedAffinity: true }; + const on = []; + for (let i = 0; i < 6; i++) on.push(await resolveCodexAuthContext(headers, config, "pool", options)); + expect(new Set(on.map(ctx => ctx.kind === "main" ? "__main__" : ctx.accountId)).size).toBe(1); + expect(on.map(ctx => ctx.affinityDecision?.move)).toEqual(["new_bind", "reused", "reused", "reused", "reused", "reused"]); + // Model the first preview and a recovery reroute that changes credential ownership. + const { codexRequestAffinityKey } = await import("../../src/codex/auth-context"); + const { previewCodexAccountForRequest } = await import("../../src/codex/routing"); + const firstKey = codexRequestAffinityKey(headers, config, config, { requestScopedMainCredential: true })!; + const recoveryKey = codexRequestAffinityKey(headers, config, config, { requestScopedMainCredential: false })!; + expect(firstKey).not.toBe(recoveryKey); + const firstAccount = on[0]!.kind === "main" ? "__main__" : on[0]!.accountId!; + const otherAccount = firstAccount === "a" ? "b" : "a"; + bindThreadAffinity(recoveryKey, otherAccount, Date.now()); + expect(previewCodexAccountForRequest(firstKey, config, Date.now(), undefined, { requestOwnedMainCredential: true, isMainAccountTokenLive: () => true })).toBe(firstAccount); + expect(previewCodexAccountForRequest(recoveryKey, config, Date.now(), undefined, { isMainAccountTokenLive: () => false })).toBe(otherAccount); + const { resolveCodexAccountForThreadDetailed, codexQuotaScopeForModel } = await import("../../src/codex/routing"); + const { getThreadAffinity, getModelDetourAffinity } = await import("../../src/codex/routing/thread-affinity"); + const selection = { requestOwnedMainCredential: true, isMainAccountTokenLive: () => true }; + const shared = resolveCodexAccountForThreadDetailed(firstKey, config, Date.now(), "shared", selection); + expect(shared.status).toBe("selected"); + expect(shared.affinity?.move).toBe("new_bind"); + const scope = codexQuotaScopeForModel("gpt-reserve")!; + expect(scope).toBe("reserve"); + const reserve = resolveCodexAccountForThreadDetailed(firstKey, config, Date.now(), scope, selection); + expect(reserve.status).toBe("selected"); + expect(reserve.affinity?.move).toBe("new_bind"); + expect(getThreadAffinity(firstKey, scope)).toBeDefined(); + const ordinary = getThreadAffinity(firstKey, "shared")!.accountId; + const alternate = ordinary === "a" ? "b" : "a"; + const detour = resolveCodexAccountForThreadDetailed(firstKey, config, Date.now(), "shared", { ...selection, modelEligibleAccountIds: new Set([alternate]) }, "gpt-daybreak-blue-latest"); + expect(detour).toMatchObject({ status: "selected", accountId: alternate }); + expect(getModelDetourAffinity(firstKey, "gpt-daybreak-blue-latest", "shared")?.accountId).toBe(alternate); + expect(getThreadAffinity(firstKey, "shared")?.accountId).toBe(ordinary); + } finally { + clearThreadAccountMap(); clearPoolRotationState(); clearCodexUpstreamHealth(); + if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; + if (oldCodex === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = oldCodex; + removeTreeWithRetry(home); + } +}); + +test("pinned caller-main keeps the namespaced conversation-state issuer", async () => { + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { conversationStateBindingFromAuth, rememberServingConversationStateIssuer, applyAccountChangeConversationStateScrub, accountChangeFileReferenceRefusal } = await import("../../src/server/responses/account-change-state"); + const cfg = { providers: {}, codexAccounts: [], accountPoolStrategy: "round-robin", codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer pinned", "session-id": "s", "thread-id": "t" }); + clearThreadAccountMap(); + const initial = await resolveCodexAuthContext(headers, cfg, "pool", { requestScopedMainCredential: true }); + const repeat = await resolveCodexAuthContext(headers, cfg, "pool", { requestScopedMainCredential: true }); + expect(initial.affinityDecision?.move).toBe("new_bind"); + expect(repeat.affinityDecision?.move).toBe("reused"); + const pool = { kind: "pool", accountId: "pool-x", writerGeneration: 0, generation: 0, accessToken: "test", chatgptAccountId: "test", affinityKey: initial.affinityKey } as const; + rememberServingConversationStateIssuer(pool); + cfg.activeCodexAccountId = "__main__"; cfg.activeCodexAccountPinned = "__main__"; + const pinned = await resolveCodexAuthContext(headers, cfg, "pool", { requestScopedMainCredential: true }); + expect(pinned.kind).toBe("main"); + expect(pinned.affinityKey).toBeUndefined(); + expect(pinned.conversationStateKey).toBe(initial.affinityKey); + const binding = conversationStateBindingFromAuth(pinned)!; + const body = { previous_response_id: "opaque" }; + expect(applyAccountChangeConversationStateScrub({ body, bindingKey: binding.bindingKey, servingAccountId: binding.accountId })).toBe(true); + expect(accountChangeFileReferenceRefusal({ body: { input: [{ file_id: "opaque" }] }, bindingKey: binding.bindingKey, servingAccountId: binding.accountId })?.status).toBe(400); + rememberServingConversationStateIssuer(pinned); + const reverse = { previous_response_id: "opaque" }; + expect(applyAccountChangeConversationStateScrub({ body: reverse, bindingKey: initial.affinityKey!, servingAccountId: "pool-x" })).toBe(true); + clearThreadAccountMap(); +}); + +test("caller-main issuer and fallback key use conversation state without routing affinity", async () => { + const { conversationStateAccountId } = await import("../../src/codex/request-owned-affinity"); + const { conversationStateBindingFromAuth, compactAlternateAffinityKey } = await import("../../src/server/responses/account-change-state"); + const main = { kind: "main", accountId: null, conversationStateKey: "owned-key" } as const; + expect(conversationStateAccountId(main)).toBe("__main__"); + expect(conversationStateBindingFromAuth(main, "raw")).toEqual({ accountId: "__main__", bindingKey: "owned-key" }); + expect(compactAlternateAffinityKey(main, () => "raw")).toBe("owned-key"); +}); + +test("fallback labels describe the caller-main detour", async () => { + const { callerOwnedFallbackAffinityDecision } = await import("../../src/codex/request-owned-affinity"); + clearThreadAccountMap(); + bindThreadAffinity("fallback-key", "__main__", Date.now()); + expect(callerOwnedFallbackAffinityDecision({ move: "reused", reason: "healthy" }, "fallback-key", undefined, undefined, false, "cooldown")).toEqual({ move: "reused", reason: "healthy" }); + expect(callerOwnedFallbackAffinityDecision({ move: "reused", reason: "healthy" }, undefined, undefined, undefined, true, "cooldown")).toEqual({ move: "unbound", reason: "request_owned" }); + clearThreadAccountMap(); +}); + +test("cooldown arriving after pool selection labels caller-main as a detour", async () => { + const { mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { removeTreeWithRetry } = await import("../helpers/remove-tree"); + const { saveCodexAccountCredential } = await import("../../src/codex/account-store"); + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { recordCodexUpstreamOutcome, clearCodexUpstreamHealth } = await import("../../src/codex/routing"); + const { noteCodexSelectionAffinity } = await import("../../src/server/request-log"); + const home = mkdtempSync(join(tmpdir(), "ocx-owned-cooldown-")); + const oldHome = process.env.OPENCODEX_HOME, oldCodex = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + try { + saveCodexAccountCredential("a", { accessToken: "access-a", refreshToken: "refresh-a", expiresAt: Date.now() + 3600000, chatgptAccountId: "account-a" }); + const config = { providers: {}, codexAccounts: [{ id: "a", label: "a" }], activeCodexAccountId: "a", pausedCodexAccountIds: ["__main__"], autoSwitchThreshold: 0, codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer distinct-caller", "chatgpt-account-id": "distinct-account", "session-id": "s", "thread-id": "t" }); + const options = { requestScopedMainCredential: true, primeCodexPoolQuotas: async () => {} }; + clearThreadAccountMap(); clearCodexUpstreamHealth(); + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); expect(second.affinityDecision?.move).toBe("reused"); + const raced = await resolveCodexAuthContext(headers, config, "pool", { ...options, beginCodexAccountSelection: () => ({ mainProfileDraining: false, claimMainProfile: () => true, release: () => { + recordCodexUpstreamOutcome(config, "a", 429, { retryAfter: "600", fixedAccount: true }); + } }) }); + expect(raced.kind).toBe("main"); + expect(raced.affinityDecision).toEqual({ move: "detour", reason: "cooldown" }); + const log = { provider: "openai", model: "test" }; + noteCodexSelectionAffinity(log, raced); + expect(log).toMatchObject({ affinity: "detour", affinityReason: "cooldown" }); + } finally { + clearThreadAccountMap(); clearCodexUpstreamHealth(); + if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; + if (oldCodex === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = oldCodex; + removeTreeWithRetry(home); + } +}); + +test("manual selection clears affinity but retains carried-state issuers until a full reset", async () => { + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { rememberConversationStateIssuer, peekConversationStateIssuer, getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const { resetCodexRoutingForManualSelection, resetCodexRoutingForClearedSelection } = await import("../../src/codex/routing/active-account"); + const config = { providers: {}, codexAccounts: [], codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer pin-reset", "session-id": "s", "thread-id": "t" }); + clearThreadAccountMap(); + try { + const first = await resolveCodexAuthContext(headers, config, "pool", { requestScopedMainCredential: true }); + const reused = await resolveCodexAuthContext(headers, config, "pool", { requestScopedMainCredential: true }); + expect(first.affinityDecision?.move).toBe("new_bind"); expect(reused.affinityDecision?.move).toBe("reused"); + const key = first.conversationStateKey!; + rememberConversationStateIssuer(key, "pool-a"); + rememberConversationStateIssuer("raw-key", "pool-b"); + expect(getThreadAffinity(key)).toBeDefined(); + resetCodexRoutingForManualSelection("__main__"); + expect(getThreadAffinity(key)).toBeUndefined(); + expect(peekConversationStateIssuer(key)).toBe("pool-a"); + expect(peekConversationStateIssuer("raw-key")).toBe("pool-b"); + clearThreadAccountMap(); + expect(peekConversationStateIssuer(key)).toBeUndefined(); + expect(peekConversationStateIssuer("raw-key")).toBeUndefined(); + } finally { clearThreadAccountMap(); resetCodexRoutingForClearedSelection(); } +}); + +test.each([[401, "unusable"], [403, "unusable"], [402, "quota_refusal"], [429, "quota_refusal"], [503, "transient"]] as const)( + "caller-owned main refusal %s releases namespaced bindings with %s", async (status, reason) => { + const { requestOwnedAffinityKey, releaseRequestOwnedMainOnOutcome, captureRequestOwnedAffinityBindings } = await import("../../src/codex/request-owned-affinity"); + const { bindModelDetourAffinity, getModelDetourAffinity, getThreadAffinity, peekPendingReleaseReason, affinityAfterRelease, rememberConversationStateIssuer, peekConversationStateIssuer } = await import("../../src/codex/routing/thread-affinity"); + const key = requestOwnedAffinityKey(new Headers({ authorization: "Bearer outcome-test", "session-id": "s", "thread-id": "t" }))!; + clearThreadAccountMap(); + bindThreadAffinity(key, "__main__", Date.now(), "shared"); + bindModelDetourAffinity(key, "__main__", Date.now(), "gpt-5.5", "shared"); + rememberConversationStateIssuer(key, "__main__"); + releaseRequestOwnedMainOnOutcome({ kind: "main", accountId: null, affinityKey: key, affinityBindingSnapshot: captureRequestOwnedAffinityBindings(key, "shared", "gpt-5.5") }, status, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")).toBeUndefined(); + expect(getModelDetourAffinity(key, "gpt-5.5", "shared")).toBeUndefined(); + expect(peekPendingReleaseReason(key)).toBe(reason); + expect(peekConversationStateIssuer(key)).toBe("__main__"); + affinityAfterRelease(key, reason); + clearThreadAccountMap(); + }, +); + +test.each([200, 302, 400, 499])("non-refusal %s retains the caller-owned main binding", async status => { + const { requestOwnedAffinityKey, releaseRequestOwnedMainOnOutcome, captureRequestOwnedAffinityBindings } = await import("../../src/codex/request-owned-affinity"); + const { getThreadAffinity, peekPendingReleaseReason } = await import("../../src/codex/routing/thread-affinity"); + const key = requestOwnedAffinityKey(new Headers({ authorization: "Bearer no-refusal", "session-id": "s", "thread-id": "t" }))!; + clearThreadAccountMap(); bindThreadAffinity(key, "__main__", Date.now(), "shared"); + releaseRequestOwnedMainOnOutcome({ kind: "main", accountId: null, affinityKey: key, affinityBindingSnapshot: captureRequestOwnedAffinityBindings(key, "shared", "gpt-5.5") }, status, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")?.accountId).toBe("__main__"); + expect(peekPendingReleaseReason(key)).toBeUndefined(); + clearThreadAccountMap(); +}); + +test("a failed caller-main model detour releases its ordinary and detour entries only", async () => { + const { mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { removeTreeWithRetry } = await import("../helpers/remove-tree"); + const { saveCodexAccountCredential } = await import("../../src/codex/account-store"); + const { requestOwnedAffinityKey, releaseRequestOwnedMainOnOutcome, captureRequestOwnedAffinityBindings } = await import("../../src/codex/request-owned-affinity"); + const { bindModelDetourAffinity, getModelDetourAffinity, getThreadAffinity, peekPendingReleaseReason, affinityAfterRelease } = await import("../../src/codex/routing/thread-affinity"); + const home = mkdtempSync(join(tmpdir(), "ocx-detour-refusal-")); const previous = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = home; + try { + saveCodexAccountCredential("a", { accessToken: "access-a", refreshToken: "refresh-a", expiresAt: Date.now() + 3600000, chatgptAccountId: "account-a" }); + const key = requestOwnedAffinityKey(new Headers({ authorization: "Bearer detour", "session-id": "s", "thread-id": "t" }))!; + clearThreadAccountMap(); + bindThreadAffinity(key, "a", Date.now(), "shared"); + bindModelDetourAffinity(key, "__main__", Date.now(), "gpt-5.5", "shared"); + const context = { kind: "main", accountId: null, affinityKey: key, affinityBindingSnapshot: captureRequestOwnedAffinityBindings(key, "shared", "gpt-5.5") } as const; + releaseRequestOwnedMainOnOutcome(context, 429, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")).toBeUndefined(); + expect(getModelDetourAffinity(key, "gpt-5.5", "shared")).toBeUndefined(); + expect(peekPendingReleaseReason(key)).toBe("quota_refusal"); + affinityAfterRelease(key, "quota_refusal"); + bindThreadAffinity(key, "a", Date.now(), "shared"); + releaseRequestOwnedMainOnOutcome({ ...context, affinityDecision: { move: "reused", reason: "healthy" } }, 429, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")?.accountId).toBe("a"); + expect(peekPendingReleaseReason(key)).toBeUndefined(); + releaseRequestOwnedMainOnOutcome({ ...context, affinityBindingSnapshot: captureRequestOwnedAffinityBindings(key, "shared", "gpt-5.5"), affinityDecision: { move: "detour", reason: "cooldown" } }, 429, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")).toBeUndefined(); + expect(peekPendingReleaseReason(key)).toBe("quota_refusal"); + affinityAfterRelease(key, "quota_refusal"); + const sameTime = Date.now(); + bindThreadAffinity(key, "a", sameTime, "shared"); + bindModelDetourAffinity(key, "__main__", sameTime, "gpt-5.5", "shared"); + const older = { ...context, affinityBindingSnapshot: captureRequestOwnedAffinityBindings(key, "shared", "gpt-5.5") }; + bindThreadAffinity(key, "a", sameTime, "shared"); + bindModelDetourAffinity(key, "__main__", sameTime, "gpt-5.5", "shared"); + releaseRequestOwnedMainOnOutcome(older, 429, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")?.accountId).toBe("a"); + expect(getModelDetourAffinity(key, "gpt-5.5", "shared")?.accountId).toBe("__main__"); + expect(peekPendingReleaseReason(key)).toBeUndefined(); + } finally { clearThreadAccountMap(); if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; removeTreeWithRetry(home); } +}); + +test("caller-main outcome cannot release a stored-credential raw binding", async () => { + const { releaseRequestOwnedMainOnOutcome } = await import("../../src/codex/request-owned-affinity"); + const { getThreadAffinity, peekPendingReleaseReason } = await import("../../src/codex/routing/thread-affinity"); + clearThreadAccountMap(); + bindThreadAffinity("app:stored-binding", "__main__", Date.now(), "shared"); + releaseRequestOwnedMainOnOutcome({ kind: "main", accountId: null, affinityKey: "app:stored-binding" }, 429, "gpt-5.5"); + expect(getThreadAffinity("app:stored-binding", "shared")?.accountId).toBe("__main__"); + expect(peekPendingReleaseReason("app:stored-binding")).toBeUndefined(); + clearThreadAccountMap(); +}); + +test.each(["pool", "main"] as const)("a delayed caller-main failure cannot retire a newer %s binding", async destination => { + const { mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { removeTreeWithRetry } = await import("../helpers/remove-tree"); + const { saveCodexAccountCredential } = await import("../../src/codex/account-store"); + const { resolveCodexAuthContext } = await import("../../src/codex/auth-context"); + const { releaseRequestOwnedMainOnOutcome } = await import("../../src/codex/request-owned-affinity"); + const { getThreadAffinity, peekPendingReleaseReason } = await import("../../src/codex/routing/thread-affinity"); + const home = mkdtempSync(join(tmpdir(), "ocx-binding-race-")); + const oldHome = process.env.OPENCODEX_HOME, oldCodex = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + try { + saveCodexAccountCredential("a", { accessToken: "access-a", refreshToken: "refresh-a", expiresAt: Date.now() + 3600000, chatgptAccountId: "account-a" }); + const config = { providers: {}, codexAccounts: [{ id: "a", label: "a" }], pausedCodexAccountIds: ["a"], codexPool: { requestOwnedAffinity: true } } as import("../../src/types").OcxConfig; + const headers = new Headers({ authorization: "Bearer race-test", "session-id": "s", "thread-id": "t" }); + clearThreadAccountMap(); + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {} }; + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const reused = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.kind).toBe("main"); expect(first.affinityDecision?.move).toBe("new_bind"); expect(reused.affinityDecision?.move).toBe("reused"); + const key = first.affinityKey!; + const sameTime = getThreadAffinity(key, "shared")!.createdAt; + bindThreadAffinity(key, "a", sameTime, "shared"); + if (destination === "main") bindThreadAffinity(key, "__main__", sameTime, "shared"); + const stale = destination === "pool" ? { ...first, affinityDecision: { move: "detour", reason: "cooldown" } as const } : first; + releaseRequestOwnedMainOnOutcome(stale, 429, "gpt-5.5"); + expect(getThreadAffinity(key, "shared")?.accountId).toBe(destination === "pool" ? "a" : "__main__"); + expect(peekPendingReleaseReason(key)).toBeUndefined(); + } finally { + clearThreadAccountMap(); + if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; + if (oldCodex === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = oldCodex; + removeTreeWithRetry(home); + } +}); diff --git a/tests/codex-integration/reserve-auth-context.test.ts b/tests/codex-integration/reserve-auth-context.test.ts index c5ab020d0d8..82302391a85 100644 --- a/tests/codex-integration/reserve-auth-context.test.ts +++ b/tests/codex-integration/reserve-auth-context.test.ts @@ -168,6 +168,24 @@ afterEach(async () => { }); describe("Reserve owned auth admission", () => { + test("opt-in caller Reserve remains keyless even after ordinary affinity is bound", async () => { + const cfg = config(); + cfg.codexPool = { requestOwnedAffinity: true }; + const headers = caller(); + headers.set("session-id", "test-session"); + headers.set("thread-id", "test-thread"); + const options = { requestScopedMainCredential: true, primeCodexPoolQuotas: async () => {} }; + const first = await resolveCodexAuthContext(headers, cfg, "pool", options); + const second = await resolveCodexAuthContext(headers, cfg, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const reserve = await resolveCodexAuthContext(headers, cfg, "pool", { ...reserveOptions, ...options }); + expect(reserve.kind).toBe("main"); + expect(reserve.affinityKey).toBeUndefined(); + expect(reserve.affinityDecision).toBeUndefined(); + const { getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + expect(getThreadAffinity(first.affinityKey!, "reserve")).toBeUndefined(); + }); test("unqualified Reserve pins stored main, requires capability WHAM, and carries private proof", async () => { const cfg = config(); const ctx = await resolveCodexAuthContext(new Headers(), cfg, "direct", reserveOptions); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 59a85e01f92..053e11343bf 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -676,7 +676,8 @@ "codex-pin-drain-projection.test.ts": "codex-integration", "codex-plan.test.ts": "codex-integration", "codex-plugins-doctor.test.ts": "codex-integration", "codex-pool-plan-exclusion.test.ts": "codex-integration", "codex-pool-refresh-backoff.test.ts": "codex-integration", - "codex-pool-request-owned-main.test.ts": "codex-integration", "codex-pool-rotation.test.ts": "codex-integration", + "codex-pool-request-owned-main.test.ts": "codex-integration", + "codex-request-owned-affinity.test.ts": "codex-integration", "codex-pool-rotation.test.ts": "codex-integration", "codex-priority-failback.test.ts": "codex-integration", "codex-idle-window.test.ts": "codex-integration", "codex-round-robin-headroom.test.ts": "codex-integration", "codex-prompt-adopt.test.ts": "codex-integration", "codex-prompt-base-variants.test.ts": "codex-integration", diff --git a/tests/responses/compact-transient-alternate.test.ts b/tests/responses/compact-transient-alternate.test.ts index 4ec20a054ee..92112f605ed 100644 --- a/tests/responses/compact-transient-alternate.test.ts +++ b/tests/responses/compact-transient-alternate.test.ts @@ -301,3 +301,105 @@ describe("native compact moves a surviving transient 5xx to one alternate accoun expect(failures(WORK)).toBe(1); }); }); + +test.each([false, true])("caller-owned compact Pool-to-main retry scrubs continuation (affinity=%s)", async enabled => { + const { fakeChatGptJwt } = await import("../helpers/agent-task-recovery"); + const { rememberServingConversationStateIssuer } = await import("../../src/server/responses/account-change-state"); + const { peekConversationStateIssuer } = await import("../../src/codex/routing/thread-affinity"); + const bearer = fakeChatGptJwt("caller-test"); + const headers = new Headers({ authorization: `Bearer ${bearer}`, "content-type": "application/json", "session-id": "test-session", "thread-id": "test-thread" }); + const config = poolConfig({ codexAccounts: [{ id: WORK, label: WORK }], pausedCodexAccountIds: ["__main__"], codexPool: { requestOwnedAffinity: enabled } }); + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {} }; + const first = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + const second = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe(enabled ? "new_bind" : "unbound"); + expect(second.affinityDecision?.move).toBe(enabled ? "reused" : "unbound"); + expect(first.kind).toBe("pool"); + rememberServingConversationStateIssuer(second); + const bodies: Record[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + sends.push(new Headers(init?.headers).get("authorization")); + bodies.push(JSON.parse(String(init?.body)) as Record); + if (sends.length === 1) config.pausedCodexAccountIds = []; + return sends.length === 1 ? overloadedOnce() : compacted(); + }) as typeof fetch; + const makeRequest = () => new Request("http://localhost/v1/responses/compact", { + method: "POST", headers, + body: JSON.stringify({ model: "gpt-5.5", previous_response_id: "opaque-previous", input: [{ role: "user", content: "continue" }] }), + }); + const response = await handleResponsesCompact(makeRequest(), config, { model: "", provider: "" }); + expect(sends.length).toBe(2); + expect(response.status).toBe(200); + expect(sends).toEqual(["Bearer work-access", `Bearer ${bearer}`]); + expect(bodies[1]?.previous_response_id).toBeUndefined(); + expect(peekConversationStateIssuer(first.conversationStateKey!)).toBe("__main__"); + config.pausedCodexAccountIds = ["__main__"]; + sends = []; bodies.length = 0; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + sends.push(new Headers(init?.headers).get("authorization")); + bodies.push(JSON.parse(String(init?.body)) as Record); + return compacted(); + }) as typeof fetch; + const next = await handleResponsesCompact(makeRequest(), config, { model: "", provider: "" }); + expect(next.status).toBe(200); + expect(sends).toEqual(["Bearer work-access"]); + expect(bodies[0]?.previous_response_id).toBeUndefined(); +}); + +test("caller-owned compact scrubs Pool to manually pinned main and back", async () => { + const { fakeChatGptJwt } = await import("../helpers/agent-task-recovery"); + const { rememberServingConversationStateIssuer } = await import("../../src/server/responses/account-change-state"); + const bearer = fakeChatGptJwt("pin-caller"); + const headers = new Headers({ authorization: `Bearer ${bearer}`, "content-type": "application/json", "session-id": "session", "thread-id": "thread" }); + const config = poolConfig({ codexAccounts: [{ id: WORK, label: WORK }], pausedCodexAccountIds: ["__main__"], codexPool: { requestOwnedAffinity: true } }); + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {} }; + const first = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + const second = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); expect(second.affinityDecision?.move).toBe("reused"); + expect(first.kind).toBe("pool"); rememberServingConversationStateIssuer(second); + const { resetCodexRoutingForManualSelection } = await import("../../src/codex/routing/active-account"); + resetCodexRoutingForManualSelection("__main__"); + config.activeCodexAccountId = "__main__"; config.activeCodexAccountPinned = "__main__"; config.pausedCodexAccountIds = []; + const bodies: Record[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + bodies.push(JSON.parse(String(init?.body)) as Record); + return compacted(); + }) as typeof fetch; + const request = (input: unknown = [{ role: "user", content: "continue" }]) => new Request("http://localhost/v1/responses/compact", { + method: "POST", headers, body: JSON.stringify({ model: "gpt-5.5", previous_response_id: "opaque", input }), + }); + const refusal = await handleResponsesCompact(request([{ role: "user", file_id: "opaque" }]), config, { model: "", provider: "" }); + expect(refusal.status).toBe(400); expect(bodies).toHaveLength(0); + const pinned = await handleResponsesCompact(request(), config, { model: "", provider: "" }); + expect(pinned.status).toBe(200); expect(bodies[0]?.previous_response_id).toBeUndefined(); + config.activeCodexAccountId = WORK; config.activeCodexAccountPinned = undefined; config.pausedCodexAccountIds = ["__main__"]; + const back = await handleResponsesCompact(request(), config, { model: "", provider: "" }); + expect(back.status).toBe(200); expect(bodies[1]?.previous_response_id).toBeUndefined(); +}); + +test("caller-owned compact main 429 releases only its conversation binding", async () => { + const { fakeChatGptJwt } = await import("../helpers/agent-task-recovery"); + const { getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const { getCodexQuotaHealthSnapshot } = await import("../../src/codex/routing"); + const bearer = fakeChatGptJwt("owned-compact-failure"); + const headers = new Headers({ authorization: `Bearer ${bearer}`, "content-type": "application/json", "session-id": "s", "thread-id": "t" }); + const config = poolConfig({ codexAccounts: [{ id: WORK, label: WORK }], pausedCodexAccountIds: [WORK], codexPool: { requestOwnedAffinity: true } }); + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {} }; + const first = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + const second = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + expect(first.kind).toBe("main"); expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + config.pausedCodexAccountIds = []; + let status = 200; + globalThis.fetch = (async () => status === 200 ? compacted() : Response.json({ error: { message: "quota exceeded" } }, { status: 429, headers: { "retry-after": "600" } })) as typeof fetch; + const makeRequest = () => new Request("http://localhost/v1/responses/compact", { method: "POST", headers, body: JSON.stringify({ model: "gpt-5.5", input: [{ role: "user", content: "continue" }] }) }); + expect((await handleResponsesCompact(makeRequest(), config, { model: "", provider: "" })).status).toBe(200); + expect(getThreadAffinity(first.affinityKey!, "shared")?.accountId).toBe("__main__"); + status = 429; + expect((await handleResponsesCompact(makeRequest(), config, { model: "", provider: "" })).status).toBe(429); + expect(getThreadAffinity(first.affinityKey!, "shared")).toBeUndefined(); + expect(getCodexQuotaHealthSnapshot("__main__", "shared")).toBeNull(); + const next = await authContext.resolveCodexAuthContext(headers, config, "pool", options); + expect(next.affinityDecision?.move).toBe("rebound"); + expect(next.affinityDecision?.reason).toBe("quota_refusal"); +}); diff --git a/tests/responses/responses-alternate-main-cancellation.test.ts b/tests/responses/responses-alternate-main-cancellation.test.ts index ca5fb4a13e9..a87f607cfde 100644 --- a/tests/responses/responses-alternate-main-cancellation.test.ts +++ b/tests/responses/responses-alternate-main-cancellation.test.ts @@ -251,3 +251,161 @@ for (const entry of ["retry helper", "messages ingress"] as const) { } }); } + +test.each([false, true])("caller-owned retry from Pool to main scrubs continuation (affinity=%s)", async enabled => { + const { rememberServingConversationStateIssuer, conversationStateBindingFromAuth, applyAccountChangeConversationStateScrub } = await import("../../src/server/responses/account-change-state"); + const { peekConversationStateIssuer } = await import("../../src/codex/routing/thread-affinity"); + const bearer = fakeChatGptJwt("fixture-caller", { exp: 4_000_000_000 }); + const headers = new Headers({ authorization: `Bearer ${bearer}`, "session-id": "session", "thread-id": "thread" }); + config.codexPool = { requestOwnedAffinity: enabled }; + config.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + const turn = tryAdmitTurn(); + expect(turn).not.toBeNull(); + const controller = new AbortController(); + try { + const authOptions = { modelId: "gpt-5.5", requestScopedMainCredential: true, primeCodexPoolQuotas: async () => {}, beginCodexAccountSelection: codexAccountSelectionForTurn(turn!) }; + const first = await resolveCodexAuthContext(headers, config, "pool", authOptions); + const reused = await resolveCodexAuthContext(headers, config, "pool", authOptions); + expect(first.affinityDecision?.move).toBe(enabled ? "new_bind" : "unbound"); + expect(reused.affinityDecision?.move).toBe(enabled ? "reused" : "unbound"); + expect(first.kind).toBe("pool"); + if (first.kind !== "pool") throw new Error("expected pool fixture"); + rememberServingConversationStateIssuer(first); + config.pausedCodexAccountIds = []; + let sentBody: Record | undefined; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + sentBody = JSON.parse(String(init?.body)) as Record; + return completedInference(); + }) as typeof fetch; + const parsed = parseRequest({ model: "gpt-5.5", previous_response_id: "opaque-previous", input: "hello", stream: false }); + const result = await retryCodexPoolOnAlternateAccount({ + callerAuthHeaders: headers, config, firstAuthCtx: first, firstResponse: quotaRefusal(), outcomeStatus: 429, + route: { providerName: "openai", modelId: "gpt-5.5", provider: config.providers.openai! }, + parsed, logCtx: { model: "", provider: "" }, + options: { abortSignal: controller.signal, translatorBudget: createTranslatorBudget(), turnAdmissionLease: turn! }, + upstream: controller, connectMs: 10_000, stream: false, httpOnly: true, + }); + expect(result.kind).toBe("retried"); + if (result.kind !== "retried") throw new Error("expected alternate retry"); + expect(result.authCtx.kind).toBe("main"); + expect(sentBody?.previous_response_id).toBeUndefined(); + rememberServingConversationStateIssuer(result.authCtx, first.conversationStateKey); + expect(peekConversationStateIssuer(first.conversationStateKey!)).toBe(MAIN_CODEX_ACCOUNT_ID); + config.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + clearCodexUpstreamHealth(); + const next = await resolveCodexAuthContext(headers, config, "pool", authOptions); + const binding = conversationStateBindingFromAuth(next)!; + const body = { previous_response_id: "opaque-main-response" }; + expect(applyAccountChangeConversationStateScrub({ body, bindingKey: binding.bindingKey, servingAccountId: binding.accountId })).toBe(true); + expect(body).not.toHaveProperty("previous_response_id"); + } finally { turn?.release(); } +}); + +test("regular Responses scrubs retained issuer on a manual main pin and on return to Pool", async () => { + const { rememberServingConversationStateIssuer } = await import("../../src/server/responses/account-change-state"); + const { forgetRuntimeActiveCodexAccount, resetCodexRoutingForManualSelection } = await import("../../src/codex/routing/active-account"); + const { handleResponses } = await import("../../src/server/responses"); + const { rememberResponseState, clearResponseStateForTests } = await import("../../src/responses/state"); + const bearer = fakeChatGptJwt("fixture-pin-caller", { exp: 4_000_000_000 }); + const headers = new Headers({ authorization: `Bearer ${bearer}`, "content-type": "application/json", "session-id": "session", "thread-id": "thread" }); + config.codexPool = { requestOwnedAffinity: true }; config.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + const turn = tryAdmitTurn(); + expect(turn).not.toBeNull(); + try { + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {}, beginCodexAccountSelection: codexAccountSelectionForTurn(turn!) }; + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.affinityDecision?.move).toBe("new_bind"); expect(second.affinityDecision?.move).toBe("reused"); + expect(first.kind).toBe("pool"); rememberServingConversationStateIssuer(second); + resetCodexRoutingForManualSelection(MAIN_CODEX_ACCOUNT_ID); + config.activeCodexAccountId = MAIN_CODEX_ACCOUNT_ID; config.activeCodexAccountPinned = MAIN_CODEX_ACCOUNT_ID; config.pausedCodexAccountIds = []; + const sent: Record[] = []; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { sent.push(JSON.parse(String(init?.body)) as Record); return completedInference(); }) as typeof fetch; + clearResponseStateForTests(); + rememberResponseState({ model: "gpt-5.5", input: [{ role: "user", content: "earlier" }] }, { id: "opaque", status: "completed", output: [] }, undefined, { force: true }); + const request = (input: unknown = [{ role: "user", content: "hello" }]) => new Request("http://localhost/v1/responses", { method: "POST", headers, + body: JSON.stringify({ model: "gpt-5.5", previous_response_id: "opaque", input, stream: false }) }); + const refusal = await handleResponses(request([{ role: "user", file_id: "opaque" }]), config, { model: "", provider: "" }, { turnAdmissionLease: turn! }); + expect(refusal.status).toBe(400); expect(await refusal.text()).toContain("uploaded file"); expect(sent).toHaveLength(0); + const pinned = await handleResponses(request(), config, { model: "", provider: "" }, { turnAdmissionLease: turn! }); + const pinnedText = await pinned.text(); + if (pinned.status !== 200) throw new Error(`pinned status ${pinned.status}: ${pinnedText}`); + expect(sent[0]?.previous_response_id).toBeUndefined(); + forgetRuntimeActiveCodexAccount(); + config.activeCodexAccountId = POOL_ID; config.activeCodexAccountPinned = undefined; config.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + const back = await handleResponses(request(), config, { model: "", provider: "" }, { turnAdmissionLease: turn! }); + expect(back.status).toBe(200); await back.text(); expect(sent[1]?.previous_response_id).toBeUndefined(); + } finally { clearResponseStateForTests(); turn?.release(); } +}); + +test.each([429, 503])("caller-owned Responses main quota refusal %s releases only its conversation binding", async refusalStatus => { + const { handleResponses } = await import("../../src/server/responses"); + const { getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const { getCodexQuotaHealthSnapshot } = await import("../../src/codex/routing"); + const headers = new Headers({ authorization: `Bearer ${fakeChatGptJwt("owned-responses-failure")}`, "content-type": "application/json", "session-id": "s", "thread-id": "t" }); + config.codexPool = { requestOwnedAffinity: true }; config.pausedCodexAccountIds = [POOL_ID]; + config.accountPoolStrategy = "round-robin"; config.accountPoolStickyLimit = 1; + const turn = tryAdmitTurn(); expect(turn).not.toBeNull(); + try { + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {}, beginCodexAccountSelection: codexAccountSelectionForTurn(turn!) }; + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.kind).toBe("main"); expect(first.affinityDecision?.move).toBe("new_bind"); expect(second.affinityDecision?.move).toBe("reused"); + config.pausedCodexAccountIds = []; + let status = 200; + globalThis.fetch = (async () => status === 200 ? completedInference() : refusalStatus === 429 ? quotaRefusal() : Response.json({ error: { code: "usage_limit_reached", message: "usage limit reached" } }, { status: 503, headers: { "retry-after": "600" } })) as typeof fetch; + const makeRequest = () => new Request("http://localhost/v1/responses", { method: "POST", headers, body: JSON.stringify({ model: "gpt-5.5", input: "hello", stream: false }) }); + const success = await handleResponses(makeRequest(), config, { model: "", provider: "" }, { turnAdmissionLease: turn! }); + expect(success.status).toBe(200); await success.text(); + expect(getThreadAffinity(first.affinityKey!, "shared")?.accountId).toBe(MAIN_CODEX_ACCOUNT_ID); + status = 429; + const failure = await handleResponses(makeRequest(), config, { model: "", provider: "" }, { turnAdmissionLease: turn! }); + expect(failure.status).toBe(refusalStatus); await failure.text(); + expect(getThreadAffinity(first.affinityKey!, "shared")).toBeUndefined(); + expect(getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, "shared")).toBeNull(); + const next = await resolveCodexAuthContext(headers, config, "pool", options); + expect(next.affinityDecision?.move).toBe("rebound"); expect(next.affinityDecision?.reason).toBe("quota_refusal"); + } finally { turn?.release(); } +}); + +test("websocket terminal recorder releases a caller-owned main binding on quota refusal", async () => { + const { handleResponses } = await import("../../src/server/responses"); + const { getThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const headers = new Headers({ authorization: `Bearer ${fakeChatGptJwt("owned-ws-failure")}`, "content-type": "application/json", "session-id": "s", "thread-id": "t" }); + config.codexPool = { requestOwnedAffinity: true }; config.pausedCodexAccountIds = [POOL_ID]; + config.accountPoolStrategy = "round-robin"; config.accountPoolStickyLimit = 1; + const turn = tryAdmitTurn(); expect(turn).not.toBeNull(); + try { + const options = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {}, beginCodexAccountSelection: codexAccountSelectionForTurn(turn!) }; + const first = await resolveCodexAuthContext(headers, config, "pool", options); + const second = await resolveCodexAuthContext(headers, config, "pool", options); + expect(first.kind).toBe("main"); expect(first.affinityDecision?.move).toBe("new_bind"); expect(second.affinityDecision?.move).toBe("reused"); + config.pausedCodexAccountIds = []; + globalThis.fetch = (async () => completedInference()) as typeof fetch; + let record: ((status: "completed" | "failed" | "incomplete", httpStatusOverride?: number) => void) | undefined; + const req = new Request("http://localhost/v1/responses", { method: "POST", headers, body: JSON.stringify({ model: "gpt-5.5", input: "hello", stream: true }) }); + const response = await handleResponses(req, config, { model: "", provider: "" }, { turnAdmissionLease: turn!, inboundTransport: "websocket", recordTerminalOutcomes: false, setTerminalOutcomeRecorder: fn => { record = fn; } }); + expect(response.status).toBe(200); await response.text(); + expect(record).toBeDefined(); + record!("incomplete", 429); + expect(getThreadAffinity(first.affinityKey!, "shared")).toBeUndefined(); + } finally { turn?.release(); } +}); + +test("unkeyed caller-main 5xx does not inspect quota response body", async () => { + const { handleResponses } = await import("../../src/server/responses"); + const account = await import("../../src/server/responses/core-codex-account"); + const { spyOn } = await import("bun:test"); + const headers = new Headers({ authorization: `Bearer ${fakeChatGptJwt("unkeyed-inspection")}`, "content-type": "application/json" }); + config.pausedCodexAccountIds = [POOL_ID]; + const turn = tryAdmitTurn(); expect(turn).not.toBeNull(); + const quotaInspector = spyOn(account, "shouldRetryCodexPoolAccountQuota").mockResolvedValue(false); + try { + globalThis.fetch = (async () => Response.json({ error: { message: "server error" } }, { status: 503, headers: { "retry-after": "600" } })) as typeof fetch; + const req = new Request("http://localhost/v1/responses", { method: "POST", headers, body: JSON.stringify({ model: "gpt-5.5", input: "hello", stream: false }) }); + const response = await handleResponses(req, config, { model: "", provider: "" }, { turnAdmissionLease: turn! }); + expect(response.status).toBe(503); + expect(quotaInspector).not.toHaveBeenCalled(); + await response.text(); + } finally { quotaInspector.mockRestore(); turn?.release(); } +}); diff --git a/tests/responses/responses-forward-incomplete-quota.test.ts b/tests/responses/responses-forward-incomplete-quota.test.ts index ee79fd9c182..25ca110e060 100644 --- a/tests/responses/responses-forward-incomplete-quota.test.ts +++ b/tests/responses/responses-forward-incomplete-quota.test.ts @@ -340,3 +340,19 @@ describe("incomplete quota endpoint reporter wiring", () => { await exerciseSpawnReporter("native-sse"); }, { timeout: SERVER_BUDGET_MS }); }); + +test("caller-owned main streamed terminal releases only on a refusal", async () => { + const { requestOwnedAffinityKey, captureRequestOwnedAffinityBindings } = await import("../../src/codex/request-owned-affinity"); + const { bindThreadAffinity, getThreadAffinity, peekPendingReleaseReason, affinityAfterRelease } = await import("../../src/codex/routing/thread-affinity"); + const key = requestOwnedAffinityKey(new Headers({ authorization: "Bearer stream-test", "session-id": "s", "thread-id": "t" }))!; + clearThreadAccountMap(); bindThreadAffinity(key, "__main__", Date.now(), "shared"); + const ctx: CodexAuthContext = { kind: "main", accountId: null, affinityKey: key, affinityBindingSnapshot: captureRequestOwnedAffinityBindings(key, "shared", "gpt-test") }; + const record = codexForwardTerminalOutcomeRecorder(getDefaultConfig(), ctx, provider, "gpt-test", { model: "gpt-test", provider: "openai" }); + expect(record).toBeDefined(); + record!("completed"); + expect(getThreadAffinity(key, "shared")?.accountId).toBe("__main__"); + record!("incomplete", 429); + expect(getThreadAffinity(key, "shared")).toBeUndefined(); + expect(peekPendingReleaseReason(key)).toBe("quota_refusal"); + affinityAfterRelease(key, "quota_refusal"); clearThreadAccountMap(); +}); diff --git a/tests/responses/responses-preview-main-read-fence.test.ts b/tests/responses/responses-preview-main-read-fence.test.ts index bb3a95a7024..d59e01df5d6 100644 --- a/tests/responses/responses-preview-main-read-fence.test.ts +++ b/tests/responses/responses-preview-main-read-fence.test.ts @@ -626,3 +626,102 @@ describe("preview and final authentication agree on the native-main read fence", expect(upstreamAuth[0]).not.toBe("Bearer physical-main-token"); }); }); + +test("subagent preview evaluates affinity separately for Reserve candidates", async () => { + const routing = await import("../../src/codex/routing"); + const auth = await import("../../src/codex/auth-context"); + const { bindThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const config = providerConfig({ codexDesktopAuthless: true, codexPool: { requestOwnedAffinity: true }, subagentModelFallback: ["gpt-reserve", "gpt-5.5"] }); + const headers = codexHeaders("caller-account", { "session-id": "preview-session", "thread-id": "preview-thread" }); + config.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + const authOptions = { requestScopedMainCredential: true, modelId: PREFERRED_MODEL, primeCodexPoolQuotas: async () => {} }; + const first = await auth.resolveCodexAuthContext(headers, config, "pool", authOptions); + const second = await auth.resolveCodexAuthContext(headers, config, "pool", authOptions); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const key = first.affinityKey!; + expect(key).toBe(auth.codexRequestAffinityKey(headers, config, config, { requestScopedMainCredential: true })); + config.pausedCodexAccountIds = []; + bindThreadAffinity(key, "pool-a", NOW, "reserve"); + noteSubagentModelFailure(PREFERRED_MODEL, "429", config, "pool-a", NOW); + noteSubagentModelFailure("gpt-reserve", "429", config, MAIN_CODEX_ACCOUNT_ID, NOW); + noteSubagentModelFailure("gpt-reserve", "429", config, "pool-a", NOW); + const observed: Array<{ model: string | undefined; key: string | null; account: string | null }> = []; + const original = routing.previewCodexAccountForRequest; + const spy = spyOn(routing, "previewCodexAccountForRequest").mockImplementation((thread, cfg, now, scope, opts, model, lineage) => { + const account = original(thread, cfg, now, scope, opts, model, lineage); + observed.push({ model, key: thread, account }); + return account; + }); + globalThis.fetch = (async () => completedCanonicalSse()) as typeof fetch; + try { + const logCtx: RequestLogContext = { model: "", provider: "" }; + const response = await postSpawn(config, { admission: { kind: "loopback", source: "loopback" } }, headers, readableInput(), PREFERRED_MODEL, logCtx); + expect(response.status).toBe(200); + expect(observed.some(row => row.model === "gpt-reserve" && row.key === null)).toBe(true); + expect(observed.some(row => row.model === "gpt-5.5" && row.key === key && row.account === "pool-a")).toBe(true); + expect((logCtx as unknown as Record).subagentModelFallbackTo).toBe("gpt-5.5"); + } finally { spy.mockRestore(); } +}); + +test("recovery preview reads the recovered ownership key and lineage", async () => { + const routing = await import("../../src/codex/routing"); + const auth = await import("../../src/codex/auth-context"); + const coreAuth = await import("../../src/server/responses/core-auth"); + const { recordCodexThreadLineage, clearCodexThreadLineageForTests } = await import("../../src/codex/lineage"); + const { bindThreadAffinity } = await import("../../src/codex/routing/thread-affinity"); + const config = providerConfig({ defaultProvider: "xai", agentTaskRecovery: { enabled: true }, codexPool: { requestOwnedAffinity: true }, subagentModelFallback: ["gpt-5.5"] }); + const headers = codexHeaders("caller-account", { "session-id": "recovery-session", "thread-id": "recovery-thread" }); + clearCodexThreadLineageForTests(); + config.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + const authOptions = { requestScopedMainCredential: true, modelId: "gpt-5.5", primeCodexPoolQuotas: async () => {} }; + const first = await auth.resolveCodexAuthContext(headers, config, "pool", authOptions); + const second = await auth.resolveCodexAuthContext(headers, config, "pool", authOptions); + expect(first.affinityDecision?.move).toBe("new_bind"); + expect(second.affinityDecision?.move).toBe("reused"); + const ownedKey = first.affinityKey!; + config.pausedCodexAccountIds = []; + const lineage = recordCodexThreadLineage(headers)!; + const rawKey = auth.codexPoolAffinityKey(headers)!; + bindThreadAffinity(rawKey, MAIN_CODEX_ACCOUNT_ID, NOW, "shared"); + noteSubagentModelFailure("gpt-5.5", "429", config, MAIN_CODEX_ACCOUNT_ID, NOW); + noteSubagentModelFailure("gpt-5.5", "429", config, "pool-a", NOW); + let recovered = false; + const observed: Array<{ key: string | null; model: string | undefined; lineage: unknown; account: string | null; recovered: boolean }> = []; + const originalPreview = routing.previewCodexAccountForRequest; + const preview = spyOn(routing, "previewCodexAccountForRequest").mockImplementation((key, cfg, now, scope, opts, model, hint) => { + const account = originalPreview(key, cfg, now, scope, opts, model, hint); + observed.push({ key, model, lineage: hint, account, recovered }); + return account; + }); + // The request-level harness cannot naturally reroute from a readable canonical candidate to a + // recovery-required provider: ciphertext restricts the first fallback pass to trusted routes. + // Switch the credential-domain result at the recovery boundary to exercise the real closure + // with the headers and ownership that a reroute would provide. + const originalDomainHeaders = coreAuth.codexRouteCredentialDomainHeaders; + const domainHeaders = spyOn(coreAuth, "codexRouteCredentialDomainHeaders").mockImplementation((...args) => recovered ? new Headers(headers) : originalDomainHeaders(...args)); + const originalOwnership = coreAuth.codexRouteCredentialOwnership; + const ownershipCalls: Array<{ recovered: boolean; hasAuth: boolean }> = []; + const ownership = spyOn(coreAuth, "codexRouteCredentialOwnership").mockImplementation((...args) => { ownershipCalls.push({ recovered, hasAuth: args[0].has("authorization") }); return { ...originalOwnership(...args), requestScopedMainCredential: recovered }; }); + globalThis.fetch = (async (input, init) => { + if (typeof init?.body === "string" && init.body.includes("capture_assignment")) { + recovered = true; + resetSubagentModelFallbackStateForTests(); + noteSubagentModelFailure("xai/grok-4.5", "429", config, null, NOW); + return new Response(recoverySse("Recovered assignment"), { headers: { "content-type": "text/event-stream" } }); + } + const url = new URL(input instanceof Request ? input.url : String(input)); + return url.hostname === "chatgpt.com" ? completedCanonicalSse() : completedResponses(FALLBACK_MODEL); + }) as typeof fetch; + try { + const logCtx: RequestLogContext = { model: "", provider: "" }; + const response = await postSpawn(config, {}, headers, encryptedInput(), "xai/grok-4.5", logCtx); + expect(response.status).toBe(200); + expect(recovered).toBe(true); + expect(observed.some(row => !row.recovered && row.key === rawKey && row.lineage !== undefined && row.account === MAIN_CODEX_ACCOUNT_ID)).toBe(true); + expect(ownershipCalls).toContainEqual({ recovered: true, hasAuth: true }); + expect(observed.filter(row => row.recovered).map(row => ({ model: row.model, owned: row.key === ownedKey, raw: row.key === rawKey, hasLineage: row.lineage !== undefined, account: row.account }))).toContainEqual({ model: "gpt-5.5", owned: true, raw: false, hasLineage: false, account: "pool-a" }); + expect((logCtx as unknown as Record).subagentModelFallbackTo).toBe("gpt-5.5"); + expect(lineage.conversationKey).toBe(rawKey); + } finally { preview.mockRestore(); ownership.mockRestore(); domainHeaders.mockRestore(); clearCodexThreadLineageForTests(); } +});