diff --git a/scripts/test-layout/layout-codex-collision.json b/scripts/test-layout/layout-codex-collision.json index 51022965cf2..b35d340f1f7 100644 --- a/scripts/test-layout/layout-codex-collision.json +++ b/scripts/test-layout/layout-codex-collision.json @@ -1,4 +1,5 @@ { + "codex-quota-persistence.test.ts": "codex-integration", "codex-sync-user-edits.test.ts": "codex-integration", "codex-stored-main-duplicate.test.ts": "codex-integration", "codex-account-slot-identity.test.ts": "codex-integration", diff --git a/src/codex/quota.ts b/src/codex/quota.ts index 0088c4968d6..ebcd30372aa 100644 --- a/src/codex/quota.ts +++ b/src/codex/quota.ts @@ -25,6 +25,8 @@ const QUOTA_CACHE_FILENAME = "codex-quota-cache.json"; /** Keep last-known bars across restarts; WHAM still refreshes on TTL in live/prime paths. */ const QUOTA_DISK_MAX_AGE_MS = 6 * 60 * 60_000; const QUOTA_PERSIST_DEBOUNCE_MS = 250; +/** Match the reset ledger's maximum deferral under sustained quota observations. */ +const QUOTA_PERSIST_MAX_DEFERRAL_MS = 1_000; type QuotaDiskFile = { version: 1; @@ -39,6 +41,7 @@ type MainPolicyQuotaObservation = Omit & { shor let mainPolicyQuota: MainPolicyQuota | null = null; let diskHydrated = false; let persistTimer: ReturnType | null = null; +let persistStartedAt: number | null = null; const MONTHLY_WINDOW_MIN_SECONDS = 28 * 24 * 60 * 60; /** @@ -778,9 +781,14 @@ function hydrateAccountQuotasFromDisk(): void { } function schedulePersistAccountQuotas(): void { + const now = performance.now(); + persistStartedAt ??= now; + const delay = Math.max(0, Math.min(QUOTA_PERSIST_DEBOUNCE_MS, + QUOTA_PERSIST_MAX_DEFERRAL_MS - (now - persistStartedAt))); if (persistTimer) clearTimeout(persistTimer); persistTimer = setTimeout(() => { persistTimer = null; + persistStartedAt = null; try { const quotas: Record = {}; for (const [accountId, quota] of accountQuota.entries()) { @@ -796,7 +804,7 @@ function schedulePersistAccountQuotas(): void { } catch { // Best-effort persistence only. } - }, QUOTA_PERSIST_DEBOUNCE_MS); + }, delay); } export function getAccountQuota(accountId: string): StoredAccountQuota | null { @@ -861,6 +869,7 @@ export function clearAccountQuota(accountId?: string): void { clearTimeout(persistTimer); persistTimer = null; } + persistStartedAt = null; try { const path = join(getConfigDir(), QUOTA_CACHE_FILENAME); if (existsSync(path)) unlinkSync(path); diff --git a/src/providers/account-quota-disk.ts b/src/providers/account-quota-disk.ts index 55dc711fe56..bf0aa1e5160 100644 --- a/src/providers/account-quota-disk.ts +++ b/src/providers/account-quota-disk.ts @@ -29,6 +29,8 @@ const FILENAME = "provider-account-quota-cache.json"; */ const DISK_MAX_AGE_MS = 6 * 60 * 60_000; const PERSIST_DEBOUNCE_MS = 250; +/** Match the reset ledger's maximum deferral under sustained quota observations. */ +const MAX_PERSIST_DEFERRAL_MS = 1_000; type DiskFile = { version: 1; @@ -38,6 +40,7 @@ type DiskFile = { }; let persistTimer: ReturnType | null = null; +let persistStartedAt: number | null = null; /** Read the snapshot. Returns an empty map for a missing, corrupt or stale file. */ export function readPersistedAccountQuotas(now = Date.now()): Map { @@ -86,14 +89,19 @@ export function readPersistedKiroVerdicts(now = Date.now()): Map Iterable<[string, ProviderQuota | KiroPersistedQuota]>, verdicts: () => Iterable<[string, KiroPersistedVerdict]> = () => [], ): void { + const now = performance.now(); + persistStartedAt ??= now; + const delay = Math.max(0, Math.min(PERSIST_DEBOUNCE_MS, + MAX_PERSIST_DEFERRAL_MS - (now - persistStartedAt))); if (persistTimer) clearTimeout(persistTimer); persistTimer = setTimeout(() => { persistTimer = null; + persistStartedAt = null; try { const out: Record = {}; for (const [key, quota] of rows()) out[key] = quota; @@ -106,12 +114,12 @@ export function schedulePersistAccountQuotas( } catch { // Best-effort persistence only. } - }, PERSIST_DEBOUNCE_MS); + }, delay); } /** Test seam: drop any pending write so a suite cannot leak one into the next file. */ export function cancelPendingAccountQuotaPersist(): void { - if (!persistTimer) return; - clearTimeout(persistTimer); + if (persistTimer) clearTimeout(persistTimer); persistTimer = null; + persistStartedAt = null; } diff --git a/structure/INDEX.md b/structure/INDEX.md index f33039a1cfd..58f8b82b5e8 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -73,6 +73,7 @@ Per-vendor contracts and the adapter authority that constructs them. | Doc | Scope | | --- | --- | +| [`quota-snapshots.md`](quota-snapshots.md) | Bounded write deferral and cancellation for Codex and provider account quota snapshots. | | [`providers/generic-account-pool.md`](providers/generic-account-pool.md) | Opt-in conversation ownership, request-local selection receipts and bounded release for generic OAuth pools. | | [`providers/anthropic-progress.md`](providers/anthropic-progress.md) | Opus 5.5 progress display, visible commentary, and exact signed-block replay. | | [`providers/anthropic-account-thresholds.md`](providers/anthropic-account-thresholds.md) | Account-owned usage thresholds, inheritance, routing boundaries and durable policy changes. | @@ -214,6 +215,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/codex/catalog/vendor-context.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md) | | `src/codex/cli-update-plan.ts` | [`codex-cli-update.md`](codex-cli-update.md) | | `src/codex/provider-routing.ts` | [`codex-provider-routing.md`](codex-provider-routing.md) | +| `src/codex/quota.ts` | [`quota-snapshots.md`](quota-snapshots.md) | | `src/codex/runtime.ts` | [`catalog-effort-clamp.md`](catalog-effort-clamp.md) | | `src/codex/signed-in-identity.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | | `src/combos/` | [`runtime.md`](runtime.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/jev-decision.md`](providers/jev-decision.md)
[`fallback-policy.md`](fallback-policy.md) | @@ -254,6 +256,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/plugins/` | [`ops/plugins.md`](ops/plugins.md) | | `src/protocols/` | [`data-planes/protocol-paths.md`](data-planes/protocol-paths.md) | | `src/providers/` | [`runtime.md`](runtime.md)
[`subagents.md`](subagents.md)
[`transports/inventory.md`](transports/inventory.md)
[`providers-and-adapters.md`](providers-and-adapters.md)
[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)
[`providers/xai-grok.md`](providers/xai-grok.md) | +| `src/providers/account-quota-disk.ts` | [`quota-snapshots.md`](quota-snapshots.md) | | `src/providers/anthropic-billing-ownership.ts` | [`providers/anthropic-key-billing.md`](providers/anthropic-key-billing.md) | | `src/providers/anthropic-billing-recovery.ts` | [`providers/anthropic-key-billing.md`](providers/anthropic-key-billing.md) | | `src/providers/anthropic-billing-refusal.ts` | [`providers/anthropic-key-billing.md`](providers/anthropic-key-billing.md) | diff --git a/structure/manifest.json b/structure/manifest.json index 7b0d19d4260..5066e40883f 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -47,6 +47,13 @@ } ], "docs": [ + { + "path": "quota-snapshots.md", + "tier": 4, + "title": "Account Quota Snapshots", + "scope": "Bounded write deferral and cancellation for Codex and provider account quota snapshots.", + "documents": ["src/codex/quota.ts", "src/providers/account-quota-disk.ts"] + }, { "path": "providers/generic-account-pool.md", "tier": 4, diff --git a/structure/quota-snapshots.md b/structure/quota-snapshots.md new file mode 100644 index 00000000000..04031e3a814 --- /dev/null +++ b/structure/quota-snapshots.md @@ -0,0 +1,25 @@ +# Account Quota Snapshots + +`src/codex/quota.ts` and `src/providers/account-quota-disk.ts` persist last-known account +quotas for restart recovery. The Codex snapshot also carries bounded observation history +and separately identity-bound main-policy evidence; provider snapshots include Kiro's +identity-bound exhaustion verdicts. Their existing hydration and expiry rules govern readers. + +## Bounded write deferral + +Both writers coalesce updates with a 250 ms idle debounce and a one-second maximum +deferral from the first pending update, measured on the monotonic clock. Continued +observations cannot postpone the scheduled write beyond that deadline; event-loop delays +and disk failures remain best-effort. This uses the same maximum deferral as the quota +reset ledger in `src/quota/reset-seen-store.ts` without making producer calls write synchronously. + +The timer snapshots the latest accepted state and resolves the configured home when it +runs. Completion, failure, and cancellation reset the deadline. Clearing all Codex quotas +cancels its pending snapshot; clearing one account persists the remaining rows with the +existing deadline. Provider cancellation discards the pending row and verdict callbacks. +Atomic replacement and account identity checks stay with the existing writer and producer. + +`tests/codex-integration/codex-quota-persistence.test.ts` and +`tests/providers/provider-account-quota-persistence.test.ts` check real disk contents under +controlled timers: idle debounce, consecutive sustained-update windows, cancellation and +home reset, removed rows, latest provider callbacks, and recovery after a failed write. diff --git a/tests/codex-integration/codex-quota-persistence.test.ts b/tests/codex-integration/codex-quota-persistence.test.ts new file mode 100644 index 00000000000..989d7f4d328 --- /dev/null +++ b/tests/codex-integration/codex-quota-persistence.test.ts @@ -0,0 +1,109 @@ +import { afterEach, beforeEach, describe, expect, jest, spyOn, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearAccountQuota, flushQuotaObservationsForTests, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const previousHome = process.env.OPENCODEX_HOME; +const FILE = "codex-quota-cache.json"; +let homes: string[]; +let home: string; +let elapsed: number; +let restoreClock: () => void; + +function advance(ms: number): void { + elapsed += ms; + jest.advanceTimersByTime(ms); +} + +function snapshot(): Record { + return JSON.parse(readFileSync(join(home, FILE), "utf8")).quotas; +} + +beforeEach(async () => { + home = mkdtempSync(join(tmpdir(), "ocx-codex-quota-persist-")); + homes = [home]; + process.env.OPENCODEX_HOME = home; + clearAccountQuota(); + await flushQuotaObservationsForTests(); + jest.useFakeTimers(); + elapsed = 0; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + restoreClock = () => clock.mockRestore(); +}); + +afterEach(async () => { + clearAccountQuota(); + await flushQuotaObservationsForTests(); + restoreClock(); + jest.clearAllTimers(); + jest.useRealTimers(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + for (const path of homes) removeTreeWithRetry(path); +}); + +describe("Codex quota snapshot persistence", () => { + test("a short burst keeps the 250 ms idle debounce and writes the latest quota", () => { + setAccountQuotaFromParsed("pool-a", { weeklyPercent: 10 }); + advance(100); + setAccountQuotaFromParsed("pool-a", { weeklyPercent: 20 }); + advance(249); + expect(existsSync(join(home, FILE))).toBe(false); + advance(1); + expect(snapshot()["pool-a"]?.weeklyPercent).toBe(20); + }); + + test("sustained updates persist the latest quota at each one-second deadline", () => { + for (let batch = 0; batch < 2; batch++) { + for (let update = 1; update <= 10; update++) { + setAccountQuotaFromParsed("pool-a", { weeklyPercent: batch * 10 + update }); + advance(100); + } + expect(snapshot()["pool-a"]?.weeklyPercent).toBe((batch + 1) * 10); + } + }); + + test("clearing one account keeps the pending deadline and excludes its row", () => { + setAccountQuotaFromParsed("removed", { weeklyPercent: 75 }); + for (let update = 1; update <= 9; update++) { + setAccountQuotaFromParsed("kept", { weeklyPercent: update }); + advance(100); + } + clearAccountQuota("removed"); + advance(100); + expect(snapshot()).toEqual({ kept: expect.objectContaining({ weeklyPercent: 9 }) }); + }); + + test("clearing all accounts cancels the old write and resets the deadline for a new home", () => { + const oldHome = home; + for (let update = 0; update < 9; update++) { + setAccountQuotaFromParsed("old-account", { weeklyPercent: update }); + advance(100); + } + clearAccountQuota(); + home = mkdtempSync(join(tmpdir(), "ocx-codex-quota-next-")); + homes.push(home); + process.env.OPENCODEX_HOME = home; + setAccountQuotaFromParsed("new-account", { weeklyPercent: 15 }); + advance(249); + expect(existsSync(join(home, FILE))).toBe(false); + expect(existsSync(join(oldHome, FILE))).toBe(false); + advance(1); + expect(snapshot()).toEqual({ "new-account": expect.objectContaining({ weeklyPercent: 15 }) }); + }); + + test("a failed write remains best-effort and the next update gets a fresh debounce", () => { + mkdirSync(join(home, FILE)); + setAccountQuotaFromParsed("pool-a", { weeklyPercent: 10 }); + expect(() => advance(250)).not.toThrow(); + rmdirSync(join(home, FILE)); + advance(1_000); + setAccountQuotaFromParsed("pool-a", { weeklyPercent: 20 }); + advance(249); + expect(existsSync(join(home, FILE))).toBe(false); + advance(1); + expect(snapshot()["pool-a"]?.weeklyPercent).toBe(20); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 13553f9b241..c402ee4eb53 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1,5 +1,6 @@ { "workbuddy-quota.test.ts": "providers", "workbuddy-login.test.ts": "oauth", "workbuddy-adapter.test.ts": "adapters", "workbuddy-provider.test.ts": "providers", "link-tunnel-history.test.ts": "clients", "link-readiness.test.ts": "clients", "link-diagnostics.test.ts": "clients", "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "responses-phase-inference.test.ts": "responses", "service-proxy-env.test.ts": "service", + "codex-quota-persistence.test.ts": "codex-integration", "bridge-terminal-refusal.test.ts": "adapters", "responses-replay-metadata-principal.test.ts": "responses", "responses-combo-replay-principal.test.ts": "responses", diff --git a/tests/providers/provider-account-quota-persistence.test.ts b/tests/providers/provider-account-quota-persistence.test.ts index c91208ff9d0..22c4ccf2618 100644 --- a/tests/providers/provider-account-quota-persistence.test.ts +++ b/tests/providers/provider-account-quota-persistence.test.ts @@ -1,10 +1,11 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { afterEach, beforeEach, describe, expect, jest, spyOn, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { cancelPendingAccountQuotaPersist, readPersistedAccountQuotas, + readPersistedKiroVerdicts, schedulePersistAccountQuotas, } from "../../src/providers/account-quota-disk"; import type { ProviderQuota } from "../../src/providers/quota-types"; @@ -105,3 +106,85 @@ describe("provider account quota persistence", () => { expect(existsSync(join(home, FILE))).toBe(false); }); }); + +describe("provider quota snapshot deadlines", () => { + let elapsed: number; + let restoreClock: () => void; + const advance = (ms: number) => { + elapsed += ms; + jest.advanceTimersByTime(ms); + }; + + beforeEach(() => { + jest.useFakeTimers(); + elapsed = 0; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + restoreClock = () => clock.mockRestore(); + }); + + afterEach(() => { + cancelPendingAccountQuotaPersist(); + restoreClock(); + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + test("a short burst keeps the 250 ms idle debounce", () => { + schedulePersistAccountQuotas(() => [[KEY, quota(10)]]); + advance(100); + schedulePersistAccountQuotas(() => [[KEY, quota(20)]]); + advance(249); + expect(existsSync(join(home, FILE))).toBe(false); + advance(1); + expect(readPersistedAccountQuotas().get(KEY)?.monthlyPercent).toBe(20); + }); + + test("sustained updates persist the latest row and verdict callbacks every second", () => { + const key = "kiro\u0000acct-a"; + const identity = "a".repeat(64); + for (let batch = 0; batch < 2; batch++) { + for (let update = 1; update <= 10; update++) { + const percent = batch * 10 + update; + schedulePersistAccountQuotas(() => [[key, { ...quota(percent), identity }]], + () => [[key, { exhausted: percent >= 20, identity, observedAt: Date.now() }]]); + advance(100); + } + expect(readPersistedAccountQuotas().get(key)?.monthlyPercent).toBe((batch + 1) * 10); + expect(readPersistedKiroVerdicts().get(key)?.exhausted).toBe(batch === 1); + } + }); + + test("cancellation resets the deadline before the next home schedules a write", () => { + const oldHome = home; + for (let update = 0; update < 9; update++) { + schedulePersistAccountQuotas(() => [[KEY, quota(update)]]); + advance(100); + } + cancelPendingAccountQuotaPersist(); + home = mkdtempSync(join(tmpdir(), "ocx-quota-persist-next-")); + process.env.OPENCODEX_HOME = home; + try { + schedulePersistAccountQuotas(() => [["cursor\u0000new-account", quota(15)]]); + advance(249); + expect(existsSync(join(home, FILE))).toBe(false); + expect(existsSync(join(oldHome, FILE))).toBe(false); + advance(1); + expect([...readPersistedAccountQuotas().keys()]).toEqual(["cursor\u0000new-account"]); + } finally { + removeTreeWithRetry(oldHome); + } + }); + + test("a failed write remains best-effort and the next update gets a fresh debounce", () => { + mkdirSync(join(home, FILE)); + schedulePersistAccountQuotas(() => [[KEY, quota(10)]]); + expect(() => advance(250)).not.toThrow(); + rmdirSync(join(home, FILE)); + advance(1_000); + schedulePersistAccountQuotas(() => [[KEY, quota(20)]]); + advance(249); + expect(existsSync(join(home, FILE))).toBe(false); + advance(1); + expect(readPersistedAccountQuotas().get(KEY)?.monthlyPercent).toBe(20); + }); +});