diff --git a/AGENTS.md b/AGENTS.md index f71dc2dd0f3..d5259be3a24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,10 +113,10 @@ different pointer, so every branch move and promotion dragged a diff. Reference *clones* are gitignored: they are third-party source carrying their own licenses and have no business in this repository's history. -Nothing in the build, typecheck, or test path reads from `devlog/`, so a -contributor who ignores it entirely still passes every gate. `privacy:scan` does -read it — that is deliberate, and it is what makes a public devlog safe rather -than merely visible. +Build and typecheck do not read `devlog/`. The test-layout membership oracle reads +`devlog/_fin/260905_test_modularization_and_windows/001_test_inventory.md`; sparse +checkouts running that test must materialize this tracked input. `privacy:scan` +also reads `devlog/` to keep public development notes safe to publish. Two mechanical guards in `tests/ci-workflows/repo-hygiene.test.ts` back this up: no `160000` gitlink may be tracked anywhere, and neither the vendored reference clones nor diff --git a/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts b/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts index da69051e673..4a6518081bf 100644 --- a/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts +++ b/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync} from "node:fs"; +import { existsSync, mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../../src/config"; @@ -8,16 +8,20 @@ import { startServer } from "../../../src/server"; import { resetNormalizeStateForTests } from "../../../src/adapters/anthropic-image-normalize"; import { sniffImageDimensions } from "../../../src/adapters/anthropic-image-guard"; import type { OcxConfig } from "../../../src/types"; +import { trackBackgroundWork } from "../../../src/lib/background-work"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../../helpers/isolated-codex-home"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; let testDir = ""; let previousHome: string | undefined; +let previousCodexHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; let upstream: ReturnType | null = null; +let proxy: ReturnType | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; isolatedCodexHome = installIsolatedCodexHome("ocx-imgretry-e2e-codex-"); testDir = mkdtempSync(join(tmpdir(), "ocx-imgretry-e2e-")); process.env.OPENCODEX_HOME = testDir; @@ -25,16 +29,88 @@ beforeEach(() => { resetNormalizeStateForTests(); }); -afterEach(() => { - upstream?.stop(true); - upstream = null; - if (previousHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = previousHome; - isolatedCodexHome?.restore(); +/** Restore process state even when shutdown rejects; retain trees until drain succeeds. */ +async function cleanupFixture(): Promise { + const codexHome = isolatedCodexHome; + const directory = testDir; isolatedCodexHome = null; - if (testDir) removeTreeWithRetry(testDir); - clearKeyCooldowns(); -}); + testDir = ""; + try { + try { await proxy?.stop(true); } + finally { + proxy = undefined; + try { await upstream?.stop(true); } + finally { upstream = null; } + } + if (directory) await removeTreeAfterBackgroundWork(directory); + } finally { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome?.restoreEnvironment(); + clearKeyCooldowns(); + } + // A failed stop or drain must leave both owned trees intact for their wrapper. + codexHome?.removeAfterDrain(); +} + +afterEach(cleanupFixture, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); + +for (const rejecting of ["proxy", "upstream"] as const) { + test(`fixture cleanup restores environment after ${rejecting} shutdown rejects`, async () => { + const directory = testDir, codexHome = isolatedCodexHome!; + const expectedHome = previousHome; + const failure = new Error(`synthetic ${rejecting} stop failure`); + const stopped: string[] = []; + // No real producer starts: these controlled failures exercise the hook itself. + proxy = { async stop() { stopped.push("proxy"); if (rejecting === "proxy") throw failure; } } as unknown as typeof proxy; + upstream = { async stop() { stopped.push("upstream"); if (rejecting === "upstream") throw failure; } } as unknown as typeof upstream; + try { + await expect(cleanupFixture()).rejects.toBe(failure); + expect(stopped).toEqual(["proxy", "upstream"]); + expect(proxy).toBeUndefined(); + expect(upstream).toBeNull(); + expect(isolatedCodexHome).toBeNull(); + expect(process.env.OPENCODEX_HOME).toBe(expectedHome); + expect(process.env.CODEX_HOME).toBe(previousCodexHome); + expect(existsSync(directory)).toBe(true); + expect(existsSync(codexHome.path)).toBe(true); + } finally { + // The synthetic stoppers own no live handles; clean up even if a regression fails. + proxy = undefined; upstream = null; isolatedCodexHome = null; testDir = ""; + if (expectedHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = expectedHome; + codexHome.restoreEnvironment(); + await removeTreeAfterBackgroundWork(directory); + codexHome.removeAfterDrain(); + } + }); +} + +test("fixture cleanup restores environment and retains trees when drain fails", async () => { + const directory = testDir, codexHome = isolatedCodexHome!; + const expectedHome = previousHome; + let release!: () => void; + const work = trackBackgroundWork("image-retry fixture drain rejection", new Promise(resolve => { release = resolve; })); + try { + await expect(cleanupFixture()).rejects.toThrow("in-process background work still running: image-retry fixture drain rejection"); + expect(proxy).toBeUndefined(); + expect(upstream).toBeNull(); + expect(isolatedCodexHome).toBeNull(); + expect(process.env.OPENCODEX_HOME).toBe(expectedHome); + expect(process.env.CODEX_HOME).toBe(previousCodexHome); + expect(existsSync(directory)).toBe(true); + expect(existsSync(codexHome.path)).toBe(true); + } finally { + release(); + await work; + proxy = undefined; upstream = null; isolatedCodexHome = null; testDir = ""; + if (expectedHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = expectedHome; + codexHome.restoreEnvironment(); + await removeTreeAfterBackgroundWork(directory); + codexHome.removeAfterDrain(); + } +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); const ONE_PX_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; @@ -81,6 +157,7 @@ function anthropicConfig(baseUrl: string, pool = false): OcxConfig { return { port: 0, hostname: "127.0.0.1", + clientIntegrations: { codex: false }, defaultProvider: "anthropic-test", providers: { "anthropic-test": { @@ -121,14 +198,30 @@ async function postImageRequest(serverUrl: string, dataUrl: string): Promise void | Promise, +): void { + describe(name, () => { + let seen: SeenRequest[]; + let dataUrl: string; + // Real Windows config/ACL and server setup is separate from the 5s request assertions. + beforeEach(async () => { + seen = []; + upstream = scriptedUpstream(statuses, seen); + saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""), pool)); + proxy = startServer(0); + dataUrl = await realPngDataUrl(1500, 1000); + }, 10_000); + test("preserves the response and retry contract", async () => { + const res = await postImageRequest(String(proxy!.url), dataUrl); + await verify(res, seen); + }); + }); +} + describe("anthropic 413 tightened-retry (end-to-end)", () => { - test("R1: upstream 413 then 200 — exactly one biased rebuild, images re-encoded a tier lower", async () => { - const seen: SeenRequest[] = []; - upstream = scriptedUpstream([413, 200], seen); - saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""))); - const server = startServer(0); - try { - const res = await postImageRequest(String(server.url), await realPngDataUrl(1500, 1000)); + imageRetryScenario("R1: upstream 413 then 200 — exactly one biased rebuild, images re-encoded a tier lower", [413, 200], false, async (res, seen) => { expect(res.status).toBe(200); expect(seen).toHaveLength(2); // First attempt: 1500px PNG rides tier-0 pass-through. @@ -139,18 +232,9 @@ describe("anthropic 413 tightened-retry (end-to-end)", () => { expect(second?.media_type).toBe("image/jpeg"); const dims = sniffImageDimensions(second?.data ?? ""); expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(1024); - } finally { - await server.stop(true); - } }); - test("R2: upstream 413 twice — exactly two calls, honest 413 surfaces, no spiral", async () => { - const seen: SeenRequest[] = []; - upstream = scriptedUpstream([413, 413], seen); - saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""))); - const server = startServer(0); - try { - const res = await postImageRequest(String(server.url), await realPngDataUrl(1500, 1000)); + imageRetryScenario("R2: upstream 413 twice — exactly two calls, honest 413 surfaces, no spiral", [413, 413], false, async (res, seen) => { expect(res.status).toBe(413); expect(seen).toHaveLength(2); expect(res.headers.get("content-type")).toContain("application/json"); @@ -160,18 +244,9 @@ describe("anthropic 413 tightened-retry (end-to-end)", () => { type: "invalid_request_error", code: "context_length_exceeded", }); - } finally { - await server.stop(true); - } }); - test("R4: 413 → biased retry → 429 → key rotation keeps the tightened tiers", async () => { - const seen: SeenRequest[] = []; - upstream = scriptedUpstream([413, 429, 200], seen); - saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""), true)); - const server = startServer(0); - try { - const res = await postImageRequest(String(server.url), await realPngDataUrl(1500, 1000)); + imageRetryScenario("R4: 413 → biased retry → 429 → key rotation keeps the tightened tiers", [413, 429, 200], true, async (res, seen) => { expect(res.status).toBe(200); expect(seen).toHaveLength(3); // Rotation happened after the biased 413 retry hit 429. @@ -181,8 +256,5 @@ describe("anthropic 413 tightened-retry (end-to-end)", () => { expect(third?.media_type).toBe("image/jpeg"); const dims = sniffImageDimensions(third?.data ?? ""); expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(1024); - } finally { - await server.stop(true); - } }); }); diff --git a/tests/claude-integration/claude-529-mapping.test.ts b/tests/claude-integration/claude-529-mapping.test.ts index e96a9b564f2..3cf45b43dba 100644 --- a/tests/claude-integration/claude-529-mapping.test.ts +++ b/tests/claude-integration/claude-529-mapping.test.ts @@ -7,30 +7,53 @@ import { startServer } from "../../src/server"; import { getRequestLogEntries } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; -import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../helpers/remove-tree"; let testDir = ""; let previousHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; +let upstream: ReturnType | undefined; +let proxy: ReturnType | undefined; +let calls = 0; +let upstreamResponse: () => Response; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; isolatedCodexHome = installIsolatedCodexHome("ocx-claude-529-"); testDir = mkdtempSync(join(tmpdir(), "ocx-claude-529-")); process.env.OPENCODEX_HOME = testDir; -}); + calls = 0; + upstreamResponse = () => Response.json({ error: { message: "bad request shape" } }, { status: 400 }); + upstream = Bun.serve({ port: 0, fetch() { calls++; return upstreamResponse(); } }); + saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); + proxy = startServer(0); +}, 10_000); -afterEach(() => { - if (previousHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = previousHome; - isolatedCodexHome?.restore(); +afterEach(async () => { + const codexHome = isolatedCodexHome; isolatedCodexHome = null; - if (testDir) removeTreeWithRetry(testDir); -}); + try { + try { await proxy?.stop(true); } + finally { + proxy = undefined; + await upstream?.stop(true); + upstream = undefined; + } + if (testDir) await removeTreeAfterBackgroundWork(testDir); + testDir = ""; + } finally { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome?.restoreEnvironment(); + } + // A failed drain leaves the owned tree for the wrapper; never delete under its writer. + codexHome?.removeAfterDrain(); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); function nativeConfig(baseUrl: string): OcxConfig { return { port: 0, + clientIntegrations: { codex: false }, defaultProvider: "native", providers: { native: { adapter: "openai-responses", baseUrl, authMode: "forward", allowPrivateNetwork: true }, @@ -47,89 +70,48 @@ function messagesBody(): string { } test("transient upstream 502 -> client 529 overloaded_error; retry fired; log keeps upstream 502", async () => { - let calls = 0; - const upstream = Bun.serve({ - port: 0, - fetch() { - calls++; - return Response.json( - { error: { message: "An error occurred while processing your request. request ID test-42" } }, - // Retry-After: 0 keeps the pre-stream retry backoff instant AND must win over the "2" fallback. - { status: 502, headers: { "Retry-After": "0" } }, - ); - }, + upstreamResponse = () => Response.json( + { error: { message: "An error occurred while processing your request. request ID test-42" } }, + // Retry-After: 0 keeps the pre-stream retry backoff instant AND must win over the "2" fallback. + { status: 502, headers: { "Retry-After": "0" } }, + ); + const response = await fetch(new URL("/v1/messages", proxy!.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: messagesBody(), }); - saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); - const server = startServer(0); - try { - const response = await fetch(new URL("/v1/messages", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: messagesBody(), - }); - // Pre-stream transient retry (010) exhausts its attempts against the persistent 502 … - expect(calls).toBe(3); - // … then the Claude envelope reclassifies the transient 5xx as retryable overload (020). - expect(response.status).toBe(529); - expect(response.headers.get("retry-after")).toBe("0"); - const json = await response.json() as { type?: string; error?: { type?: string; message?: string } }; - expect(json.error?.type).toBe("overloaded_error"); - expect(json.error?.message).toBe("The provider failed to complete the request."); - // Diagnostics stay honest: the request log keeps the upstream status, not the client 529. - const entry = getRequestLogEntries().findLast(e => e.surface === "claude"); - expect(entry?.status).toBe(502); - } finally { - await server.stop(true); - await upstream.stop(true); - } + // Pre-stream transient retry (010) exhausts its attempts against the persistent 502 … + expect(calls).toBe(3); + // … then the Claude envelope reclassifies the transient 5xx as retryable overload (020). + expect(response.status).toBe(529); + expect(response.headers.get("retry-after")).toBe("0"); + const json = await response.json() as { type?: string; error?: { type?: string; message?: string } }; + expect(json.error?.type).toBe("overloaded_error"); + expect(json.error?.message).toBe("The provider failed to complete the request."); + // Diagnostics stay honest: the request log keeps the upstream status, not the client 529. + const entry = getRequestLogEntries().findLast(e => e.surface === "claude"); + expect(entry?.status).toBe(502); }, 15_000); test("transient 502 without Retry-After -> fallback Retry-After 2 on the 529 response", async () => { - const upstream = Bun.serve({ - port: 0, - fetch() { - return Response.json({ error: { message: "boom" } }, { status: 502 }); - }, + upstreamResponse = () => Response.json({ error: { message: "boom" } }, { status: 502 }); + const response = await fetch(new URL("/v1/messages", proxy!.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: messagesBody(), }); - saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); - const server = startServer(0); - try { - const response = await fetch(new URL("/v1/messages", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: messagesBody(), - }); - expect(response.status).toBe(529); - expect(response.headers.get("retry-after")).toBe("2"); - } finally { - await server.stop(true); - await upstream.stop(true); - } + expect(response.status).toBe(529); + expect(response.headers.get("retry-after")).toBe("2"); }, 15_000); test("non-transient upstream 400 stays 400 invalid_request_error; no retry", async () => { - let calls = 0; - const upstream = Bun.serve({ - port: 0, - fetch() { - calls++; - return Response.json({ error: { message: "bad request shape" } }, { status: 400 }); - }, + const response = await fetch(new URL("/v1/messages", proxy!.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: messagesBody(), }); - saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); - const server = startServer(0); - try { - const response = await fetch(new URL("/v1/messages", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: messagesBody(), - }); - expect(calls).toBe(1); - expect(response.status).toBe(400); - const json = await response.json() as { error?: { type?: string } }; - expect(json.error?.type).toBe("invalid_request_error"); - } finally { - await server.stop(true); - await upstream.stop(true); - } + expect(calls).toBe(1); + expect(response.status).toBe(400); + const json = await response.json() as { error?: { type?: string } }; + expect(json.error?.type).toBe("invalid_request_error"); }); diff --git a/tests/claude-integration/messages-error-contracts.test.ts b/tests/claude-integration/messages-error-contracts.test.ts index 2460c947686..d5ddeb82e27 100644 --- a/tests/claude-integration/messages-error-contracts.test.ts +++ b/tests/claude-integration/messages-error-contracts.test.ts @@ -134,6 +134,28 @@ for (const [size, cap, type, code] of [[1000, 250, "request_too_large", "transla expect(error.type).toBe(type); expect(error.code).toBe(code); }); +type SourceUses = { path: string; constructors: boolean[]; writers: number }; +let sourceUses: Promise | undefined; +/** Scan every source once with bounded reads; retain only oracle facts, not all source text. */ +function readSourceUses(): Promise { + return sourceUses ??= (async () => { + const paths = [...new Bun.Glob("src/**/*.ts").scanSync(repoRoot())]; + const uses = new Array(paths.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(8, paths.length) }, async () => { + for (;;) { + const index = next++; + if (index >= paths.length) return; + const path = paths[index]!, source = await Bun.file(repoPath(path)).text(); + uses[index] = { path: path.replaceAll("\\", "/"), + constructors: [...source.matchAll(/new AnthropicRequestError\(/g)].map(call => source.slice(0, call.index).endsWith("markProxyError(")), + writers: source.split("\n").filter(line => /anthropicError(Response|Body)\(/.test(line)).length }; + } + })); + return uses.filter(row => row.constructors.length || row.writers); + })(); +} + const ingressProducerRows: [string, number, string[]][] = [ ["src/claude/inbound.ts", 10, ["File-backed images require native Anthropic passthrough", "tool_result requires tool_use_id", "tool_use requires id and name", "malformed ocxr1 reasoning signature", "OpenCodex reasoning continuity cannot be replayed as an Anthropic signature", "each message must be an object", "unsupported message role: unsupported", "request body must be a JSON object", "model is required", "messages must be a non-empty array"]], ["src/protocols/codecs/messages-ir.ts", 9, ["tool_result requires tool_use_id", "tool_use requires id and name", "MALFORMED_REASONING_SIGNATURE", "OpenCodex reasoning continuity cannot be replayed as an Anthropic signature", "each message must be an object", "unsupported message role: unsupported", "request body must be a JSON object", "model is required", "messages must be a non-empty array"]], @@ -144,11 +166,9 @@ const ingressProducerRows: [string, number, string[]][] = [ ]; test("C1/C2 ingress constructors have explicit producer rows and local identity", async () => { const actual: Record = {}; - for (const path of new Bun.Glob("src/**/*.ts").scanSync(repoRoot())) { - const source = readFileSync(repoPath(path), "utf8"); - const calls = [...source.matchAll(/new AnthropicRequestError\(/g)]; - if (calls.length) actual[path.replaceAll("\\", "/")] = calls.length; - for (const call of calls) expect(source.slice(0, call.index).endsWith("markProxyError(")).toBe(true); + for (const { path, constructors } of await readSourceUses()) { + if (constructors.length) actual[path] = constructors.length; + for (const marked of constructors) expect(marked).toBe(true); } expect(actual).toEqual(Object.fromEntries(ingressProducerRows.map(([path, count]) => [path, count]))); for (const [path, , diagnostics] of ingressProducerRows) { @@ -168,12 +188,10 @@ const envelopeWriterRows = [ ["src/server/index/serve-options.ts", 4, "fixed endpoint auth/origin bypasses"], ["src/server/messages-native-tool-guard.ts", 1, "deferred local guard terminal"], ] as const; -test("W1/W3 envelope writers are covered by explicit dispositions", () => { +test("W1/W3 envelope writers are covered by explicit dispositions", async () => { const actual: Record = {}; - for (const path of new Bun.Glob("src/**/*.ts").scanSync(repoRoot())) { - const lines = readFileSync(repoPath(path), "utf8").split("\n"); - const count = lines.filter(line => /anthropicError(Response|Body)\(/.test(line)).length; - if (count) actual[path.replaceAll("\\", "/")] = count; + for (const { path, writers } of await readSourceUses()) { + if (writers) actual[path] = writers; } expect(actual).toEqual(Object.fromEntries(envelopeWriterRows.map(([path, count]) => [path, count]))); const bypassRows = [ diff --git a/tests/helpers/messages-overflow-case.ts b/tests/helpers/messages-overflow-case.ts index 4327b68fd7c..e19b55a3a24 100644 --- a/tests/helpers/messages-overflow-case.ts +++ b/tests/helpers/messages-overflow-case.ts @@ -49,7 +49,12 @@ export async function observeMessagesOverflowBody(response: Response, lane: Mess return { text, bodySha256, bodyLength }; } -export async function runMessagesOverflowCase(lane: MessagesOverflowLane, fixture: MessagesOverflowFixture) { +/** A slice may lend its real spend owner; single-case callers retain their own lease. */ +export async function runMessagesOverflowCase( + lane: MessagesOverflowLane, + fixture: MessagesOverflowFixture, + acquireSpendHome: () => () => void = acquireOwnedSpendHome, +) { const tail = (fixture.messageStop || fixture.id.startsWith("frame-complete-")) ? 'event: message_stop\ndata: {"type":"message_stop"}\n\n' : ""; const prefix = fixture.prefix ? `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: fixture.prefix } })}\n\nevent: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n` : ""; const error = { ...(fixture.type ? { type: fixture.type } : {}), ...(fixture.status !== undefined ? { status: fixture.status } : {}), message: `${fixture.message} ${overflowSentinel}`, ...(fixture.code !== undefined ? { code: fixture.code } : {}) }; @@ -112,7 +117,7 @@ export async function runMessagesOverflowCase(lane: MessagesOverflowLane, fixtur } else { const passthrough = lane === "passthrough" || lane === "count_tokens"; const native = lane.startsWith("native"); - release = acquireOwnedSpendHome(); clearKeyCooldowns(); + release = acquireSpendHome(); clearKeyCooldowns(); const builtin = lane === "native-fold-builtin"; const nativeFold = lane === "native-fold" || builtin; const reply = () => { diff --git a/tests/helpers/messages-overflow-differential.ts b/tests/helpers/messages-overflow-differential.ts index 5290c4d98eb..d41d32336ec 100644 --- a/tests/helpers/messages-overflow-differential.ts +++ b/tests/helpers/messages-overflow-differential.ts @@ -66,6 +66,7 @@ function replaceOnce(source: string, needle: string, replacement: string): strin } }); } const { runMessagesOverflowCase } = await import("./messages-overflow-case"); +const { acquireOwnedSpendHome } = await import("./owned-spend-home"); const { upstreamMessagesRequestIdHeaders } = await import("../../src/server/messages-response-headers"); if (base && upstreamMessagesRequestIdHeaders(new Headers({ "request-id": "req_fixture" }))["request-id"] !== "req_fixture") throw new Error("Baseline modules were not loaded"); const rows = []; @@ -80,35 +81,47 @@ function intern(values: T[], value: T): number { if (index >= 0) return index; return values.push(value) - 1; } -for (const lane of messagesOverflowLanes.filter(lane => !selectedLane || lane === selectedLane)) for (const input of applicableMessagesOverflowFixtures(lane).slice(first, first + count)) { - const observations = new Map(); - (globalThis as any).__messagesOverflowRecord = (tag: string, value: boolean) => { - const values = observations.get(tag) ?? []; values.push(value); observations.set(tag, values); - return value; - }; - if (input.fault) process.stderr.write(`fixture ${lane}/${input.id}\n`); - const result = await runMessagesOverflowCase(lane, input); - const passthrough = lane === "passthrough" || lane === "count_tokens"; - if (!passthrough && input.rawBody === undefined && !input.fault && !input.localBranch && observations.size === 0) throw new Error(`Missing baseline observation for ${lane}/${input.id}`); - const internalDecisions: Record = {}; - for (const [tag, values] of observations) { - if (values.some(value => value !== values[0])) throw new Error(`Inconsistent ${tag} decisions for ${lane}/${input.id}`); - internalDecisions[tag] = values[0]!; - } - if (base) { - const id = messagesOverflowCaseId(lane, input); - if (Object.hasOwn(outcomes, id)) throw new Error(`Duplicate case id: ${lane}/${input.id}`); - // Outcome-only tuples keep the base fixture small; dictionaries retain exact compared values. - const outcome: unknown[] = [messagesOverflowLanes.indexOf(lane), result.status, result.clientRecognizes, - intern(decisions, internalDecisions), input.classification[lane], intern(types, result.type ?? null), - intern(codes, result.code ?? null), intern(mediaTypes, result.mediaType), result.clientError]; - if (input.classification[lane] === "success" || input.fixedLocal) { - outcome.push(result.bodySha256, result.bodyLength); +// A real server owns its spend home for its lifetime. Keep that same ownership +// across this sequential slice, avoiding a full journal replay for every row. +// Pure encoder lanes never acquire a lease; single-case helpers keep their default. +let releaseSpendHome: (() => void) | undefined; +const acquireSliceSpendHome = () => { + releaseSpendHome ??= acquireOwnedSpendHome(); + return () => {}; // Borrowed by the row; the slice's finally owns release. +}; +try { + for (const lane of messagesOverflowLanes.filter(lane => !selectedLane || lane === selectedLane)) for (const input of applicableMessagesOverflowFixtures(lane).slice(first, first + count)) { + const observations = new Map(); + (globalThis as any).__messagesOverflowRecord = (tag: string, value: boolean) => { + const values = observations.get(tag) ?? []; values.push(value); observations.set(tag, values); + return value; + }; + if (input.fault) process.stderr.write(`fixture ${lane}/${input.id}\n`); + const result = await runMessagesOverflowCase(lane, input, acquireSliceSpendHome); + const passthrough = lane === "passthrough" || lane === "count_tokens"; + if (!passthrough && input.rawBody === undefined && !input.fault && !input.localBranch && observations.size === 0) throw new Error(`Missing baseline observation for ${lane}/${input.id}`); + const internalDecisions: Record = {}; + for (const [tag, values] of observations) { + if (values.some(value => value !== values[0])) throw new Error(`Inconsistent ${tag} decisions for ${lane}/${input.id}`); + internalDecisions[tag] = values[0]!; } - outcomes[id] = outcome; - } else rows.push({ lane, input, head: { ...result, internalDecisions } }); + if (base) { + const id = messagesOverflowCaseId(lane, input); + if (Object.hasOwn(outcomes, id)) throw new Error(`Duplicate case id: ${lane}/${input.id}`); + // Outcome-only tuples keep the base fixture small; dictionaries retain exact compared values. + const outcome: unknown[] = [messagesOverflowLanes.indexOf(lane), result.status, result.clientRecognizes, + intern(decisions, internalDecisions), input.classification[lane], intern(types, result.type ?? null), + intern(codes, result.code ?? null), intern(mediaTypes, result.mediaType), result.clientError]; + if (input.classification[lane] === "success" || input.fixedLocal) { + outcome.push(result.bodySha256, result.bodyLength); + } + outcomes[id] = outcome; + } else rows.push({ lane, input, head: { ...result, internalDecisions } }); + } +} finally { + delete (globalThis as any).__messagesOverflowRecord; + releaseSpendHome?.(); } -delete (globalThis as any).__messagesOverflowRecord; if (base) { const header = { schemaVersion: 8, baseCommit: "72e80d280d614bdf4f1aada9b233f6732404afec", caseTableSha256: messagesOverflowCaseTableHash(), columns: ["lane", "status", "clientRecognizes", "decisions", "class", "type", "code", "mediaType", "clientError", "bodySha256?", "bodyLength?"], diff --git a/tests/helpers/remove-tree.ts b/tests/helpers/remove-tree.ts index 5dbf5f2e622..be5079c4532 100644 --- a/tests/helpers/remove-tree.ts +++ b/tests/helpers/remove-tree.ts @@ -3,6 +3,10 @@ import { pendingBackgroundWork, settleBackgroundWork } from "../../src/lib/backg import { flushWindowsSecretAclReapsBeforeRemoval } from "../../src/lib/windows-secret-acl"; import { REMOVE_RETRY_BUDGET_MS, removeTestTempTree, removeTestTempTreeAsync } from "../../scripts/test-temp"; +// Background settlement and removal retries each own one budget. Leave scheduling headroom; +// an unreaped ACL child still blocks removal and may fail this caller-facing deadline. +export const BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS = 2 * REMOVE_RETRY_BUDGET_MS + 5_000; + type RemoveTreeWithRetryOptions = Readonly<{ remove?: (path: string) => void; sleep?: (milliseconds: number) => void; diff --git a/tests/providers/forward-admission-separation.test.ts b/tests/providers/forward-admission-separation.test.ts index bb6d66a88e1..6533437b323 100644 --- a/tests/providers/forward-admission-separation.test.ts +++ b/tests/providers/forward-admission-separation.test.ts @@ -6,7 +6,7 @@ import { saveConfig } from "../../src/config"; import { resolveFirstUsableOpenAiSidecar } from "../../src/providers/openai-sidecar"; import { startServer } from "../../src/server"; import type { OcxConfig } from "../../src/types"; -import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../helpers/remove-tree"; const originalFetch = globalThis.fetch; const previousHome = process.env.OPENCODEX_HOME; @@ -19,6 +19,7 @@ function forwardConfig(): OcxConfig { return { port: 0, hostname: "127.0.0.1", + clientIntegrations: { codex: false }, defaultProvider: "openai", openaiProviderTierVersion: 2, providers: { @@ -49,7 +50,7 @@ beforeEach(() => { }) as typeof fetch; }); -afterEach(() => { +afterEach(async () => { globalThis.fetch = originalFetch; if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; @@ -57,9 +58,9 @@ afterEach(() => { else process.env.OPENCODEX_API_AUTH_TOKEN = previousDataToken; if (previousAdminToken === undefined) delete process.env.OPENCODEX_ADMIN_AUTH_TOKEN; else process.env.OPENCODEX_ADMIN_AUTH_TOKEN = previousAdminToken; - if (testHome) removeTreeWithRetry(testHome); + if (testHome) await removeTreeAfterBackgroundWork(testHome); testHome = ""; -}); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); describe("management credentials never leave data-plane forwarding paths", () => { test("the shared OpenAI sidecar refuses a management bearer", async () => { diff --git a/tests/providers/kiro/kiro-pool-load-settings.test.ts b/tests/providers/kiro/kiro-pool-load-settings.test.ts index e661e93c87d..a67a0250d77 100644 --- a/tests/providers/kiro/kiro-pool-load-settings.test.ts +++ b/tests/providers/kiro/kiro-pool-load-settings.test.ts @@ -1,13 +1,14 @@ -import { afterEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { createHash, randomUUID } from "node:crypto"; import { mkdtempSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { acquireAccountLease } from "../../../src/oauth/kiro-account-load"; import { clearGenericFailoverHealth, preferredInitialAccount, rotateGenericOAuthAccountOnRefusal } from "../../../src/oauth/generic-account-failover"; -import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { getAccountSet, replaceProviderAccountSet } from "../../../src/oauth/store"; import { genericPoolSettingsDto, parseGenericPoolStrategy, parseKiroAccountCap, unifiedPoolSettingsDto } from "../../../src/oauth/pool-settings-capability"; import type { OcxConfig } from "../../../src/types"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; import { saveConfig, loadConfig } from "../../../src/config"; import { startServer } from "../../../src/server"; import { managementFetch } from "../../helpers/management-auth"; @@ -16,29 +17,35 @@ import { installIsolatedCodexHome } from "../../helpers/isolated-codex-home"; const oldHome = process.env.OPENCODEX_HOME; let home = ""; const held: Array<{ release(): void }> = []; -afterEach(() => { +afterEach(async () => { held.splice(0).forEach(lease => lease.release()); clearGenericFailoverHealth(); if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; - if (home) removeTreeWithRetry(home); + if (home) await removeTreeAfterBackgroundWork(home); home = ""; -}); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); async function seed(provider = "kiro") { home = mkdtempSync(join(tmpdir(), "ocx-load-settings-")); process.env.OPENCODEX_HOME = home; - for (let i = 0; i < 3; i++) await saveCredential(provider, { - access: `load-${i}`, refresh: `refresh-${i}`, expires: Date.now() + 3_600_000, - accountId: `account-${i}`, - }, { addAccount: true }); - const ids = getAccountSet(provider)!.accounts.map(row => row.id); - await setActiveAccount(provider, ids[0]!); + const now = Date.now(); + const accounts = Array.from({ length: 3 }, (_, i) => ({ + id: createHash("sha256").update(`account-${i}`).digest("hex").slice(0, 32), + credential: { access: `load-${i}`, refresh: `refresh-${i}`, expires: now + 3_600_000, accountId: `account-${i}` }, + addedAt: now, loginId: randomUUID(), + })); + // Pool behavior is under test; keep one real fenced/ACL-checked store write for fixture setup. + await replaceProviderAccountSet(provider, { activeAccountId: accounts[0]!.id, accounts }); + const stored = getAccountSet(provider)!; + const ids = stored.accounts.map(row => row.id); + expect(ids).toEqual(accounts.map(row => row.id)); + expect(stored.activeAccountId).toBe(ids[0]!); return ids; } function config(strategy: "least-loaded" | "quota" = "least-loaded", enabled = true): OcxConfig { - return { pool: { kernel: true }, providers: { kiro: { adapter: "kiro", authMode: "oauth", + return { pool: { kernel: true }, clientIntegrations: { codex: false }, providers: { kiro: { adapter: "kiro", authMode: "oauth", baseUrl: "https://runtime.us-east-1.kiro.dev", models: ["claude-sonnet-4.5"], oauthAccountFailover: { strategy, enabled, maxConcurrentPerAccount: 1 } } } } as OcxConfig; } @@ -53,15 +60,24 @@ test("Kiro reads least-loaded and maxConcurrentPerAccount in both DTOs", () => { expect(unifiedPoolSettingsDto(cfg, "anthropic", "anthropic").maxConcurrentPerAccount).toBeNull(); }); -test("Kiro persists and reads least-loaded and maxConcurrentPerAccount", async () => { - await seed(); - const isolated = installIsolatedCodexHome("ocx-load-settings-codex-"); - const saved = config("quota"); - saved.providers.xai = { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1", - models: ["test"] }; - saveConfig({ ...saved, defaultProvider: "kiro", hostname: "127.0.0.1", port: 0 } as OcxConfig); - const server = startServer(0); - try { +describe("Kiro pool settings management", () => { + let server: ReturnType | undefined; + let isolated: ReturnType | undefined; + beforeEach(async () => { + await seed(); + isolated = installIsolatedCodexHome("ocx-load-settings-codex-"); + const saved = config("quota"); + saved.providers.xai = { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1", + models: ["test"] }; + saveConfig({ ...saved, defaultProvider: "kiro", hostname: "127.0.0.1", port: 0 } as OcxConfig); + server = startServer(0); + }, 10_000); // Real Windows ACL/startup work has its own bound; the API assertions keep 5 seconds. + afterEach(async () => { + try { await server?.stop(true); } + finally { server = undefined; isolated?.restore(); isolated = undefined; } + }); + test("Kiro persists and reads least-loaded and maxConcurrentPerAccount", async () => { + if (!server) throw new Error("pool-settings fixture did not start"); const write = await managementFetch(new URL("/api/pool/settings", server.url), { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ provider: "kiro", strategy: "least-loaded", maxConcurrentPerAccount: 2 }) }); @@ -85,7 +101,7 @@ test("Kiro persists and reads least-loaded and maxConcurrentPerAccount", async ( expect(invalid.status).toBe(400); } expect(statSync(configFile).mtimeMs).toBe(before); - } finally { await server.stop(true); isolated.restore(); } + }); }); test("non-Kiro strategy and cap values are rejected by the provider parser", () => { diff --git a/tests/providers/kiro/kiro-quota-metrics.test.ts b/tests/providers/kiro/kiro-quota-metrics.test.ts index 932154526aa..0b67e0e9bce 100644 --- a/tests/providers/kiro/kiro-quota-metrics.test.ts +++ b/tests/providers/kiro/kiro-quota-metrics.test.ts @@ -1,13 +1,15 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createHash, randomUUID } from "node:crypto"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { getAccountSet, saveCredential } from "../../../src/oauth/store"; +import { getAccountSet, replaceProviderAccountSet, saveCredential } from "../../../src/oauth/store"; +import { oauthAccountLogLabel } from "../../../src/codex/account-label"; import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; import { cachedKiroQuotaMetricRows } from "../../../src/providers/kiro-quota-metrics"; import { sanitizeKiroQuota } from "../../../src/providers/kiro-account-state-disk"; import { createRequestMetricsOwner } from "../../../src/server/request-metrics"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; const previousHome = process.env.OPENCODEX_HOME; const realFetch = globalThis.fetch; @@ -17,13 +19,13 @@ beforeEach(() => { process.env.OPENCODEX_HOME = home; clearAccountQuotaCache(); }); -afterEach(() => { +afterEach(async () => { globalThis.fetch = realFetch; clearAccountQuotaCache(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; - removeTreeWithRetry(home); -}); + await removeTreeAfterBackgroundWork(home); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); async function account(id: string) { await saveCredential("kiro", { access: `access-${id}`, refresh: `refresh-${id}`, @@ -73,13 +75,22 @@ test("future, stale, reset-passed, and identity-mismatched rows are omitted", as test("stale leading accounts do not hide 32 later valid distinct gauges", async () => { const now = Date.now(); - for (let i = 0; i < 35; i++) { - const a = await account(String(i).padStart(3, "0")); - seed(a.id, i < 3 ? now - 31 * 60_000 : now); - } + const accounts = Array.from({ length: 35 }, (_, i) => { + const identity = String(i).padStart(3, "0"); + return { id: createHash("sha256").update(identity).digest("hex").slice(0, 32), + credential: { access: `access-${identity}`, refresh: `refresh-${identity}`, expires: now + 3_600_000, accountId: identity }, + addedAt: now, loginId: randomUUID() }; + }).sort((a, b) => a.id.localeCompare(b.id)); + // Exercise the real store once; this regression covers projection order, not repeated logins. + await replaceProviderAccountSet("kiro", { activeAccountId: accounts[34]!.id, accounts }); + const stored = getAccountSet("kiro")!.accounts; + expect(stored.map(row => row.id)).toEqual(accounts.map(row => row.id)); + expect(new Set(stored.map(row => row.credential.accountId)).size).toBe(35); + for (let i = 0; i < stored.length; i++) seed(stored[i]!.id, i < 3 ? now - 31 * 60_000 : now); const rows = cachedKiroQuotaMetricRows(now); expect(rows).toHaveLength(32); expect(new Set(rows.map(row => row.account)).size).toBe(32); + expect(rows.map(row => row.account)).toEqual(stored.slice(3).map(row => oauthAccountLogLabel(row.id, "kiro"))); }); test("duplicate or malformed labels never produce a second Prometheus series", () => { diff --git a/tests/windows/windows-tray.test.ts b/tests/windows/windows-tray.test.ts index cce64c7fafa..b1253b24dbe 100644 --- a/tests/windows/windows-tray.test.ts +++ b/tests/windows/windows-tray.test.ts @@ -652,6 +652,8 @@ describe("Windows tray packaging and command safety", () => { for (const scenario of ["Offline", "Online", "BadgeOverflowStdout", "BadgeOverflowStderr", "BadgeStaleFailure"]) { const directory = mkdtempSync(join(tmpdir(), "ocx-tray-fixture-")); const pidFile = join(directory, "pids.txt"); + // The delayed scenario holds open the empty-file window before PID publication. + if (scenario === "Online") writeFileSync(pidFile, ""); const child = Bun.spawn([process.execPath, helperPath("windows-tray-probe-child.ts"), "__startup-health"], { env: { ...process.env, OCX_PROBE_TEST_PID_FILE: pidFile, OCX_PROBE_TEST_SCENARIO: scenario }, stdout: "pipe", stderr: "pipe", @@ -670,9 +672,14 @@ describe("Windows tray packaging and command safety", () => { const stderr = drain(child.stderr); try { const deadline = Date.now() + INTERNAL_DEADLINE_MS; - while (!existsSync(pidFile) && child.exitCode === null && Date.now() < deadline) await Bun.sleep(10); + let publishedPid = ""; + do { + publishedPid = existsSync(pidFile) ? readFileSync(pidFile, "utf8").trim() : ""; + if (publishedPid === String(child.pid) || child.exitCode !== null || Date.now() >= deadline) break; + await Bun.sleep(10); + } while (true); expect(existsSync(pidFile), scenario).toBe(true); - expect(readFileSync(pidFile, "utf8").trim(), scenario).toBe(String(child.pid)); + expect(publishedPid, scenario).toBe(String(child.pid)); if (scenario.startsWith("BadgeOverflow")) { expect(await Promise.race([overflow.then(() => true), Bun.sleep(INTERNAL_DEADLINE_MS).then(() => false)]), scenario).toBe(true); }