From fa6cf36e7779f3eee50c2b3028b39b8c6261ec5c Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 23 Oct 2026 09:47:51 +0000 Subject: [PATCH 1/6] fix(responses): preserve gateway citations during compact --- src/adapters/openai-responses/passthrough.ts | 4 ++-- src/adapters/openai-responses/request-strips.ts | 2 +- src/server/responses/compact.ts | 7 +++++-- structure/providers-and-adapters.md | 2 +- tests/responses/compact-key-selection.test.ts | 10 ++++++---- .../responses-url-citation-title-repair.test.ts | 5 +++-- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/adapters/openai-responses/passthrough.ts b/src/adapters/openai-responses/passthrough.ts index 57c7ccf0e6b..e8abc6aab58 100644 --- a/src/adapters/openai-responses/passthrough.ts +++ b/src/adapters/openai-responses/passthrough.ts @@ -40,7 +40,7 @@ import { emittedFastWire, } from "../../providers/fastwire"; import { dropResponsesReasoningInputItems, mapRoutedResponsesReasoningEffort, normalizeConfiguredReasoningSummaryDelivery, sanitizeReasoningInputContent, stripDisabledReasoningSummaries, stripDisabledVerbosity, stripNoneReasoningSummary, stripUnsupportedReasoningSummaryDelivery } from "./reasoning"; -import { scrubOcxCompactionItems, stripCanonicalOnlyInputItems, stripCanonicalOnlyToolFields, stripCanonicalOnlyTopLevelFields, stripInternalChatMessageMetadataPassthrough, stripInvalidItemIds, stripItemIdsWhenUnstored, stripProviderInputItemMetadata, stripRejectedSamplingParams } from "./request-strips"; +import { repairUrlCitationAnnotations, scrubOcxCompactionItems, stripCanonicalOnlyInputItems, stripCanonicalOnlyToolFields, stripCanonicalOnlyTopLevelFields, stripInternalChatMessageMetadataPassthrough, stripInvalidItemIds, stripItemIdsWhenUnstored, stripProviderInputItemMetadata, stripRejectedSamplingParams } from "./request-strips"; import { notePlaintextReasoningDropped } from "./plaintext-reasoning-notice"; import { stripCanonicalForwardPromptCacheOptions, stripDeprecatedPromptCacheRetention } from "./prompt-cache"; import { isPlainObject } from "./internal"; @@ -315,7 +315,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig): outBody = repairLegacyDottedToolCallNames(outBody); // Shared HTTP/WS outbound projection, including concrete combo/fallback destinations. if (isOpenAiOperatedResponsesDestination(provider)) { - outBody = stripProviderInputItemMetadata(outBody); + outBody = repairUrlCitationAnnotations(stripProviderInputItemMetadata(outBody)); } if (!isCanonicalOpenAiForwardProvider(provider)) { if (provider.preserveResponsesMessageMetadata !== true) { diff --git a/src/adapters/openai-responses/request-strips.ts b/src/adapters/openai-responses/request-strips.ts index 6fb0f591854..2b5ab93d25c 100644 --- a/src/adapters/openai-responses/request-strips.ts +++ b/src/adapters/openai-responses/request-strips.ts @@ -24,7 +24,7 @@ export function stripProviderInputItemMetadata(body: unknown): unknown { input ??= [...body.input]; input[index] = next; } - return repairUrlCitationAnnotations(input ? { ...body, input } : body); + return input ? { ...body, input } : body; } function isUrlCitationIndex(value: unknown): value is number { diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 28cb3fbc718..f1e583c25ba 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -25,7 +25,7 @@ import { parseRequest } from "../../responses/parser"; import { buildCompactV1Output, COMPACT_PROMPT, decodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction"; import { applyReasoningRetention, appendRetentionNotice, type ReasoningRetentionResult } from "../../responses/reasoning-retention"; import { FORWARD_HEADERS, sanitizeReasoningInputContent } from "../../adapters/openai-responses"; -import { stripProviderInputItemMetadata } from "../../adapters/openai-responses/request-strips"; +import { repairUrlCitationAnnotations, stripProviderInputItemMetadata } from "../../adapters/openai-responses/request-strips"; import { expandPreviousResponseInput, previousResponseProviderState, rememberResponseState } from "../../responses/state"; import { repairLegacyDottedToolCallNames } from "../../responses/legacy-dotted-tool-name-repair"; import { NoEligiblePolicyCandidateError, routeCompactionModel } from "../../router"; @@ -950,7 +950,7 @@ export async function handleResponsesCompact( // call-name repair the adapter applies. A damaged item here refuses the compaction itself, // which is the request a long task depends on to keep going. // This branch is OpenAI-only and bypasses the adapter's Google replay projection (#7111). - const compactBody = repairLegacyDottedToolCallNames( + let compactBody = repairLegacyDottedToolCallNames( stripProviderInputItemMetadata(sanitizeReasoningInputContent(compactBodyRaw, { preserveRawReasoningContent: !isOpenAiOperatedResponsesDestination(compactProvider) && compactProvider.preserveResponsesReasoningContent === true, @@ -959,6 +959,9 @@ export async function handleResponsesCompact( requirePlaintextReasoning: requireCompactPlaintextReasoning, })), ) as typeof compactBodyRaw; + if (isOpenAiOperatedResponsesDestination(compactProvider)) { + compactBody = repairUrlCitationAnnotations(compactBody) as typeof compactBodyRaw; + } { const binding = conversationStateBindingFromAuth(authCtx, codexPoolAffinityKey(req.headers)); if (binding) { diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index df5e3e01c37..658dc8550d2 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -453,7 +453,7 @@ The shared Responses path follows the [bounded multipart recovery contract](suba Independent of that opt-in, the same final builder projects unreferenced `web_search_call` cells for every Responses destination whenever the final body declares no hosted search tool (`declaresHostedWebSearch` in `src/adapters/openai-responses/compaction-search-history.ts`: a `web_search*` type in `tools` or a Lite `additional_tools` wrapper, read after `stripUnsupportedHostedTools`). A function tool named `web_search` is not a declaration. This covers native v2 compaction triggers with an empty tool list; portable compaction is already projected by its builder and native `/responses/compact` is a separate path. `tests/adapters/openai/openai-native-hosted-search-history.test.ts` pins the #7250 request, the declared controls and the stripped-tool case. -The Codex-native Lite-history policy for ordinary turns and native v2 compaction triggers in `src/adapters/openai-responses/passthrough.ts` is off by default: it projects unreferenced remaining `web_search_call` cells through `src/adapters/openai-responses/compaction-search-history.ts` only when the selected provider has `projectResponsesLiteHostedSearchHistory === true`, after bridge restoration, query backfill and schema normalization. The optional setting is a strictly validated boolean; omission, `false`, and malformed runtime values never enable projection. Exact canonical ChatGPT `/responses` still requires explicit Responses Lite intent: final `x-openai-internal-codex-responses-lite` header `true`/`false` is authoritative, otherwise an exact `true` in a valid string-map `client_metadata.ws_request_header_x_openai_internal_codex_responses_lite` selects the existing WS metadata intent. This reads the final request without changing headers, metadata or transport behavior. Non-Lite or unrecognized effective intent keeps structural hosted history; models, subscription auth, missing hosted tools and `additional_tools` alone are not a gate. The policy remains independent of diagnostic controls; only a same-scope bridge memo proves a restorable call/result, never an id prefix, model name or cache miss. Same-input `item_reference.id` matches retain their hosted cell; `store:false` removes references earlier. Notes are untrusted metadata, not instructions/page content: action and top-level `sources` copy only url/title/type, strings cap at 2,048 UTF-16 units, lists at 20, projected notes at 65,536 bytes with an omission note. Tool declarations/Lite wrappers, choices, non-hosted order, call/results, reasoning and caller raw input keep their contracts. Native `/responses/compact` stays unchanged, including account scrubbing and opaque output; portable/routed summaries keep their existing triggers/default projection (now preserving allowlisted top-level sources), not the ordinary reference exemption. `tests/adapters/openai/openai-native-hosted-search-history.test.ts` and `tests/responses/responses-native-compact-search-history.test.ts` pin these boundaries. This is an explicit lossy policy, not issuer inference, retry, or a claim that Codex rejects valid hosted schema. +The Codex-native Lite-history policy for ordinary turns and native v2 compaction triggers in `src/adapters/openai-responses/passthrough.ts` is off by default: it projects unreferenced remaining `web_search_call` cells through `src/adapters/openai-responses/compaction-search-history.ts` only when the selected provider has `projectResponsesLiteHostedSearchHistory === true`, after bridge restoration, query backfill and schema normalization. The optional setting is a strictly validated boolean; omission, `false`, and malformed runtime values never enable projection. Exact canonical ChatGPT `/responses` still requires explicit Responses Lite intent: final `x-openai-internal-codex-responses-lite` header `true`/`false` is authoritative, otherwise an exact `true` in a valid string-map `client_metadata.ws_request_header_x_openai_internal_codex_responses_lite` selects the existing WS metadata intent. This reads the final request without changing headers, metadata or transport behavior. Non-Lite or unrecognized effective intent keeps structural hosted history; models, subscription auth, missing hosted tools and `additional_tools` alone are not a gate. The policy remains independent of diagnostic controls; only a same-scope bridge memo proves a restorable call/result, never an id prefix, model name or cache miss. Same-input `item_reference.id` matches retain their hosted cell; `store:false` removes references earlier. Notes are untrusted metadata, not instructions/page content: action and top-level `sources` copy only url/title/type, strings cap at 2,048 UTF-16 units, lists at 20, projected notes at 65,536 bytes with an omission note. Tool declarations/Lite wrappers, choices, non-hosted order, call/results, reasoning and caller raw input keep their contracts. Native `/responses/compact` stays unchanged, including account scrubbing and opaque output; portable/routed summaries keep their existing triggers/default projection (now preserving allowlisted top-level sources), not the ordinary reference exemption. `tests/adapters/openai/openai-native-hosted-search-history.test.ts` and `tests/responses/responses-native-compact-search-history.test.ts` pin these boundaries. This is an explicit lossy policy, not issuer inference, retry, or a claim that Codex rejects valid hosted schema. OpenAI-operated Responses destinations repair replayed `url_citation` annotations to the public API's required title and integer-index shape. The ordinary adapter and direct native `/responses/compact` path apply that projection only after destination classification; opting a third-party Responses gateway into native compact support is a protocol capability and does not authorize rewriting its citation history. Provider metadata stripping remains a separate projection. The opt-in key-auth Responses hosted-search bridge in `src/server/responses/passthrough-delivery.ts` captures the request binding that served the first leg, after any permitted initial reselection. Before every continuation dispatch, after provider pacing, that binding must remain an API-key selection matching the configured entry, reference, revision, resolved key, authentication mode, and base URL; a disabled or removed provider fails the same check. Drift produces the bridge's failed terminal diff --git a/tests/responses/compact-key-selection.test.ts b/tests/responses/compact-key-selection.test.ts index 289e0689872..9fb2b1808ec 100644 --- a/tests/responses/compact-key-selection.test.ts +++ b/tests/responses/compact-key-selection.test.ts @@ -65,10 +65,10 @@ async function queue(config: OcxConfig, id: string) { await Bun.sleep(5); } } -function start(config: OcxConfig, abort: AbortController) { +function start(config: OcxConfig, abort: AbortController, input: unknown[] = [{ role: "user", content: "fixture question" }]) { const logCtx: RequestLogContext = { model: "", provider: "" }; const req = new Request("http://localhost/v1/responses/compact", { method: "POST", headers: { "content-type": "application/json" }, - signal: AbortSignal.any([abort.signal, AbortSignal.timeout(5_000)]), body: JSON.stringify({ model: `${config.defaultProvider}/test`, input: [{ role: "user", content: "fixture question" }], stream: false }) }); + signal: AbortSignal.any([abort.signal, AbortSignal.timeout(5_000)]), body: JSON.stringify({ model: `${config.defaultProvider}/test`, input, stream: false }) }); const pending = handleResponsesCompact(req, config, logCtx).then(async response => ({ response, text: await response.text(), logCtx })); void pending.catch(() => {}); return pending; @@ -308,9 +308,11 @@ for (const reselect of [false, true]) test(`gateway compact preserves custom tra provider.supportsNativeCompactEndpoint = true; provider.headers = { "x-gateway-static": "initial" }; saveConfig(config); + const citation = { type: "url_citation", url: "https://gateway.example/source", start_index: "3", end_index: -1 }; + const input = [{ role: "user", content: "fixture question" }, { type: "message", role: "assistant", content: [{ type: "output_text", text: "fixture answer", annotations: [citation] }] }]; const held = reselect ? await waitForProviderRequestSlot(config.defaultProvider, provider) : undefined; const abort = new AbortController(); - const pending = start(config, abort); + const pending = start(config, abort, input); try { if (reselect) { await queue(config, "k1"); @@ -327,7 +329,7 @@ for (const reselect of [false, true]) test(`gateway compact preserves custom tra expect(seen[0]!.url).toBe(`https://gateway.example/v1/${reselect ? "refreshed-responses" : "custom-responses"}/compact`); expect(seen[0]!.headers.get("x-gateway-static")).toBe(reselect ? "replacement" : "initial"); expect(seen[0]!.headers.get("chatgpt-account-id")).toBeNull(); - expect(seen[0]!.body).toMatchObject({ model: "test", input: [{ role: "user", content: "fixture question" }] }); + expect(seen[0]!.body).toMatchObject({ model: "test", input }); noLeaks(config); } finally { abort.abort(); held?.release(); await pending.catch(() => undefined); } }); diff --git a/tests/responses/responses-url-citation-title-repair.test.ts b/tests/responses/responses-url-citation-title-repair.test.ts index aba11a1b83b..9ec46566495 100644 --- a/tests/responses/responses-url-citation-title-repair.test.ts +++ b/tests/responses/responses-url-citation-title-repair.test.ts @@ -64,9 +64,10 @@ describe("url_citation annotation repair on outbound history", () => { expect(repairUrlCitationAnnotations(null)).toBeNull(); }); - test("stripProviderInputItemMetadata (the OpenAI-operated outbound projection) applies it", () => { + test("metadata stripping and citation repair remain independent projections", () => { const body = { input: [{ role: "user", content: "hi" }, assistantWith([{ type: "url_citation", url: "https://a.example", start_index: 0, end_index: 0 }])] }; - expect(annotationsOf(stripProviderInputItemMetadata(body))[0]!.title).toBe("https://a.example"); + expect(annotationsOf(stripProviderInputItemMetadata(body))[0]).not.toHaveProperty("title"); + expect(annotationsOf(repairUrlCitationAnnotations(body))[0]!.title).toBe("https://a.example"); }); const native: OcxProviderConfig[] = [ From b095b04adaf57c1ef1b853eddeaaba6c62792acc Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 23 Oct 2026 09:57:58 +0000 Subject: [PATCH 2/6] fix(responses): project compact citations at the final destination Apply citation repair in the native compact send builder after queued provider refresh. Preserve original gateway annotations while retaining repair for official OpenAI API and ChatGPT sends. Cover both destination transitions, exact annotation preservation, independent metadata stripping, and native positive controls. Document the destination boundary in the public adapter reference and structure contracts. --- .../src/content/docs/reference/adapters.md | 6 +++ src/server/responses/compact.ts | 10 ++-- structure/INDEX.md | 5 +- structure/manifest.json | 11 +++++ structure/providers-and-adapters.md | 4 +- structure/transports/responses-citations.md | 19 ++++++++ structure/transports/responses.md | 2 +- tests/responses/compact-key-selection.test.ts | 48 +++++++++++++++++-- ...ses-native-item-extensions-compact.test.ts | 13 ++++- ...esponses-url-citation-title-repair.test.ts | 20 ++++++-- 10 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 structure/transports/responses-citations.md diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 435e83f1a90..018ad34f51e 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -169,6 +169,12 @@ API, the adapter removes item-level `extra_content` from the outgoing HTTP reque create. Native `/responses/compact` applies the same projection. The original history keeps the Google thought signature for later Google turns. Other Responses gateways retain their extensions. +OpenAI-operated destinations receive replayed `url_citation` annotations with missing or invalid +titles and indices repaired. This applies to ordinary HTTP/WebSocket requests and native +`/responses/compact`. Custom gateways keep the original annotations even when +`supportsNativeCompactEndpoint: true` enables native compaction. Citation repair leaves the +client's stored history unchanged. + The adapter preserves the incoming client's `User-Agent` as a fallback in both auth modes because some Responses-compatible providers use the Codex client fingerprint for compatibility behavior. An explicitly configured provider `User-Agent` remains authoritative regardless of header casing; diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index f1e583c25ba..f9d687e1e09 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -949,8 +949,8 @@ export async function handleResponsesCompact( // #5095: the compact endpoint forwards `raw` directly, so it needs the same legacy dotted // call-name repair the adapter applies. A damaged item here refuses the compaction itself, // which is the request a long task depends on to keep going. - // This branch is OpenAI-only and bypasses the adapter's Google replay projection (#7111). - let compactBody = repairLegacyDottedToolCallNames( + // Direct compact bypasses the adapter's Google replay projection (#7111). + const compactBody = repairLegacyDottedToolCallNames( stripProviderInputItemMetadata(sanitizeReasoningInputContent(compactBodyRaw, { preserveRawReasoningContent: !isOpenAiOperatedResponsesDestination(compactProvider) && compactProvider.preserveResponsesReasoningContent === true, @@ -959,9 +959,6 @@ export async function handleResponsesCompact( requirePlaintextReasoning: requireCompactPlaintextReasoning, })), ) as typeof compactBodyRaw; - if (isOpenAiOperatedResponsesDestination(compactProvider)) { - compactBody = repairUrlCitationAnnotations(compactBody) as typeof compactBodyRaw; - } { const binding = conversationStateBindingFromAuth(authCtx, codexPoolAffinityKey(req.headers)); if (binding) { @@ -1163,8 +1160,9 @@ export async function handleResponsesCompact( if (!isOpenAiOperatedResponsesDestination(sendProvider) && sendProvider.preserveResponsesMessageMetadata !== true) { replayBody = stripInternalChatMessageMetadataPassthrough(replayBody) as typeof replayBody; } + // A queued key selection can refresh the destination; project only for this send. return isOpenAiOperatedResponsesDestination(sendProvider) - ? replayBody : stripCanonicalOnlyInputItems(stripCanonicalOnlyTopLevelFields(replayBody)); + ? repairUrlCitationAnnotations(replayBody) : stripCanonicalOnlyInputItems(stripCanonicalOnlyTopLevelFields(replayBody)); }; const doFetch = (upstreamRecovery?: UpstreamSendRecovery) => fetchWithHeaderTimeout( compactUrl, diff --git a/structure/INDEX.md b/structure/INDEX.md index e4982e65274..002be9cd5cd 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -51,6 +51,7 @@ The wire surfaces a client actually talks to. | [`transports/anthropic-thinking-recovery.md`](transports/anthropic-thinking-recovery.md) | One-shot exact signature-binding recovery before output, and successful-retry memo boundaries. | | [`transports/responses.md`](transports/responses.md) | The Responses HTTP/SSE endpoint, dispatch, credential and upload boundaries, and core module ownership. | | [`transports/native-reasoning-retention.md`](transports/native-reasoning-retention.md) | Optional native reasoning ciphertext forwarding on model/account switches, configuration and management controls. | +| [`transports/responses-citations.md`](transports/responses-citations.md) | Destination-gated citation repair for ordinary Responses and every native compact send. | | [`transports/responses-wire-shapes.md`](transports/responses-wire-shapes.md) | Mixed-wire model defaults, xAI agent-message continuation, declared-tool membership, and passthrough SSE stream shapes. | | [`transports/responses-failover.md`](transports/responses-failover.md) | Upstream reset retry, the ambiguous-resend gate, combo failover and commit boundaries, compaction routing, and output headroom. | | [`transports/responses-combo-boundaries.md`](transports/responses-combo-boundaries.md) | Combo preflight output commitment, turn lease lifetime, and bounded hop warnings. | @@ -178,7 +179,8 @@ A source area can be described by more than one doc, because these docs are orga | `src/adapters/openai-chat.ts` | [`providers/mimo-bare-calls.md`](providers/mimo-bare-calls.md) | | `src/adapters/openai-chat/` | [`providers/mimo-bare-calls.md`](providers/mimo-bare-calls.md) | | `src/adapters/openai-responses/opengateway-input.ts` | [`providers/opengateway.md`](providers/opengateway.md) | -| `src/adapters/openai-responses/passthrough.ts` | [`providers/opengateway.md`](providers/opengateway.md) | +| `src/adapters/openai-responses/passthrough.ts` | [`transports/responses-citations.md`](transports/responses-citations.md)
[`providers/opengateway.md`](providers/opengateway.md) | +| `src/adapters/openai-responses/request-strips.ts` | [`transports/responses-citations.md`](transports/responses-citations.md) | | `src/adapters/workbuddy-schema.ts` | [`providers/workbuddy.md`](providers/workbuddy.md) | | `src/adapters/workbuddy.ts` | [`providers/workbuddy.md`](providers/workbuddy.md) | | `src/advisor/` | [`advisor.md`](advisor.md) | @@ -295,6 +297,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/server/responses/advisor-plan-slot.ts` | [`advisor.md`](advisor.md) | | `src/server/responses/advisor-slot.ts` | [`advisor.md`](advisor.md) | | `src/server/responses/anthropic-thinking-recovery.ts` | [`transports/anthropic-thinking-recovery.md`](transports/anthropic-thinking-recovery.md) | +| `src/server/responses/compact.ts` | [`transports/responses-citations.md`](transports/responses-citations.md) | | `src/server/responses/opengateway-completion-guard.ts` | [`providers/opengateway.md`](providers/opengateway.md) | | `src/server/responses/passthrough-completion.ts` | [`providers/opengateway.md`](providers/opengateway.md) | | `src/server/responses/passthrough-dispatch.ts` | [`providers/opengateway.md`](providers/opengateway.md) | diff --git a/structure/manifest.json b/structure/manifest.json index 3f6079e5566..212ed558ada 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -302,6 +302,17 @@ "gui/" ] }, + { + "path": "transports/responses-citations.md", + "tier": 3, + "title": "Replayed URL Citations", + "scope": "Destination-gated citation repair for ordinary Responses and every native compact send.", + "documents": [ + "src/adapters/openai-responses/request-strips.ts", + "src/adapters/openai-responses/passthrough.ts", + "src/server/responses/compact.ts" + ] + }, { "path": "transports/responses-wire-shapes.md", "tier": 3, diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 658dc8550d2..9e755125211 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -445,7 +445,7 @@ picker, so the catalog and the wire agree; a rung the upstream then refuses is r error rather than replayed without the effort, because the operator asked for it. The flag is part of the compatibility behavior record for the same reason as above. -The shared Responses path follows the [bounded multipart recovery contract](subagents.md#multipart-encrypted-task-recovery); credential admission and retry policy remain unchanged. +The shared Responses path follows the [bounded multipart recovery contract](subagents.md#multipart-encrypted-task-recovery); credential admission and retry policy remain unchanged. Ordinary requests and native compact apply the [destination-specific citation projection](transports/responses-citations.md) independently of provider metadata stripping. ## Hosted-search continuation binding > Decision record: [ADR-6930](decisions/ADR-6930-hosted-search-continuation-binding.md) > Decision record: [ADR-6931](decisions/ADR-6931-lite-hosted-search-opt-in.md) @@ -453,7 +453,7 @@ The shared Responses path follows the [bounded multipart recovery contract](suba Independent of that opt-in, the same final builder projects unreferenced `web_search_call` cells for every Responses destination whenever the final body declares no hosted search tool (`declaresHostedWebSearch` in `src/adapters/openai-responses/compaction-search-history.ts`: a `web_search*` type in `tools` or a Lite `additional_tools` wrapper, read after `stripUnsupportedHostedTools`). A function tool named `web_search` is not a declaration. This covers native v2 compaction triggers with an empty tool list; portable compaction is already projected by its builder and native `/responses/compact` is a separate path. `tests/adapters/openai/openai-native-hosted-search-history.test.ts` pins the #7250 request, the declared controls and the stripped-tool case. -The Codex-native Lite-history policy for ordinary turns and native v2 compaction triggers in `src/adapters/openai-responses/passthrough.ts` is off by default: it projects unreferenced remaining `web_search_call` cells through `src/adapters/openai-responses/compaction-search-history.ts` only when the selected provider has `projectResponsesLiteHostedSearchHistory === true`, after bridge restoration, query backfill and schema normalization. The optional setting is a strictly validated boolean; omission, `false`, and malformed runtime values never enable projection. Exact canonical ChatGPT `/responses` still requires explicit Responses Lite intent: final `x-openai-internal-codex-responses-lite` header `true`/`false` is authoritative, otherwise an exact `true` in a valid string-map `client_metadata.ws_request_header_x_openai_internal_codex_responses_lite` selects the existing WS metadata intent. This reads the final request without changing headers, metadata or transport behavior. Non-Lite or unrecognized effective intent keeps structural hosted history; models, subscription auth, missing hosted tools and `additional_tools` alone are not a gate. The policy remains independent of diagnostic controls; only a same-scope bridge memo proves a restorable call/result, never an id prefix, model name or cache miss. Same-input `item_reference.id` matches retain their hosted cell; `store:false` removes references earlier. Notes are untrusted metadata, not instructions/page content: action and top-level `sources` copy only url/title/type, strings cap at 2,048 UTF-16 units, lists at 20, projected notes at 65,536 bytes with an omission note. Tool declarations/Lite wrappers, choices, non-hosted order, call/results, reasoning and caller raw input keep their contracts. Native `/responses/compact` stays unchanged, including account scrubbing and opaque output; portable/routed summaries keep their existing triggers/default projection (now preserving allowlisted top-level sources), not the ordinary reference exemption. `tests/adapters/openai/openai-native-hosted-search-history.test.ts` and `tests/responses/responses-native-compact-search-history.test.ts` pin these boundaries. This is an explicit lossy policy, not issuer inference, retry, or a claim that Codex rejects valid hosted schema. OpenAI-operated Responses destinations repair replayed `url_citation` annotations to the public API's required title and integer-index shape. The ordinary adapter and direct native `/responses/compact` path apply that projection only after destination classification; opting a third-party Responses gateway into native compact support is a protocol capability and does not authorize rewriting its citation history. Provider metadata stripping remains a separate projection. +The Codex-native Lite-history policy for ordinary turns and native v2 compaction triggers in `src/adapters/openai-responses/passthrough.ts` is off by default: it projects unreferenced remaining `web_search_call` cells through `src/adapters/openai-responses/compaction-search-history.ts` only when the selected provider has `projectResponsesLiteHostedSearchHistory === true`, after bridge restoration, query backfill and schema normalization. The optional setting is a strictly validated boolean; omission, `false`, and malformed runtime values never enable projection. Exact canonical ChatGPT `/responses` still requires explicit Responses Lite intent: final `x-openai-internal-codex-responses-lite` header `true`/`false` is authoritative, otherwise an exact `true` in a valid string-map `client_metadata.ws_request_header_x_openai_internal_codex_responses_lite` selects the existing WS metadata intent. This reads the final request without changing headers, metadata or transport behavior. Non-Lite or unrecognized effective intent keeps structural hosted history; models, subscription auth, missing hosted tools and `additional_tools` alone are not a gate. The policy remains independent of diagnostic controls; only a same-scope bridge memo proves a restorable call/result, never an id prefix, model name or cache miss. Same-input `item_reference.id` matches retain their hosted cell; `store:false` removes references earlier. Notes are untrusted metadata, not instructions/page content: action and top-level `sources` copy only url/title/type, strings cap at 2,048 UTF-16 units, lists at 20, projected notes at 65,536 bytes with an omission note. Tool declarations/Lite wrappers, choices, non-hosted order, call/results, reasoning and caller raw input keep their contracts. Native `/responses/compact` stays unchanged, including account scrubbing and opaque output; portable/routed summaries keep their existing triggers/default projection (now preserving allowlisted top-level sources), not the ordinary reference exemption. `tests/adapters/openai/openai-native-hosted-search-history.test.ts` and `tests/responses/responses-native-compact-search-history.test.ts` pin these boundaries. This is an explicit lossy policy, not issuer inference, retry, or a claim that Codex rejects valid hosted schema. The opt-in key-auth Responses hosted-search bridge in `src/server/responses/passthrough-delivery.ts` captures the request binding that served the first leg, after any permitted initial reselection. Before every continuation dispatch, after provider pacing, that binding must remain an API-key selection matching the configured entry, reference, revision, resolved key, authentication mode, and base URL; a disabled or removed provider fails the same check. Drift produces the bridge's failed terminal diff --git a/structure/transports/responses-citations.md b/structure/transports/responses-citations.md new file mode 100644 index 00000000000..1e7f465a312 --- /dev/null +++ b/structure/transports/responses-citations.md @@ -0,0 +1,19 @@ +# Replayed URL Citations + +OpenAI-operated Responses destinations repair replayed `url_citation` annotations to the required +string title and non-negative integer indices through `repairUrlCitationAnnotations` in +`src/adapters/openai-responses/request-strips.ts`. Provider metadata stripping remains an +independent projection. The ordinary HTTP/WS adapter in +`src/adapters/openai-responses/passthrough.ts` applies citation repair only after destination +classification. + +The direct native `/responses/compact` path in `src/server/responses/compact.ts` applies the same +gate in its final body builder for every physical send, including a provider destination refreshed +after key admission. It keeps the shared history free of citation repair so an OpenAI-to-gateway +refresh preserves the gateway's original annotations and the reverse refresh still repairs OpenAI +history. Opting a third-party gateway into native compaction declares protocol support, not +permission to rewrite its citations. `tests/responses/compact-key-selection.test.ts` covers exact +gateway preservation and both refresh directions; +`tests/responses/responses-native-item-extensions-compact.test.ts` covers official API and ChatGPT +compact repair, while `tests/responses/responses-url-citation-title-repair.test.ts` covers ordinary +HTTP/WS projection, helper independence and input immutability. diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 02f6eb0a8b6..d2a19b5bcb1 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -329,7 +329,7 @@ now: the TTL was previously evaluated only on read or on a generation change, so again held its entry for the life of the process. > Decision record: [ADR-0038](../decisions/ADR-0038-responses-http-sse.md) -Custom `openai-responses` gateways declare native v2 triggers with `supportsNativeCompactionTrigger` and native v1 endpoints with `supportsNativeCompactEndpoint`; neither flag implies blob replay or OpenAI identity. `src/adapters/openai-responses/transport.ts` shares normal URL/auth/header construction with custom native v1 key/local/forward routes; generic OAuth retains routed compaction so the existing credential selection, refresh and dispatch fence own authentication. Native v2 remains an independent declaration. Custom native v1 honors `preserveResponsesReasoningContent: true` in both sanitizer passes only for custom destinations; absent/false and OpenAI-operated compact destinations blank raw reasoning content. Proxy-owned `ocxr1:` envelopes are always stripped, and plaintext preservation grants neither native generation nor ciphertext replay. Both custom v1 sanitizer passes omit non-array reasoning content channels and reuse `requiresPlaintextReasoningReplay` from `src/adapters/openai-responses/passthrough.ts`: custom providers with plaintext preservation and strict tool-result adjacency strip opaque reasoning ciphertext and backfill missing `reasoning_text` from `summary_text` or the existing minimal placeholder, while preserving available plaintext. Each physical send applies `dropResponsesReasoningItems: true` and strips private input message metadata unless `preserveResponsesMessageMetadata: true`, using the ordinary adapter policy; OpenAI-operated compact metadata, reasoning items and null channels retain their established policy. Custom v1 scrubs compaction ciphertext, reasoning ciphertext and foreign reasoning ids after each serving bind, using ordinary turns' parent-thread-first replay scope; provenance commits require a successful HTTP response and a validated replacement history with replayable compact ciphertext or nonblank message text, absent/completed status and absent/null error (`nativeCompactResponseSchema` in `src/responses/schema.ts`). Invalid or empty bodies preserve the prior identity; validation never rewrites the relayed body or request-log metadata. Daybreak endpoint restrictions remain canonical-only. A native 404 handoff preserves User-Agent and allowlisted provider-opted-in metadata through `src/server/responses/core-auth.ts` without widening credential forwarding. Every custom native v1 send carries only the gateway transport's headers, and Codex pool recovery (401 refresh replay, quota and transient alternates) and pool outcome recording skip gateway sends, so recovery cannot put a stored ChatGPT credential on a gateway wire (`tests/responses/native-compaction-gateway-credentials.test.ts`). Readable reasoning retention also bypasses native v2-capable gateways; the capability predicate controls local retention as well as dispatch. V1 fallback remains portable even for v2-capable gateways; cross-provider overrides still force portable summaries. `tests/responses/native-compaction-gateway.test.ts`, `tests/responses/native-compaction-gateway-policy.test.ts` `tests/responses/native-compaction-provenance.test.ts` and `tests/responses/native-compaction-oauth.test.ts` pin these boundaries. Key-auth compact revalidates the serving key after pacing, rebuilding custom gateway paths, transport headers and replay sanitization against that physical selection; `tests/responses/compact-key-selection.test.ts` covers initial gateway dispatch and queued reselection. +Custom `openai-responses` gateways declare native v2 triggers with `supportsNativeCompactionTrigger` and native v1 endpoints with `supportsNativeCompactEndpoint`; neither flag implies blob replay or OpenAI identity. `src/adapters/openai-responses/transport.ts` shares normal URL/auth/header construction with custom native v1 key/local/forward routes; generic OAuth retains routed compaction so the existing credential selection, refresh and dispatch fence own authentication. Native v2 remains an independent declaration. Custom native v1 honors `preserveResponsesReasoningContent: true` in both sanitizer passes only for custom destinations; absent/false and OpenAI-operated compact destinations blank raw reasoning content. Proxy-owned `ocxr1:` envelopes are always stripped, and plaintext preservation grants neither native generation nor ciphertext replay. Both custom v1 sanitizer passes omit non-array reasoning content channels and reuse `requiresPlaintextReasoningReplay` from `src/adapters/openai-responses/passthrough.ts`: custom providers with plaintext preservation and strict tool-result adjacency strip opaque reasoning ciphertext and backfill missing `reasoning_text` from `summary_text` or the existing minimal placeholder, while preserving available plaintext. Each physical send applies `dropResponsesReasoningItems: true` and strips private input message metadata unless `preserveResponsesMessageMetadata: true`, using the ordinary adapter policy; OpenAI-operated compact metadata, reasoning items and null channels retain their established policy. Custom v1 scrubs compaction ciphertext, reasoning ciphertext and foreign reasoning ids after each serving bind, using ordinary turns' parent-thread-first replay scope; provenance commits require a successful HTTP response and a validated replacement history with replayable compact ciphertext or nonblank message text, absent/completed status and absent/null error (`nativeCompactResponseSchema` in `src/responses/schema.ts`). Invalid or empty bodies preserve the prior identity; validation never rewrites the relayed body or request-log metadata. Daybreak endpoint restrictions remain canonical-only. A native 404 handoff preserves User-Agent and allowlisted provider-opted-in metadata through `src/server/responses/core-auth.ts` without widening credential forwarding. Every custom native v1 send carries only the gateway transport's headers, and Codex pool recovery (401 refresh replay, quota and transient alternates) and pool outcome recording skip gateway sends, so recovery cannot put a stored ChatGPT credential on a gateway wire (`tests/responses/native-compaction-gateway-credentials.test.ts`). Readable reasoning retention also bypasses native v2-capable gateways; the capability predicate controls local retention as well as dispatch. V1 fallback remains portable even for v2-capable gateways; cross-provider overrides still force portable summaries. `tests/responses/native-compaction-gateway.test.ts`, `tests/responses/native-compaction-gateway-policy.test.ts` `tests/responses/native-compaction-provenance.test.ts` and `tests/responses/native-compaction-oauth.test.ts` pin these boundaries. Key-auth compact revalidates the serving key after pacing, rebuilding custom gateway paths, transport headers, replay sanitization and [citation projection](responses-citations.md) against that physical selection; `tests/responses/compact-key-selection.test.ts` covers initial gateway dispatch and queued reselection. Canonical native v1's 404 fallback also bypasses local readable retention: its native v2 turn receives the original reasoning and creates no archive (`tests/responses/reasoning-retention-routing.test.ts`). A replayed compaction item carries an `encrypted_content` blob only its minting backend can decode, and the client replays it on every later turn. The proxy's own `ocx1:`/`ocx2:` envelopes are transparent base64, so they always lower to plain user messages. A native blob is relayed only when there is no known serving-identity mismatch and the destination is known to decode native blobs — the canonical ChatGPT forward surface, the official OpenAI API, or a provider with the explicit diff --git a/tests/responses/compact-key-selection.test.ts b/tests/responses/compact-key-selection.test.ts index 9fb2b1808ec..277647b5c30 100644 --- a/tests/responses/compact-key-selection.test.ts +++ b/tests/responses/compact-key-selection.test.ts @@ -308,8 +308,12 @@ for (const reselect of [false, true]) test(`gateway compact preserves custom tra provider.supportsNativeCompactEndpoint = true; provider.headers = { "x-gateway-static": "initial" }; saveConfig(config); - const citation = { type: "url_citation", url: "https://gateway.example/source", start_index: "3", end_index: -1 }; - const input = [{ role: "user", content: "fixture question" }, { type: "message", role: "assistant", content: [{ type: "output_text", text: "fixture answer", annotations: [citation] }] }]; + const annotations = [ + { type: "url_citation", url: "https://gateway.example/source", start_index: "3", end_index: -1, gateway: { source: "fixture" } }, + { type: "url_citation", url: "https://gateway.example/untitled", title: null, start_index: 0, end_index: 1 }, + { type: "file_citation", file_id: "file_fixture", index: 0 }, + ]; + const input = [{ role: "user", content: "fixture question" }, { type: "message", role: "assistant", content: [{ type: "output_text", text: "fixture answer", annotations }] }]; const held = reselect ? await waitForProviderRequestSlot(config.defaultProvider, provider) : undefined; const abort = new AbortController(); const pending = start(config, abort, input); @@ -329,7 +333,45 @@ for (const reselect of [false, true]) test(`gateway compact preserves custom tra expect(seen[0]!.url).toBe(`https://gateway.example/v1/${reselect ? "refreshed-responses" : "custom-responses"}/compact`); expect(seen[0]!.headers.get("x-gateway-static")).toBe(reselect ? "replacement" : "initial"); expect(seen[0]!.headers.get("chatgpt-account-id")).toBeNull(); - expect(seen[0]!.body).toMatchObject({ model: "test", input }); + expect(seen[0]!.body).toMatchObject({ model: "test" }); + expect((seen[0]!.body as { input: unknown }).input).toEqual(input); noLeaks(config); } finally { abort.abort(); held?.release(); await pending.catch(() => undefined); } }); + +for (const toOpenAi of [false, true]) test(`compact citation projection follows the destination refreshed after admission (OpenAI=${toOpenAi})`, async () => { + const config = fixture(); + const provider = config.providers[NAME]!; + config.providers = { "gateway-key-fixture": provider }; + config.defaultProvider = "gateway-key-fixture"; + const officialBase = "https://api.openai.com/v1"; + const gatewayBase = "https://gateway.example/v1"; + provider.baseUrl = toOpenAi ? gatewayBase : officialBase; + provider.supportsNativeCompactEndpoint = true; + saveConfig(config); + const citation = { type: "url_citation", url: "https://gateway.example/source", start_index: "3", end_index: -1, gateway: { source: "fixture" } }; + const user = { role: "user", content: "fixture question" }; + const assistant = { type: "message", role: "assistant", content: [{ type: "output_text", text: "fixture answer", annotations: [citation] }] }; + const input = [user, assistant]; + const before = structuredClone(input); + const held = await waitForProviderRequestSlot(config.defaultProvider, provider); + const abort = new AbortController(); + const pending = start(config, abort, input); + try { + await queue(config, "k1"); + expect(seen).toEqual([]); + provider.baseUrl = toOpenAi ? officialBase : gatewayBase; + held.release(); + const result = await pending; + expect(result.response.status).toBe(200); + expect(seen).toHaveLength(1); + expect(seen[0]!.url).toBe(`${provider.baseUrl}/responses/compact`); + expect(seen[0]!.key).toBe(`Bearer ${KEYS[0]}`); + const expectedCitation = toOpenAi ? { ...citation, title: citation.url, start_index: 3, end_index: 0 } : citation; + expect((seen[0]!.body as { input: unknown }).input).toEqual([ + user, { ...assistant, content: [{ ...assistant.content[0]!, annotations: [expectedCitation] }] }, + ]); + expect(input).toEqual(before); + noLeaks(config); + } finally { abort.abort(); held.release(); await pending.catch(() => undefined); } +}); diff --git a/tests/responses/responses-native-item-extensions-compact.test.ts b/tests/responses/responses-native-item-extensions-compact.test.ts index 2d32a987635..55def6f784a 100644 --- a/tests/responses/responses-native-item-extensions-compact.test.ts +++ b/tests/responses/responses-native-item-extensions-compact.test.ts @@ -44,7 +44,7 @@ describe("native compact item extensions (#7111)", () => { ]; for (const [name, provider] of providers) { - test(`strips Google replay metadata on the direct ${name} compact send`, async () => { + test(`strips Google replay metadata and repairs citations on the direct ${name} compact send`, async () => { const calls: Array<{ url: string; body: Record }> = []; const mockFetch = (async (input, init) => { const req = new Request(input, init); @@ -58,12 +58,17 @@ describe("native compact item extensions (#7111)", () => { }; const call = { type: "function_call", call_id: "call_probe", name: "probe", arguments: '{"value":1}' }; const output = { type: "function_call_output", call_id: "call_probe", output: '{"extra_content":"tool data"}' }; + const citation = { type: "url_citation", url: "https://source.example/reference", start_index: "3", end_index: -1, extra: "keep" }; + const assistant = { type: "message", role: "assistant", content: [ + { type: "output_text", text: "Fixture answer.", annotations: [citation] }, + ] }; const body = { model: `${name}/gpt-6-sol`, input: [ { role: "user", content: "Compact the completed tool call." }, { ...call, extra_content: { google: { thought_signature: "synthetic-signature-7111" } } }, output, + assistant, ], }; const before = structuredClone(body); @@ -76,7 +81,11 @@ describe("native compact item extensions (#7111)", () => { await response.text(); expect(calls).toHaveLength(1); expect(calls[0]!.url).toBe(`${provider.baseUrl}/responses/compact`); - expect(calls[0]!.body.input).toEqual([body.input[0], call, output]); + expect(calls[0]!.body.input).toEqual([body.input[0], call, output, { + ...assistant, content: [{ ...assistant.content[0], annotations: [ + { ...citation, title: citation.url, start_index: 3, end_index: 0 }, + ] }], + }]); expect(body).toEqual(before); }); } diff --git a/tests/responses/responses-url-citation-title-repair.test.ts b/tests/responses/responses-url-citation-title-repair.test.ts index 9ec46566495..94fea6873ed 100644 --- a/tests/responses/responses-url-citation-title-repair.test.ts +++ b/tests/responses/responses-url-citation-title-repair.test.ts @@ -65,9 +65,23 @@ describe("url_citation annotation repair on outbound history", () => { }); test("metadata stripping and citation repair remain independent projections", () => { - const body = { input: [{ role: "user", content: "hi" }, assistantWith([{ type: "url_citation", url: "https://a.example", start_index: 0, end_index: 0 }])] }; - expect(annotationsOf(stripProviderInputItemMetadata(body))[0]).not.toHaveProperty("title"); - expect(annotationsOf(repairUrlCitationAnnotations(body))[0]!.title).toBe("https://a.example"); + const citation = { type: "url_citation", url: "https://a.example", start_index: "3", end_index: -1, extra: "keep" }; + const message = { ...assistantWith([citation]), extra_content: { google: { thought_signature: "fixture-signature" } } }; + const body = { input: [{ role: "user", content: "hi" }, message] }; + const before = structuredClone(body); + + const stripped = stripProviderInputItemMetadata(body) as { input: Record[] }; + expect(stripped.input[1]).not.toHaveProperty("extra_content"); + expect(stripped.input[1]!.content).toBe(message.content); + expect(annotationsOf(stripped)).toBe(message.content[0]!.annotations); + expect(annotationsOf(stripped)).toEqual([citation]); + + const repaired = repairUrlCitationAnnotations(body) as { input: Record[] }; + expect(repaired.input[1]!.extra_content).toBe(message.extra_content); + expect(annotationsOf(repaired)).toEqual([ + { ...citation, title: citation.url, start_index: 3, end_index: 0 }, + ]); + expect(body).toEqual(before); }); const native: OcxProviderConfig[] = [ From 9e5a917eddb8d1f1ab32dc0421e415ddb71ff3e9 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 23 Oct 2026 21:25:44 +0900 Subject: [PATCH 3/6] fix(test): promote Bun 1.4.4 with independent runner role verification --- .github/actions/setup-project-bun/action.yml | 4 +-- .github/workflows/ci.yml | 10 +++--- CONTRIBUTING.md | 6 ++-- docs-site/src/content/docs/contributing.md | 6 ++-- package.json | 2 +- structure/runtime.md | 4 +-- tests/ci-workflows/ci-scope-gaps.test.ts | 2 +- tests/ci-workflows/install-scripts.test.ts | 6 ++-- tests/test-layout-tooling.test.ts | 34 ++++++++++++++++---- 9 files changed, 48 insertions(+), 26 deletions(-) diff --git a/.github/actions/setup-project-bun/action.yml b/.github/actions/setup-project-bun/action.yml index 1ab22dd1dc9..cf28f6f9d43 100644 --- a/.github/actions/setup-project-bun/action.yml +++ b/.github/actions/setup-project-bun/action.yml @@ -8,8 +8,8 @@ description: >- # (#6713 run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9). It moved from 1.4.0 to # 1.4.3 for oven-sh/bun#44581: 1.4.0's Bun.spawnSync could lose a child's exit after # finalizers ran inside an earlier call and spin past its own timeout (#4956). -# Rejoin only after the upstream runner crash is fixed and a green lane=all -# dispatch proves the unified version; then delete testRunnerBun and this role input. +# Both roles now default to the package's 1.4.4 security transition, retaining role validation. +# Promotion requires a green full cross-platform lane=all run on the exact candidate head. inputs: role: description: Bun role to install (runtime or test-runner). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 028c28ca318..7bcd0eaa3f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -578,12 +578,10 @@ jobs: printf 'Scheduled run: workflow from %s, testing dev at %s\n' "$GITHUB_SHA" "$sha" >> "$GITHUB_STEP_SUMMARY" printf 'scheduled run tests dev at %s (workflow %s)\n' "$sha" "$GITHUB_SHA" - # Bun roles (#6713): runtime 1.4.2 fixes #6684's Windows Bun.fetch streaming; - # the test runner skipped 1.4.2's multi-file --isolate preload segfault (#4821 - # reverted #4064; run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9) and runs - # 1.4.3, whose oven-sh/bun#44581 stops Bun.spawnSync losing a child's exit (#4956). - # Rejoin after the upstream runner crash is fixed and a green lane=all dispatch - # proves the unified version, then delete testRunnerBun and the action's role input. + # Bun runtime and test-runner roles both read their 1.4.4 defaults from package.json. + # Keep role validation after the historical isolate preload and spawnSync failures + # (#6713, #4956). Promotion requires a green full cross-platform lane=all run on + # the exact candidate head; focused local results do not replace that proof. # The suite, split by file across four Linux runners. # # `scripts/ci/run-bun-test-batches.sh` assigns files to shards by the per-file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d4aa94f580..bbcd248f8f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,9 +69,11 @@ A ready-for-review PR is the author's claim that the change is complete, underst ## Local validation and hooks -The suite uses Bun 1.4.3 (`testRunnerBun`) while the shipped runtime is 1.4.4. -`scripts/test.ts` finds a 1.4.3 binary on PATH or in `~/.bun/bin`, or uses +The suite uses Bun 1.4.4 (`testRunnerBun`) while the shipped runtime is 1.4.4. +`scripts/test.ts` finds a 1.4.4 binary on PATH or in `~/.bun/bin`, or uses `OCX_TEST_RUNNER_BUN`; install that version locally if it is missing. +The roles remain independently checked even when their versions match. Promotion requires +a successful full cross-platform `lane=all` run on the candidate head. Run `bun run test` before review readiness. If the full local suite is too costly for the task or available resources, run at least focused regression tests for diff --git a/docs-site/src/content/docs/contributing.md b/docs-site/src/content/docs/contributing.md index 7fe0b09831b..d0021b7f12a 100644 --- a/docs-site/src/content/docs/contributing.md +++ b/docs-site/src/content/docs/contributing.md @@ -45,9 +45,11 @@ bun run privacy:scan # credential/privacy scan used by CI bun run prepare:package # refresh package launchers/assets ``` -The suite uses Bun 1.4.3 (`package.json`'s `testRunnerBun`) while the shipped runtime is 1.4.4. -`scripts/test.ts` finds a 1.4.3 binary on PATH or in `~/.bun/bin`, or uses `OCX_TEST_RUNNER_BUN`. +The suite uses Bun 1.4.4 (`package.json`'s `testRunnerBun`) while the shipped runtime is 1.4.4. +`scripts/test.ts` finds a 1.4.4 binary on PATH or in `~/.bun/bin`, or uses `OCX_TEST_RUNNER_BUN`. Install that version locally if it is missing; the resolver never downloads it. +The roles remain independently checked even when their versions match. Promotion requires +a successful full cross-platform `lane=all` run on the candidate head. Run `bun run test` by default. If a full run is disproportionately expensive for the task size, available machine resources, or concurrent worktrees, you must at least run focused regression tests diff --git a/package.json b/package.json index 89176da2851..a270427faf2 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "trustedDependencies": [ "bun" ], - "testRunnerBun": "1.4.3", + "testRunnerBun": "1.4.4", "scripts": { "dev": "bun run src/cli/index.ts start", "dev:proxy": "bun run src/cli/index.ts start", diff --git a/structure/runtime.md b/structure/runtime.md index 43e071d457f..763f3e90c3d 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -4,7 +4,7 @@ The minute sweep checks persisted activation deadlines locally; only missing dea ## Bun runtime and test runner -`package.json` pins the shipped npm runtime (`dependencies.bun`) to 1.4.4 and the test runner (`testRunnerBun`) to 1.4.3; `@types/bun` remains unchanged. Runtime 1.4.2 fixed #6684's Windows Bun.fetch truncation after the first cleartext upstream body chunk; 1.4.4 keeps that fix and ships NSS 3.131.1 roots, which drop the compromised eMudhra CAs. +`package.json` pins both the shipped npm runtime (`dependencies.bun`) and the test runner (`testRunnerBun`) to 1.4.4; the roles remain independently validated and `@types/bun` remains unchanged. Runtime 1.4.2 fixed #6684's Windows Bun.fetch truncation after the first cleartext upstream body chunk; 1.4.4 keeps that fix and ships NSS 3.131.1 roots, which drop the compromised eMudhra CAs. The runner pin skipped 1.4.2's multi-file `bun test --isolate` preload reload segfault (`load_preloads -> JSModuleLoader::loadModule -> JSPromise::status`, address 0x10): #4064 introduced 1.4.2 on 2026-09-09 and #4821 reverted it on 2026-09-17. It left 1.4.0 for 1.4.3 because of oven-sh/bun#44581 (#4956). Bun 1.4.0's `Bun.spawnSync` pointed the VM at its private wait loop, so a finalizer that ran during one call (an earlier file's stdout/stderr sink, a stream reader) released its poll on that loop; the cached loop's poll count drifted to zero and a later call stopped polling for its child, spinning past its own `timeout` until the 120-second batch deadline. In a multi-file `--isolate` process that depends on what earlier files left to finalize, so single-file attribution stayed clean. `tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts` reproduces the mechanism in a child under a bounded parent and fails on 1.4.0. #6713 run 37631558263 reproduces that 1.4.2 preload segfault on Linux 1/4, macOS 2/2 and Windows 9/9; single-file success does not prove the multi-file runner works. @@ -12,7 +12,7 @@ The runner pin skipped 1.4.2's multi-file `bun test --isolate` preload reload se Test jobs, including single-file development-version validation, select `test-runner`. They also build dashboard fixtures; `gates` uploads a dashboard preview and checks release-helper syntax on that pin. Release packaging, Docker, installed-artifact and service smoke jobs, desktop shell and widget preparation use `runtime`. Shipped binaries compile through `process.execPath` in `scripts/build-standalone.ts` and `desktop/scripts/prepare-sidecar.ts` on the runtime pin. -`scripts/lib/test-runner-bun.ts` selects the test pin for `scripts/test.ts`, `scripts/release.ts` preflight, `scripts/test-layout/verify.ts`, provider-option gates and GUI package tests via `scripts/test-with-pinned-bun.ts`: matching current Bun first, then a version-checked `OCX_TEST_RUNNER_BUN`, then PATH outside node_modules and `~/.bun/bin`. Missing or mismatched binaries fail with install guidance, without downloading; package-script PATH shadowing cannot select the shipped runtime for tests. Rejoin only after an upstream fix and green unified `lane=all` proof; then remove the runner pin, resolver and CI role split. +`scripts/lib/test-runner-bun.ts` selects the test pin for `scripts/test.ts`, `scripts/release.ts` preflight, `scripts/test-layout/verify.ts`, provider-option gates and GUI package tests via `scripts/test-with-pinned-bun.ts`: matching current Bun first, then a version-checked `OCX_TEST_RUNNER_BUN`, then PATH outside node_modules and `~/.bun/bin`. Missing or mismatched binaries fail with install guidance, without downloading. The caller need not already run the test pin; only a version match permits reusing its executable. `tests/test-layout-tooling.test.ts` verifies matching/different caller fixtures, the selected isolate command, and refusal before spawning tests for a wrong-version override. The 1.4.4 security transition aligns the default versions while retaining the role fields, resolver and CI role checks. Promotion requires successful full cross-platform `lane=all` proof on the exact candidate head; focused local tests do not establish that proof. ## Resolved static model policy diff --git a/tests/ci-workflows/ci-scope-gaps.test.ts b/tests/ci-workflows/ci-scope-gaps.test.ts index f780d71aa90..0729ab6945d 100644 --- a/tests/ci-workflows/ci-scope-gaps.test.ts +++ b/tests/ci-workflows/ci-scope-gaps.test.ts @@ -65,7 +65,7 @@ describe("an edit to the setup action alone", () => { }); }); -// #6713: user runtimes and compiled artifacts need 1.4.2; isolate runners need the testRunnerBun pin (1.4.3, #4956). +// Keep runtime and test-runner role selection distinct even when package.json pins both to 1.4.4. // Check the active setup at each test command, so a later role switch cannot evade the guard. const runsBunTests = (step: Step): boolean => /\bbun\s+test\b|\bbun\s+run\s+test(?::changed)?(?:\s|$)|\bbun\s+(?:run\s+)?scripts\/test\.ts\b|scripts\/ci\/run-bun-test-batches\.sh/.test( diff --git a/tests/ci-workflows/install-scripts.test.ts b/tests/ci-workflows/install-scripts.test.ts index 51e62b0b6fc..23ed7b7461e 100644 --- a/tests/ci-workflows/install-scripts.test.ts +++ b/tests/ci-workflows/install-scripts.test.ts @@ -94,10 +94,10 @@ describe("install scripts", () => { // and spun past its own timeout. Scratch run 38213764818 stalled 8 of 252 Linux shard 1/4 // batch-5 runs that way (CLI child a zombie, its pidfd armed but never polled). // tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts keeps the runner above it. - // Rejoin after the upstream runner crash is fixed and a green `lane=all` dispatch - // proves the unified version; then delete testRunnerBun and the setup action's role. + // The security transition now defaults both roles to 1.4.4. Keep their independent + // validation; promotion still requires a green full cross-platform lane=all candidate run. expect(pkg.dependencies?.bun).toBe("1.4.4"); - expect(pkg.testRunnerBun).toBe("1.4.3"); + expect(pkg.testRunnerBun).toBe("1.4.4"); expect(pkg.dependencies?.zod).toBe("4.4.3"); expect(pkg.devDependencies?.typescript).toBe("7.0.2"); expect(pkg.devDependencies?.["@types/bun"]).toBe("1.4.0"); diff --git a/tests/test-layout-tooling.test.ts b/tests/test-layout-tooling.test.ts index 354e3943fe0..cb4a7482819 100644 --- a/tests/test-layout-tooling.test.ts +++ b/tests/test-layout-tooling.test.ts @@ -7,6 +7,7 @@ import { repoPath, repoRoot } from "./helpers/repo-root"; import { listTestFiles, planMoves } from "../scripts/test-layout/plan"; import { runMove } from "../scripts/test-layout/move"; import { runVerify } from "../scripts/test-layout/verify"; +import * as runnerBun from "../scripts/lib/test-runner-bun"; import { testRunnerBun } from "../package.json"; import { anchors, @@ -469,24 +470,43 @@ describe("move end to end", () => { } }); - test("domain verification launches isolate tests through the pinned runner", () => { + const otherBunVersion = testRunnerBun === "1.4.3" ? "1.4.4" : "1.4.3"; + test.each([ + ["matching caller", testRunnerBun, testRunnerBun, false], + ["different caller", otherBunVersion, testRunnerBun, false], + ["wrong override version", otherBunVersion, otherBunVersion, true], + ] as const)("domain verification preserves the configured runner role: %s", (_name, currentVersion, overrideVersion, refused) => { const { root, cleanup } = scratchRepo(); const layoutPath = join(root, "scripts", "test-layout", "layout.json"); try { runMove({ root, domains: ["server", "providers"], dryRun: false, layoutPath, skipVerify: true, log: () => {} }); + const binaryName = process.platform === "win32" ? "bun.exe" : "bun"; + const pinnedRunner = join(root, "test-runner", binaryName); + const caller = currentVersion === testRunnerBun ? pinnedRunner : join(root, "runtime", binaryName); + const probes: string[] = []; + // Keep role resolution real, but version probes independent of the host and the spawn mock. + const select = spyOn(runnerBun, "getTestRunnerBun").mockImplementation(() => runnerBun.resolveTestRunnerBun({ + pin: testRunnerBun, currentVersion, execPath: caller, + env: { OCX_TEST_RUNNER_BUN: pinnedRunner }, pathEntries: [], homeDir: root, platform: process.platform, + probeVersion: binary => { probes.push(binary); return binary === pinnedRunner ? overrideVersion : undefined; }, + })); const spawn = spyOn(Bun, "spawnSync").mockReturnValue({ exitCode: 0, stdout: Buffer.from(""), stderr: Buffer.from(""), } as ReturnType); try { - const report = runVerify({ root, domains: ["server", "providers"], layoutPath, log: () => {} }); - expect(report.testExit).toBe(0); - // The test itself runs on the required pin, so its executable is the resolver's first choice. - expect(Bun.version).toBe(testRunnerBun); - expect(spawn.mock.calls.at(-1)?.[0]).toEqual([ - process.execPath, "test", "--isolate", "tests/server", "tests/providers", + const verify = () => runVerify({ root, domains: ["server", "providers"], layoutPath, log: () => {} }); + if (refused) expect(verify).toThrow(`OCX_TEST_RUNNER_BUN reports ${overrideVersion}`); + else expect(verify()).toMatchObject({ ok: true, testExit: 0 }); + expect(select).toHaveBeenCalledTimes(1); + expect(probes).toEqual(currentVersion === testRunnerBun ? [] : [pinnedRunner]); + const testSpawns = spawn.mock.calls.filter(([command]) => Array.isArray(command) && command[1] === "test"); + expect(testSpawns.map(([command]) => command)).toEqual(refused ? [] : [ + [pinnedRunner, "test", "--isolate", "tests/server", "tests/providers"], ]); + if (!refused) expect(testSpawns[0]?.[1]).toMatchObject({ cwd: root, stdout: "inherit", stderr: "inherit" }); } finally { spawn.mockRestore(); + select.mockRestore(); } } finally { cleanup(); From 1deb67d274c8246df9ad539bff93f958782032cb Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 23 Oct 2026 21:38:09 +0900 Subject: [PATCH 4/6] docs: align Docker Bun comments with unified role defaults --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 82fe86f4cf1..54e0153e0f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,12 @@ # Keep the runtime aligned with package.json and pin the multi-platform image index. # Runtime 1.4.4 keeps 1.4.2's #6684 Windows Bun.fetch streaming fix and ships NSS -# 3.131.1 roots without the compromised eMudhra CAs; CI tests run on 1.4.3 -# because 1.4.2 segfaults reloading preloads in multi-file --isolate (#4821; -# #6713 run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9) and 1.4.0's spawnSync -# can lose a child's exit (#4956, oven-sh/bun#44581). Rejoin after the -# upstream crash fix passes a unified-version lane=all dispatch, then delete -# package.json's testRunnerBun pin and setup-project-bun's role input. +# 3.131.1 roots without the compromised eMudhra CAs. Runtime and test-runner +# defaults are both 1.4.4; keep package.json's testRunnerBun pin and +# setup-project-bun's role input independently validated. Promotion requires +# an exact-head cross-platform lane=all dispatch, including multi-file --isolate: +# historical 1.4.2 preload crashes (#4821; #6713 run 37631558263) and 1.4.0 +# spawnSync exit loss (#4956, oven-sh/bun#44581) remain regression risks. ARG BUN_IMAGE=oven/bun:1.4.4@sha256:1d22548f177f4ff38e0c15baa3c2c46c6c2060ba2d1bc573bb4720b407548f1d FROM ${BUN_IMAGE} AS manifest From 41f08fffd3e7dea64a6eed79af918aed74a153dd Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 23 Oct 2026 22:07:35 +0900 Subject: [PATCH 5/6] test: use a draining TLS origin for early upload replies --- .../claude-picker-upload.test.ts | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/tests/claude-integration/claude-picker-upload.test.ts b/tests/claude-integration/claude-picker-upload.test.ts index 8d4cd0d1859..95c6351f04d 100644 --- a/tests/claude-integration/claude-picker-upload.test.ts +++ b/tests/claude-integration/claude-picker-upload.test.ts @@ -3,14 +3,39 @@ import { Agent, createServer, request } from "node:https"; import { connect, constants, type ClientHttp2Session, type ClientHttp2Stream } from "node:http2"; import type { ClientRequest, IncomingMessage, ServerResponse } from "node:http"; import type { Duplex } from "node:stream"; +import { createServer as createTlsServer } from "node:tls"; import { createLocalInterceptCa, issueLocalInterceptLeaf } from "../../src/claude/intercept/local-ca"; import { startPickerListener, type PickerListenerOptions } from "../../src/claude/intercept/picker-listener"; -async function fixture(handler: (req: IncomingMessage, res: ServerResponse) => void, +async function fixture(handler: ((req: IncomingMessage, res: ServerResponse) => void) | "early-fin", options: Partial = {}) { const ca = createLocalInterceptCa(); const leaf = issueLocalInterceptLeaf(ca, ["claude.ai"]); - const upstream = createServer({ cert: leaf.certPem, key: leaf.keyPem }, handler); + const earlyReplies: Array<{ requestComplete: boolean; flushed: boolean }> = []; + // A wire-level origin sends the reply and FIN while continuing to drain input. Bun's HTTP + // server also tears down unfinished requests; overriding destroySoon alone does not prevent + // that reset racing ahead of the reply on Windows with a buffered upload. + const upstream = handler === "early-fin" ? createTlsServer({ cert: leaf.certPem, key: leaf.keyPem, allowHalfOpen: true }, socket => { + socket.on("error", () => socket.destroy()); + let headers = ""; + let replied = false; + socket.on("data", chunk => { + if (replied) return; + headers += chunk.toString("latin1"); + const end = headers.indexOf("\r\n\r\n"); + if (end < 0) return; + replied = true; + const early = headers.startsWith("POST /early "); + const body = early ? "early" : "ordinary"; + const length = /\r\ncontent-length: (\d+)\r\n/i.exec(headers.slice(0, end + 2)); + const requestBody = headers.slice(end + 4); + const observation = { requestComplete: length ? requestBody.length >= Number(length[1]) + : requestBody.endsWith("0\r\n\r\n"), flushed: false }; + if (early) earlyReplies.push(observation); + socket.once("finish", () => { observation.flushed = true; }); + socket.end(`HTTP/1.1 ${early ? "413 Payload Too Large" : "200 OK"}\r\nContent-Length: ${body.length}\r\nConnection: close\r\n\r\n${body}`); + }); + }) : createServer({ cert: leaf.certPem, key: leaf.keyPem }, handler); const sockets = new Set(); for (const event of ["connection", "secureConnection"] as const) upstream.on(event, socket => { sockets.add(socket); socket.once("close", () => sockets.delete(socket)); @@ -34,7 +59,7 @@ async function fixture(handler: (req: IncomingMessage, res: ServerResponse) => v rejectUnauthorized: true, agent: h1Agent, path, method, headers: { Host: "claude.ai", ...headers } }); clients.push(req); return req; }; - return { logs, h2, h1, relay, + return { logs, h2, h1, relay, earlyReplies, async close() { const closed = clients.map(client => client.destroyed ? Promise.resolve() : observeClose(client)); await relay.close(); @@ -353,16 +378,7 @@ for (const protocol of ["h1", "h2"] as const) { // "buffered" answers early while 4 MiB of an advertised 32 MiB body is still queued. for (const input of protocol === "h1" ? ["small", "buffered"] as const : ["small"] as const) bounded(`${protocol} early upstream response closes ${input === "buffered" ? "a buffered" : "an"} unfinished upload without losing the reply`, async wait => { - const f = await fixture((req, res) => { - // Answer like a real origin that replies early: keep reading the body and close with a FIN. - // Bun's own server would close this unfinished request with destroySoon(), a reset that - // Windows delivers before the reply bytes, so the relay could never read what it should relay. - req.on("error", () => res.destroy()); req.resume(); - const socket = req.socket; - socket.destroySoon = () => { socket.end(); return socket; }; - if (req.url === "/early") { res.writeHead(413, { "Content-Length": "5" }); res.end("early"); } - else res.end("ordinary"); - }); + const f = await fixture("early-fin"); try { const session = f.h2(); const upload = protocol === "h2" @@ -373,6 +389,7 @@ for (const protocol of ["h1", "h2"] as const) { upload.write(input === "buffered" ? Buffer.alloc(4 * 1024 * 1024, 0x61) : "x"); expect(await wait(result)).toEqual({ status: 413, text: "early" }); await wait(closed); + expect(f.earlyReplies).toEqual([{ requestComplete: false, flushed: true }]); if (protocol === "h2") expect((upload as ClientHttp2Stream).rstCode).toBe(constants.NGHTTP2_NO_ERROR); expect(await wait(normalH2(session))).toEqual({ status: 200, text: "ordinary" }); } finally { await wait(f.close()); } From a9c3d127d3728f50533df32504b787823b5220ee Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 23 Oct 2026 22:09:36 +0900 Subject: [PATCH 6/6] test: confine draining origin to buffered HTTP/1 upload --- .../claude-integration/claude-picker-upload.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/claude-integration/claude-picker-upload.test.ts b/tests/claude-integration/claude-picker-upload.test.ts index 95c6351f04d..d0bb998b873 100644 --- a/tests/claude-integration/claude-picker-upload.test.ts +++ b/tests/claude-integration/claude-picker-upload.test.ts @@ -378,7 +378,16 @@ for (const protocol of ["h1", "h2"] as const) { // "buffered" answers early while 4 MiB of an advertised 32 MiB body is still queued. for (const input of protocol === "h1" ? ["small", "buffered"] as const : ["small"] as const) bounded(`${protocol} early upstream response closes ${input === "buffered" ? "a buffered" : "an"} unfinished upload without losing the reply`, async wait => { - const f = await fixture("early-fin"); + const f = await fixture(input === "buffered" ? "early-fin" : (req, res) => { + // Answer like a real origin that replies early: keep reading the body and close with a FIN. + // Bun's own server would close this unfinished request with destroySoon(), a reset that + // Windows delivers before the reply bytes, so the relay could never read what it should relay. + req.on("error", () => res.destroy()); req.resume(); + const socket = req.socket; + socket.destroySoon = () => { socket.end(); return socket; }; + if (req.url === "/early") { res.writeHead(413, { "Content-Length": "5" }); res.end("early"); } + else res.end("ordinary"); + }); try { const session = f.h2(); const upload = protocol === "h2" @@ -389,7 +398,7 @@ for (const protocol of ["h1", "h2"] as const) { upload.write(input === "buffered" ? Buffer.alloc(4 * 1024 * 1024, 0x61) : "x"); expect(await wait(result)).toEqual({ status: 413, text: "early" }); await wait(closed); - expect(f.earlyReplies).toEqual([{ requestComplete: false, flushed: true }]); + if (input === "buffered") expect(f.earlyReplies).toEqual([{ requestComplete: false, flushed: true }]); if (protocol === "h2") expect((upload as ClientHttp2Stream).rstCode).toBe(constants.NGHTTP2_NO_ERROR); expect(await wait(normalH2(session))).toEqual({ status: 200, text: "ordinary" }); } finally { await wait(f.close()); }