From 838a54b219b016ee5bc2847a585bb6b602327ad4 Mon Sep 17 00:00:00 2001 From: Valerio Coltre Date: Fri, 6 Nov 2026 19:35:56 +0700 Subject: [PATCH 1/3] fix(dictation): allow ten-minute recordings with bounded finalization --- .../content/docs/reference/proxy-formats.md | 9 ++- src/server/audio-dictation.ts | 5 +- src/server/index/live-sideband.ts | 18 ++++- src/server/index/serve-options.ts | 1 + src/server/index/websocket-handler.ts | 2 + src/server/ws-bridge.ts | 2 + .../providers/codex-installation-identity.md | 5 ++ tests/server/audio-dictation.test.ts | 70 ++++++++++++++++++- 8 files changed, 107 insertions(+), 5 deletions(-) diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index b17aaec2094..890abd20065 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -256,7 +256,14 @@ Use the actual sample rate of mono PCM16 audio. Wait for `session.started`, then send `{"type":"audio.append","audio":""}`. These are JSON text frames, not WAV files or binary WebSocket frames. The gateway accepts sample rates from 8,000 through 192,000 Hz; upstream support is account/service-dependent. -Client frames are limited to 64 KiB and sessions to five minutes. Unsupported or +Client frames are limited to 64 KiB. The proxy allows up to ten minutes +(600,000 ms) for recording from relay attachment. A valid `session.close` before +that deadline replaces the recording timer with a separate, bounded 60-second +window for final transcripts. Keep the WebSocket open during finalization. +`config.session_ttl_ms` accepts 1–600,000 and is forwarded unchanged; it does not +set the proxy's recording timer (existing clients may still send 300,000). +These are OpenCodex application budgets, not demonstrated official OpenAI limits. +GPT Live and other audio transports retain their existing lifetimes. Unsupported or malformed event/config fields close the stream with code 1008. `transcript.segment` and `transcript.final` contain `utterance_id`, `revision`, and diff --git a/src/server/audio-dictation.ts b/src/server/audio-dictation.ts index 27734289fba..200c6facf8d 100644 --- a/src/server/audio-dictation.ts +++ b/src/server/audio-dictation.ts @@ -7,7 +7,8 @@ import { resolveAudioUpstream, TRANSCRIPTION_MODEL, type AudioUpstream } from ". import type { RequestLogContext } from "./request-log"; import { authorizationCredential } from "../lib/authorization-scheme"; -export const DICTATION_SESSION_MAX_MS = 300_000; +export const DICTATION_SESSION_MAX_MS = 600_000; +export const DICTATION_FINALIZATION_MAX_MS = 60_000; const DICTATION_FRAME_MAX_BYTES = 64 * 1024; type JsonObject = Record; function object(value: unknown): value is JsonObject { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -54,6 +55,7 @@ export interface AudioSocketTarget { protocols?: string[]; validateFrame?: (frame: string | Buffer) => boolean; maxSessionMs: number; + finalizationMs?: number; finish: (outcome?: number | "timeout" | "connect_error") => void; } @@ -90,6 +92,7 @@ export async function resolveDictationSocket( protocols: ["chatgpt-dictation", `openai-bearer.${token}`, "codex-desktop"], validateFrame: createDictationFrameValidator(), maxSessionMs: DICTATION_SESSION_MAX_MS, + finalizationMs: DICTATION_FINALIZATION_MAX_MS, finish: finishAudioUpstream(relay), }; } diff --git a/src/server/index/live-sideband.ts b/src/server/index/live-sideband.ts index bf3c0982cee..6ea61f208d9 100644 --- a/src/server/index/live-sideband.ts +++ b/src/server/index/live-sideband.ts @@ -306,6 +306,19 @@ export function closeLiveSideband(ws: ServerWebSocket, code = 1000, reas } } +/** Only dictation opts in: a validated close gets a separate bounded drain window. */ +export function beginAudioFinalization(ws: ServerWebSocket, raw: string | Buffer): void { + if (ws.data.liveFinalizationMs === undefined || ws.data.liveFinalizing || ws.data.liveClosing) return; + // The frame validator has already accepted this JSON text frame. + if (typeof raw !== "string" || JSON.parse(raw).type !== "session.close") return; + ws.data.liveFinalizing = true; + if (ws.data.liveSessionTimer !== undefined) clearTimeout(ws.data.liveSessionTimer); + ws.data.liveSessionTimer = setTimeout(() => { + ws.data.liveOutcome = "timeout"; + closeLiveSideband(ws, 1000, "audio finalization expired"); + }, ws.data.liveFinalizationMs); +} + /** * Dial the upstream sideband and report whether its handshake reached 101. * @@ -505,7 +518,10 @@ export function attachLiveSidebandUpstream( ws.data.liveOutcome = "timeout"; closeLiveSideband(ws, 1011, "audio connection timed out"); }, 10_000); - ws.data.liveSessionTimer = setTimeout(() => closeLiveSideband(ws, 1000, "audio session expired"), ws.data.liveMaxSessionMs); + ws.data.liveSessionTimer = setTimeout(() => { + if (ws.data.liveFinalizationMs !== undefined) ws.data.liveOutcome = "timeout"; + closeLiveSideband(ws, 1000, "audio session expired"); + }, ws.data.liveMaxSessionMs); } upstream.addEventListener("close", (event) => { diff --git a/src/server/index/serve-options.ts b/src/server/index/serve-options.ts index 391eba4f8cb..9db7ac46e5b 100644 --- a/src/server/index/serve-options.ts +++ b/src/server/index/serve-options.ts @@ -1864,6 +1864,7 @@ export function createServeOptions(ctx: ServeOptionsContext) { liveUpstreamProtocols: audio?.protocols, liveValidateFrame: audio?.validateFrame, liveMaxSessionMs: audio?.maxSessionMs, + liveFinalizationMs: audio?.finalizationMs, liveFinish: finish, liveAbortSignal: audioController?.signal, livePending: [], diff --git a/src/server/index/websocket-handler.ts b/src/server/index/websocket-handler.ts index 37781d2014a..37ff862f76e 100644 --- a/src/server/index/websocket-handler.ts +++ b/src/server/index/websocket-handler.ts @@ -10,6 +10,7 @@ import { MAX_WS_FRAME_BYTES, WEBSOCKET_IDLE_TIMEOUT_SECONDS, attachLiveSidebandUpstream, + beginAudioFinalization, clientCloseForUpstream, closeLiveSideband, closeLiveSidebandBeforeUpgrade, @@ -156,6 +157,7 @@ export function createWebsocketHandler( closeLiveSideband(ws, 1008, "invalid audio event"); return; } + beginAudioFinalization(ws, raw); const rawBytes = webSocketFrameBytes(raw); if (exceedsLiveSidebandFrameByteLimit(rawBytes)) { closeLiveSideband(ws, 1009, "message too large"); diff --git a/src/server/ws-bridge.ts b/src/server/ws-bridge.ts index 22dd877b36e..c7bf5d9a5d4 100644 --- a/src/server/ws-bridge.ts +++ b/src/server/ws-bridge.ts @@ -53,6 +53,8 @@ export interface WsData { liveFinish?: AudioSocketTarget["finish"]; liveOutcome?: number | "timeout" | "connect_error"; liveMaxSessionMs?: number; + liveFinalizationMs?: number; + liveFinalizing?: boolean; liveConnectTimer?: ReturnType; liveSessionTimer?: ReturnType; liveAbortSignal?: AbortSignal; diff --git a/structure/providers/codex-installation-identity.md b/structure/providers/codex-installation-identity.md index 03a91abf9ab..4bd7fab4d7b 100644 --- a/structure/providers/codex-installation-identity.md +++ b/structure/providers/codex-installation-identity.md @@ -60,6 +60,11 @@ JSON send, including live's converted multipart session. Live inspects JSON cont of Content-Type; non-JSON SDP retains its original bytes. Their selected headers authoritatively replace or remove mixed-case provider identity defaults; main and main-pool preserve caller values. Vision, audio and sidecar credential selection share materialization. +Dictation in `src/server/audio-dictation.ts` has a 600,000 ms recording budget and +opts into a separate 60,000 ms finalization window after a validated `session.close`. +`src/server/index/live-sideband.ts` replaces the recording timer on that event; +expiry and cancellation retain the existing authenticated transport cleanup and lease ownership. +The forwarded session TTL does not shorten the recording budget; other audio transports do not opt in. `src/server/audio-live.ts` projects the authenticated call's cloned session before JSON or multipart serialization and returns a sanitized 503 without sending on identity failure. Its socket resolver and `src/server/audio-dictation.ts` carry the selected auth context through diff --git a/tests/server/audio-dictation.test.ts b/tests/server/audio-dictation.test.ts index 24d50d2cd6f..a69f23499cc 100644 --- a/tests/server/audio-dictation.test.ts +++ b/tests/server/audio-dictation.test.ts @@ -8,7 +8,9 @@ import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex import * as routing from "../../src/codex/routing"; import { saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; -import { createDictationFrameValidator } from "../../src/server/audio-dictation"; +import { createDictationFrameValidator, DICTATION_SESSION_MAX_MS, DICTATION_FINALIZATION_MAX_MS } from "../../src/server/audio-dictation"; +import { attachLiveSidebandUpstream, beginAudioFinalization } from "../../src/server/index/live-sideband"; +import type { WsData } from "../../src/server/ws-bridge"; import { abortAndReleaseAllTurns, resetLifecycleDrainStateForTests } from "../../src/server/lifecycle"; import { LiveCallBindings } from "../../src/server/live-call-bindings"; import type { OcxConfig } from "../../src/types"; @@ -171,10 +173,74 @@ describe("dictation protocol validation", () => { expect(validate(JSON.stringify({ type: "audio.append", audio: "AAA=" }))).toBe(false); }); test("bad formats and oversized configurations do not start sessions", () => { - for (const change of [{ num_channels: 2 }, { sample_rate_hz: 0 }, { session_ttl_ms: 300001 }, { max_buffer_size_bytes: 4194305 }]) { + for (const change of [{ num_channels: 2 }, { sample_rate_hz: 0 }, { session_ttl_ms: 600001 }, { max_buffer_size_bytes: 4194305 }]) { expect(createDictationFrameValidator()(JSON.stringify({ ...startEvent, config: { ...startEvent.config, ...change } }))).toBe(false); } }); + test("ten-minute application ceiling accepts the old TTL and rejects larger TTLs", () => { + expect(DICTATION_SESSION_MAX_MS).toBe(600_000); + for (const ttl of [300_000, 600_000]) { + expect(createDictationFrameValidator()(JSON.stringify({ ...startEvent, config: { ...startEvent.config, session_ttl_ms: ttl } }))).toBe(true); + } + }); + test("relay recording and finalization clocks are bounded and cleaned on expiry or cancel", () => { + for (const mode of ["recording", "finalizing", "cancel", "complete", "other-transport"] as const) { + let now = 0; + let id = 0; + const timers = new Map void }>(); + const set = spyOn(globalThis, "setTimeout").mockImplementation(((run: () => void, ms: number) => { + timers.set(++id, { at: now + ms, run }); return id; + }) as typeof setTimeout); + const clear = spyOn(globalThis, "clearTimeout").mockImplementation(((key: number) => { timers.delete(key); }) as typeof clearTimeout); + const advance = (ms: number) => { + now += ms; + for (const [key, timer] of [...timers]) if (timer.at <= now) { timers.delete(key); timer.run(); } + }; + const upstream = new EventTarget() as EventTarget & { readyState: number; close: () => void }; + upstream.readyState = WebSocket.OPEN; + upstream.close = () => { upstream.readyState = WebSocket.CLOSED; upstream.dispatchEvent(new CloseEvent("close", { code: 1000 })); }; + let releases = 0; + const outcomes: unknown[] = []; + const data: WsData = { + liveUpstream: upstream as unknown as WebSocket, + liveUpstreamHandoff: { failure: () => undefined, take: () => ({ ok: true, frames: [] }) }, + liveMaxSessionMs: DICTATION_SESSION_MAX_MS, liveFinalizationMs: DICTATION_FINALIZATION_MAX_MS, + liveFinish: outcome => { outcomes.push(outcome); }, + liveTurnAdmissionLease: { release: () => { releases++; } } as never, + }; + const ws = { data, readyState: WebSocket.OPEN, close() {} }; + if (mode === "other-transport") data.liveFinalizationMs = undefined; + try { + attachLiveSidebandUpstream(ws as never); + expect(data.liveConnectTimer).toBeUndefined(); + advance(599_999); + expect(releases).toBe(0); + if (mode === "recording") advance(1); + else { + beginAudioFinalization(ws as never, '{"type":"session.close"}'); + if (mode === "other-transport") { + expect(data.liveFinalizing).toBeUndefined(); + advance(1); + } else { + advance(1); + expect(releases).toBe(0); + if (mode === "cancel") data.cancel!(); + else if (mode === "complete") upstream.close(); + else { + advance(59_998); + expect(releases).toBe(0); + advance(1); + } + } + } + expect(releases).toBe(1); + expect(outcomes).toEqual([["cancel", "complete", "other-transport"].includes(mode) ? undefined : "timeout"]); + expect(timers.size).toBe(0); + expect(data.liveSessionTimer).toBeUndefined(); + expect(data.liveAbortListener).toBeUndefined(); + } finally { set.mockRestore(); clear.mockRestore(); } + } + }); }); describe("external audio sockets", () => { From c1c01f57015d38153c7120e9fb7547c8d2cd435d Mon Sep 17 00:00:00 2001 From: Valerio Coltre Date: Fri, 6 Nov 2026 19:43:34 +0700 Subject: [PATCH 2/3] fix(dictation): accept ScribeTray streaming frames without weakening validation --- .../content/docs/reference/proxy-formats.md | 24 ++++++ src/server/audio-dictation.ts | 23 ++++-- structure/data-planes/inbound-compat.md | 4 +- .../providers/codex-installation-identity.md | 3 + tests/server/audio-dictation.test.ts | 82 ++++++++++++++++++- 5 files changed, 126 insertions(+), 10 deletions(-) diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index 890abd20065..fa9489b1c21 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -271,6 +271,30 @@ malformed event/config fields close the stream with code 1008. do not concatenate revisions. Finish with `{"type":"session.close"}` and wait for final text and `session.updated` with `session.status="closed"`. +The ScribeTray streaming client uses this additional supported start shape +(replace the example UUIDs with client-generated identifiers): + +```json +{"type":"session.start","dictation_session_id":"78d201cb-1b49-40e1-a5b5-e06a8f8489a1","attempt_id":"8b3c2e4f-d501-4a66-b123-456789abcdef","config":{"input_audio_format":"pcm16","sample_rate_hz":16000,"num_channels":1,"max_buffer_size_bytes":4194304,"max_utterance_duration_ms":30000,"session_ttl_ms":300000,"provider_mode":"streaming_sse","transcript_delivery_mode":"final_only","session_asset_mode":"none","vad":{"type":"server_vad"}},"client_capabilities":["server_stop_requested"]} +``` + +The identifiers are optional, nonblank strings of at most 128 UTF-8 bytes. +`client_capabilities` is optional and accepts an empty array or just +`"server_stop_requested"`. Optional `config.language` is a nonblank string of at +most 64 UTF-8 bytes (for example `"it"`); these strings cannot contain control +characters. Optional `config.session_asset_mode` accepts only `"none"`. +VAD needs only `{"type":"server_vad"}`. If supplied, `threshold` must be finite +and within 0–1, `prefix_padding_ms` an integer within 0–1,000, and +`silence_duration_ms` an integer within 0–5,000. + +After start, `{"type":"audio.flush"}` is also accepted and forwarded unchanged. +It does not close the session; more audio may follow until `session.close`. +ScribeTray finishes by sending `audio.flush`, then `session.close`, then waiting +for final results. This supports that client's sequence without asserting that +flush is required instead of close alone. No audio or other client events are +accepted after `session.close`. The earlier start and close-only payloads remain +supported. Unknown fields and unsupported capability values remain rejected. + ## `POST /v1/responses` This is the native opencodex data-plane shape. The request body must be a JSON object with a diff --git a/src/server/audio-dictation.ts b/src/server/audio-dictation.ts index 200c6facf8d..a3b5e9fb108 100644 --- a/src/server/audio-dictation.ts +++ b/src/server/audio-dictation.ts @@ -15,6 +15,9 @@ function object(value: unknown): value is JsonObject { return !!value && typeof function integer(value: unknown, min: number, max: number): boolean { return typeof value === "number" && Number.isInteger(value) && value >= min && value <= max; } +function boundedString(value: unknown, max: number): boolean { + return typeof value === "string" && value.trim().length > 0 && Buffer.byteLength(value) <= max && !/[\u0000-\u001f\u007f]/.test(value); +} export function createDictationFrameValidator(): (frame: string | Buffer) => boolean { let started = false; @@ -26,21 +29,29 @@ export function createDictationFrameValidator(): (frame: string | Buffer) => boo if (!object(event)) return false; if (event.type === "session.start" && !started) { const config = event.config; - if (!object(config) || Object.keys(event).some(key => key !== "type" && key !== "config")) return false; - const fields = new Set(["input_audio_format", "sample_rate_hz", "num_channels", "max_buffer_size_bytes", "max_utterance_duration_ms", "session_ttl_ms", "provider_mode", "transcript_delivery_mode", "vad"]); + if (!object(config) || Object.keys(event).some(key => !["type", "config", "dictation_session_id", "attempt_id", "client_capabilities"].includes(key))) return false; + for (const key of ["dictation_session_id", "attempt_id"]) if (key in event && !boundedString(event[key], 128)) return false; + if ("client_capabilities" in event && (!Array.isArray(event.client_capabilities) || event.client_capabilities.length > 1 + || event.client_capabilities.some(value => value !== "server_stop_requested"))) return false; + const fields = new Set(["input_audio_format", "sample_rate_hz", "num_channels", "max_buffer_size_bytes", "max_utterance_duration_ms", "session_ttl_ms", "provider_mode", "transcript_delivery_mode", "session_asset_mode", "language", "vad"]); + if ("session_asset_mode" in config && config.session_asset_mode !== "none") return false; + if ("language" in config && !boundedString(config.language, 64)) return false; if (Object.keys(config).some(key => !fields.has(key)) || config.input_audio_format !== "pcm16" || config.num_channels !== 1 || !integer(config.sample_rate_hz, 8000, 192000) || !integer(config.max_buffer_size_bytes, 1, 4 * 1024 * 1024) || !integer(config.max_utterance_duration_ms, 1, 30000) || !integer(config.session_ttl_ms, 1, DICTATION_SESSION_MAX_MS) - || !["buffered", "streaming_sse"].includes(String(config.provider_mode)) - || !["final_only", "segment", "delta"].includes(String(config.transcript_delivery_mode))) return false; + || typeof config.provider_mode !== "string" || !["buffered", "streaming_sse"].includes(config.provider_mode) + || typeof config.transcript_delivery_mode !== "string" || !["final_only", "segment", "delta"].includes(config.transcript_delivery_mode)) return false; const vad = config.vad; if (!object(vad) || Object.keys(vad).some(key => !["type", "threshold", "prefix_padding_ms", "silence_duration_ms"].includes(key)) - || vad.type !== "server_vad" || typeof vad.threshold !== "number" || !Number.isFinite(vad.threshold) || vad.threshold < 0 || vad.threshold > 1 - || !integer(vad.prefix_padding_ms, 0, 1000) || !integer(vad.silence_duration_ms, 0, 5000)) return false; + || vad.type !== "server_vad" + || ("threshold" in vad && (typeof vad.threshold !== "number" || !Number.isFinite(vad.threshold) || vad.threshold < 0 || vad.threshold > 1)) + || ("prefix_padding_ms" in vad && !integer(vad.prefix_padding_ms, 0, 1000)) + || ("silence_duration_ms" in vad && !integer(vad.silence_duration_ms, 0, 5000))) return false; started = true; return true; } if (!started) return false; + if (event.type === "audio.flush" && Object.keys(event).length === 1) return true; if (event.type === "session.close" && Object.keys(event).length === 1) { closed = true; return true; } if (event.type !== "audio.append" || Object.keys(event).some(key => key !== "type" && key !== "audio") || typeof event.audio !== "string" || !event.audio) return false; const bytes = Buffer.from(event.audio, "base64"); diff --git a/structure/data-planes/inbound-compat.md b/structure/data-planes/inbound-compat.md index 404d933b7e7..7bacec47299 100644 --- a/structure/data-planes/inbound-compat.md +++ b/structure/data-planes/inbound-compat.md @@ -63,8 +63,8 @@ unchanged, on both join credentials described under streaming audio. Coverage li `src/server/audio-client.ts` recognizes explicit audio keys before local legacy admission. Browser sockets offer opencodex-audio and opencodex-key.; only the public marker is selected downstream. Invalid presented keys cannot become credential-free native calls. -`src/server/audio-dictation.ts` maps the validated desktop session.start/audio.append/session.close -protocol to ChatGPT dictation with server-owned credentials and a five-minute lifetime. +`src/server/audio-dictation.ts` validates session.start/audio.append/audio.flush/session.close for ChatGPT dictation with server-owned credentials, a ten-minute recording budget, and a separate bounded 60-second finalization window after close. +The forwarded TTL does not set the recording budget. ScribeTray's bounded optional metadata and minimal server VAD preserve PCM checks and refusal of audio after close. The `src/server/index.ts` byte relay retains bounded queues, handshake/session deadlines and the account/turn lifecycle until its upstream closes. `src/server/ws-bridge.ts` carries the in-memory callbacks and signal; handshake credentials are cleared after socket construction. diff --git a/structure/providers/codex-installation-identity.md b/structure/providers/codex-installation-identity.md index 4bd7fab4d7b..33e264e6cf3 100644 --- a/structure/providers/codex-installation-identity.md +++ b/structure/providers/codex-installation-identity.md @@ -65,6 +65,9 @@ opts into a separate 60,000 ms finalization window after a validated `session.cl `src/server/index/live-sideband.ts` replaces the recording timer on that event; expiry and cancellation retain the existing authenticated transport cleanup and lease ownership. The forwarded session TTL does not shorten the recording budget; other audio transports do not opt in. +Dictation validation also accepts ScribeTray's bounded identifiers, stop-request capability, +language, asset mode `none`, minimal server VAD, and nonterminal `audio.flush`. +Optional VAD values retain their numeric bounds; start/close ordering and PCM checks remain enforced. `src/server/audio-live.ts` projects the authenticated call's cloned session before JSON or multipart serialization and returns a sanitized 503 without sending on identity failure. Its socket resolver and `src/server/audio-dictation.ts` carry the selected auth context through diff --git a/tests/server/audio-dictation.test.ts b/tests/server/audio-dictation.test.ts index a69f23499cc..084fedf25e4 100644 --- a/tests/server/audio-dictation.test.ts +++ b/tests/server/audio-dictation.test.ts @@ -34,8 +34,18 @@ const startEvent = { type: "session.start", config: { provider_mode: "streaming_sse", transcript_delivery_mode: "segment", vad: { type: "server_vad", threshold: 0.5, prefix_padding_ms: 300, silence_duration_ms: 500 }, } }; +// Matches ScribeTray src/openai_streaming.rs::start_message (UUIDs fixed for repeatability). +const scribeStart = { + type: "session.start", dictation_session_id: "78d201cb-1b49-40e1-a5b5-e06a8f8489a1", + attempt_id: "8b3c2e4f-d501-4a66-b123-456789abcdef", config: { + input_audio_format: "pcm16", sample_rate_hz: 16000, num_channels: 1, + max_buffer_size_bytes: 4194304, max_utterance_duration_ms: 30000, session_ttl_ms: 300000, + provider_mode: "streaming_sse", transcript_delivery_mode: "final_only", session_asset_mode: "none", + vad: { type: "server_vad" }, + }, client_capabilities: ["server_stop_requested"], +}; -function createFixture(options: { failDictation?: boolean; answer?: "invalid" | "ok200" } = {}) { +function createFixture(options: { failDictation?: boolean; finalOnClose?: boolean; answer?: "invalid" | "ok200" } = {}) { const creates: Headers[] = []; const handshakes: Array<{ url: string; headers: Headers; protocols?: string[] }> = []; const frames: string[] = []; @@ -61,8 +71,9 @@ function createFixture(options: { failDictation?: boolean; answer?: "invalid" | return; } if (event.type === "session.start") ws.send(JSON.stringify({ type: "session.started", sequence_no: 1, session: { session_id: "fixture", status: "active", config: { provider_mode: "streaming_sse", transcript_delivery_mode: "segment" } } })); - else if (event.type === "audio.append") ws.send(JSON.stringify({ type: "transcript.final", sequence_no: 2, utterance_id: "u1", revision: 1, text: "fixture transcript" })); + else if (event.type === "audio.append") ws.send(JSON.stringify({ type: options.finalOnClose ? "speech.started" : "transcript.final", sequence_no: 2, utterance_id: "u1", revision: 1, text: "fixture transcript" })); else if (event.type === "session.close") { + if (options.finalOnClose) ws.send(JSON.stringify({ type: "transcript.final", utterance_id: "u1", revision: 1, text: "the last synthetic words" })); ws.send(JSON.stringify({ type: "session.updated", sequence_no: 3, session: { session_id: "fixture", status: "closed", config: { provider_mode: "streaming_sse", transcript_delivery_mode: "segment" } } })); ws.close(1000); } else ws.send(message); @@ -162,6 +173,53 @@ async function createCall(): Promise { } describe("dictation protocol validation", () => { + test("exact ScribeTray start with and without language preserves flush/close sequencing", () => { + for (const language of [undefined, "it", "en-US"]) { + const validate = createDictationFrameValidator(); + const accepts = (value: unknown) => validate(JSON.stringify(value)); + expect(accepts({ type: "audio.flush" })).toBe(false); + expect(accepts({ type: "session.close" })).toBe(false); + expect(accepts({ ...scribeStart, config: { ...scribeStart.config, ...(language ? { language } : {}) } })).toBe(true); + expect(accepts({ type: "audio.append", audio: "AAA=" })).toBe(true); + expect(accepts({ type: "audio.flush" })).toBe(true); + // Flush is nonterminal: more audio is legal until close. + expect(accepts({ type: "audio.append", audio: "AgA=" })).toBe(true); + expect(accepts({ type: "audio.flush" })).toBe(true); + expect(accepts({ type: "session.close" })).toBe(true); + for (const type of ["audio.append", "audio.flush", "session.close", "session.start"]) expect(accepts({ type, audio: "AAA=" })).toBe(false); + } + }); + test("optional VAD settings remain validated when present", () => { + for (const vad of [ + { threshold: -0.1 }, { threshold: 1.1 }, { threshold: "0.5" }, { threshold: null }, + { prefix_padding_ms: -1 }, { prefix_padding_ms: 1001 }, { prefix_padding_ms: 0.5 }, + { silence_duration_ms: -1 }, { silence_duration_ms: 5001 }, { silence_duration_ms: "500" }, + { type: "client_vad" }, { unexpected: true }, + ]) expect(createDictationFrameValidator()(JSON.stringify({ ...scribeStart, config: { ...scribeStart.config, vad: { type: "server_vad", ...vad } } }))).toBe(false); + for (const vad of [{ threshold: 0 }, { threshold: 1, prefix_padding_ms: 1000, silence_duration_ms: 5000 }]) { + expect(createDictationFrameValidator()(JSON.stringify({ ...scribeStart, config: { ...scribeStart.config, vad: { type: "server_vad", ...vad } } }))).toBe(true); + } + }); + test("compatibility additions do not admit arbitrary fields or unbounded data", () => { + for (const change of [ + { dictation_session_id: "x".repeat(129) }, { attempt_id: null }, { attempt_id: "" }, + { dictation_session_id: "bad\nvalue" }, { client_capabilities: "server_stop_requested" }, + { client_capabilities: ["unknown"] }, { client_capabilities: ["server_stop_requested", "server_stop_requested"] }, + { arbitrary: true }, + ]) expect(createDictationFrameValidator()(JSON.stringify({ ...scribeStart, ...change }))).toBe(false); + for (const change of [{ language: "" }, { language: "x".repeat(65) }, { language: 3 }, { session_asset_mode: "all" }, { arbitrary: true }, { provider_mode: ["streaming_sse"] }, { transcript_delivery_mode: ["final_only"] }]) { + expect(createDictationFrameValidator()(JSON.stringify({ ...scribeStart, config: { ...scribeStart.config, ...change } }))).toBe(false); + } + const validate = createDictationFrameValidator(); + expect(validate(Buffer.from(JSON.stringify(scribeStart)))).toBe(false); + expect(validate("{" )).toBe(false); + expect(validate(JSON.stringify(scribeStart))).toBe(true); + for (const event of [ + { type: "audio.append", audio: "A".repeat(65536) }, { type: "audio.append", audio: "not base64!" }, + { type: "audio.append", audio: "AA==" }, { type: "audio.append", audio: "" }, + { type: "audio.flush", extra: true }, { type: "session.close", extra: true }, + ]) expect(validate(JSON.stringify(event))).toBe(false); + }); test("PCM frames require a valid start, canonical base64 and an open session", () => { const validate = createDictationFrameValidator(); expect(validate(JSON.stringify({ type: "audio.append", audio: "AAA=" }))).toBe(false); @@ -276,6 +334,26 @@ describe("external audio sockets", () => { expect(fixture.handshakes[0]!.protocols?.[1]).toStartWith("openai-bearer."); expect(fixture.handshakes[0]!.protocols?.join(",")).not.toContain(KEY); }); + test("ScribeTray frames reach upstream unchanged and ordered, with finals after close", async () => { + fixture = createFixture({ finalOnClose: true }); + const ws = socket("/v1/audio/transcriptions/stream", true); + const start = { ...scribeStart, config: { ...scribeStart.config, language: "it" } }; + await receive(ws, start, "session.started"); + const append = { type: "audio.append", audio: "AAA=" }; + await receive(ws, append, "speech.started"); + const flush = { type: "audio.flush" }; + const close = { type: "session.close" }; + const finals: string[] = []; + ws.addEventListener("message", event => { + const value = JSON.parse(String(event.data)); + if (value.type === "transcript.final") finals.push(value.text); + }); + const closed = receive(ws, flush, "session.updated"); + ws.send(JSON.stringify(close)); + await closed; + expect(finals).toEqual(["the last synthetic words"]); + expect(fixture.frames).toEqual([start, append, flush, close].map(value => JSON.stringify(value))); + }); test("standalone live defaults model and negotiation only for external clients", async () => { fixture = createFixture(); const ws = socket("/v1/live"); From 74d1ad1530ab9f7279b5390a5398a9fefc77218e Mon Sep 17 00:00:00 2001 From: Valerio Coltre Date: Sat, 7 Nov 2026 00:00:11 +0700 Subject: [PATCH 3/3] fix(dictation): keep local budget expiry neutral for account health --- src/server/index/live-sideband.ts | 2 -- .../providers/codex-installation-identity.md | 1 + tests/server/audio-dictation.test.ts | 23 ++++++++++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/server/index/live-sideband.ts b/src/server/index/live-sideband.ts index 6ea61f208d9..a610b3fd8e8 100644 --- a/src/server/index/live-sideband.ts +++ b/src/server/index/live-sideband.ts @@ -314,7 +314,6 @@ export function beginAudioFinalization(ws: ServerWebSocket, raw: string ws.data.liveFinalizing = true; if (ws.data.liveSessionTimer !== undefined) clearTimeout(ws.data.liveSessionTimer); ws.data.liveSessionTimer = setTimeout(() => { - ws.data.liveOutcome = "timeout"; closeLiveSideband(ws, 1000, "audio finalization expired"); }, ws.data.liveFinalizationMs); } @@ -519,7 +518,6 @@ export function attachLiveSidebandUpstream( closeLiveSideband(ws, 1011, "audio connection timed out"); }, 10_000); ws.data.liveSessionTimer = setTimeout(() => { - if (ws.data.liveFinalizationMs !== undefined) ws.data.liveOutcome = "timeout"; closeLiveSideband(ws, 1000, "audio session expired"); }, ws.data.liveMaxSessionMs); } diff --git a/structure/providers/codex-installation-identity.md b/structure/providers/codex-installation-identity.md index 33e264e6cf3..4acd347b789 100644 --- a/structure/providers/codex-installation-identity.md +++ b/structure/providers/codex-installation-identity.md @@ -64,6 +64,7 @@ Dictation in `src/server/audio-dictation.ts` has a 600,000 ms recording budget a opts into a separate 60,000 ms finalization window after a validated `session.close`. `src/server/index/live-sideband.ts` replaces the recording timer on that event; expiry and cancellation retain the existing authenticated transport cleanup and lease ownership. +Recording and finalization budget expiry is neutral for account health; an upstream connection timeout remains a timeout. The forwarded session TTL does not shorten the recording budget; other audio transports do not opt in. Dictation validation also accepts ScribeTray's bounded identifiers, stop-request capability, language, asset mode `none`, minimal server VAD, and nonterminal `audio.flush`. diff --git a/tests/server/audio-dictation.test.ts b/tests/server/audio-dictation.test.ts index 084fedf25e4..602e1e20357 100644 --- a/tests/server/audio-dictation.test.ts +++ b/tests/server/audio-dictation.test.ts @@ -242,7 +242,7 @@ describe("dictation protocol validation", () => { } }); test("relay recording and finalization clocks are bounded and cleaned on expiry or cancel", () => { - for (const mode of ["recording", "finalizing", "cancel", "complete", "other-transport"] as const) { + for (const mode of ["recording", "finalizing", "cancel", "complete", "other-transport", "connect-timeout"] as const) { let now = 0; let id = 0; const timers = new Map void }>(); @@ -268,8 +268,25 @@ describe("dictation protocol validation", () => { }; const ws = { data, readyState: WebSocket.OPEN, close() {} }; if (mode === "other-transport") data.liveFinalizationMs = undefined; + if (mode === "connect-timeout") { + data.liveUpstream = undefined; + data.liveUpstreamUrl = "ws://fixture.invalid"; + upstream.readyState = WebSocket.CONNECTING; + } try { - attachLiveSidebandUpstream(ws as never); + attachLiveSidebandUpstream(ws as never, () => upstream as unknown as WebSocket); + if (mode === "connect-timeout") { + expect(data.liveConnectTimer).toBeDefined(); + advance(9_999); + expect(releases).toBe(0); + advance(1); + expect(releases).toBe(1); + expect(outcomes).toEqual(["timeout"]); + expect(timers.size).toBe(0); + expect(data.liveConnectTimer).toBeUndefined(); + expect(data.liveSessionTimer).toBeUndefined(); + continue; + } expect(data.liveConnectTimer).toBeUndefined(); advance(599_999); expect(releases).toBe(0); @@ -292,7 +309,7 @@ describe("dictation protocol validation", () => { } } expect(releases).toBe(1); - expect(outcomes).toEqual([["cancel", "complete", "other-transport"].includes(mode) ? undefined : "timeout"]); + expect(outcomes).toEqual([undefined]); expect(timers.size).toBe(0); expect(data.liveSessionTimer).toBeUndefined(); expect(data.liveAbortListener).toBeUndefined();