fix(dictation): extend streaming protocol support and allow ten-minute recordings

docs-site/src/content/docs/reference/proxy-formats.md

@@ -256,14 +256,45 @@ Use the actual sample rate of mono PCM16 audio. Wait for `session.started`, then
 send `{"type":"audio.append","audio":"<base64 PCM bytes>"}`. These are JSON text
 frames, not WAV files or binary WebSocket frames. The gateway accepts sample
 rates from 8,000 through 192,000 Hz; upstream support is account/service-dependent.
-Client frames are limited to 64 KiB and sessions to five minutes. Unsupported or
+Client frames are limited to 64 KiB. The proxy allows up to ten minutes
+(600,000 ms) for recording from relay attachment. A valid `session.close` before
+that deadline replaces the recording timer with a separate, bounded 60-second
+window for final transcripts. Keep the WebSocket open during finalization.
+`config.session_ttl_ms` accepts 1–600,000 and is forwarded unchanged; it does not
+set the proxy's recording timer (existing clients may still send 300,000).
+These are OpenCodex application budgets, not demonstrated official OpenAI limits.
+GPT Live and other audio transports retain their existing lifetimes. Unsupported or
 malformed event/config fields close the stream with code 1008.
 
 `transcript.segment` and `transcript.final` contain `utterance_id`, `revision`, and
 `text`. Replace prior text for the same utterance when its revision increases;
 do not concatenate revisions. Finish with `{"type":"session.close"}` and wait
 for final text and `session.updated` with `session.status="closed"`.
 
+The ScribeTray streaming client uses this additional supported start shape
+(replace the example UUIDs with client-generated identifiers):
+
+```json
+{"type":"session.start","dictation_session_id":"78d201cb-1b49-40e1-a5b5-e06a8f8489a1","attempt_id":"8b3c2e4f-d501-4a66-b123-456789abcdef","config":{"input_audio_format":"pcm16","sample_rate_hz":16000,"num_channels":1,"max_buffer_size_bytes":4194304,"max_utterance_duration_ms":30000,"session_ttl_ms":300000,"provider_mode":"streaming_sse","transcript_delivery_mode":"final_only","session_asset_mode":"none","vad":{"type":"server_vad"}},"client_capabilities":["server_stop_requested"]}
+```
+
+The identifiers are optional, nonblank strings of at most 128 UTF-8 bytes.
+`client_capabilities` is optional and accepts an empty array or just
+`"server_stop_requested"`. Optional `config.language` is a nonblank string of at
+most 64 UTF-8 bytes (for example `"it"`); these strings cannot contain control
+characters. Optional `config.session_asset_mode` accepts only `"none"`.
+VAD needs only `{"type":"server_vad"}`. If supplied, `threshold` must be finite
+and within 0–1, `prefix_padding_ms` an integer within 0–1,000, and
+`silence_duration_ms` an integer within 0–5,000.
+
+After start, `{"type":"audio.flush"}` is also accepted and forwarded unchanged.
+It does not close the session; more audio may follow until `session.close`.
+ScribeTray finishes by sending `audio.flush`, then `session.close`, then waiting
+for final results. This supports that client's sequence without asserting that
+flush is required instead of close alone. No audio or other client events are
+accepted after `session.close`. The earlier start and close-only payloads remain
+supported. Unknown fields and unsupported capability values remain rejected.
+
 ## `POST /v1/responses`
 
 This is the native opencodex data-plane shape. The request body must be a JSON object with a

src/server/audio-dictation.ts

@@ -7,13 +7,17 @@ import { resolveAudioUpstream, TRANSCRIPTION_MODEL, type AudioUpstream } from ".
 import type { RequestLogContext } from "./request-log";
 import { authorizationCredential } from "../lib/authorization-scheme";
 
-export const DICTATION_SESSION_MAX_MS = 300_000;
+export const DICTATION_SESSION_MAX_MS = 600_000;
+export const DICTATION_FINALIZATION_MAX_MS = 60_000;
 const DICTATION_FRAME_MAX_BYTES = 64 * 1024;
 type JsonObject = Record<string, unknown>;
 function object(value: unknown): value is JsonObject { return !!value && typeof value === "object" && !Array.isArray(value); }
 function integer(value: unknown, min: number, max: number): boolean {
   return typeof value === "number" && Number.isInteger(value) && value >= min && value <= max;
 }
+function boundedString(value: unknown, max: number): boolean {
+  return typeof value === "string" && value.trim().length > 0 && Buffer.byteLength(value) <= max && !/[\u0000-\u001f\u007f]/.test(value);
+}
 
 export function createDictationFrameValidator(): (frame: string | Buffer) => boolean {
   let started = false;
@@ -25,21 +29,29 @@ export function createDictationFrameValidator(): (frame: string | Buffer) => boo
     if (!object(event)) return false;
     if (event.type === "session.start" && !started) {
       const config = event.config;
-      if (!object(config) || Object.keys(event).some(key => key !== "type" && key !== "config")) return false;
-      const fields = new Set(["input_audio_format", "sample_rate_hz", "num_channels", "max_buffer_size_bytes", "max_utterance_duration_ms", "session_ttl_ms", "provider_mode", "transcript_delivery_mode", "vad"]);
+      if (!object(config) || Object.keys(event).some(key => !["type", "config", "dictation_session_id", "attempt_id", "client_capabilities"].includes(key))) return false;
+      for (const key of ["dictation_session_id", "attempt_id"]) if (key in event && !boundedString(event[key], 128)) return false;
+      if ("client_capabilities" in event && (!Array.isArray(event.client_capabilities) || event.client_capabilities.length > 1
+        || event.client_capabilities.some(value => value !== "server_stop_requested"))) return false;
+      const fields = new Set(["input_audio_format", "sample_rate_hz", "num_channels", "max_buffer_size_bytes", "max_utterance_duration_ms", "session_ttl_ms", "provider_mode", "transcript_delivery_mode", "session_asset_mode", "language", "vad"]);
+      if ("session_asset_mode" in config && config.session_asset_mode !== "none") return false;
+      if ("language" in config && !boundedString(config.language, 64)) return false;
       if (Object.keys(config).some(key => !fields.has(key)) || config.input_audio_format !== "pcm16" || config.num_channels !== 1
         || !integer(config.sample_rate_hz, 8000, 192000) || !integer(config.max_buffer_size_bytes, 1, 4 * 1024 * 1024)
         || !integer(config.max_utterance_duration_ms, 1, 30000) || !integer(config.session_ttl_ms, 1, DICTATION_SESSION_MAX_MS)
-        || !["buffered", "streaming_sse"].includes(String(config.provider_mode))
-        || !["final_only", "segment", "delta"].includes(String(config.transcript_delivery_mode))) return false;
+        || typeof config.provider_mode !== "string" || !["buffered", "streaming_sse"].includes(config.provider_mode)
+        || typeof config.transcript_delivery_mode !== "string" || !["final_only", "segment", "delta"].includes(config.transcript_delivery_mode)) return false;
       const vad = config.vad;
       if (!object(vad) || Object.keys(vad).some(key => !["type", "threshold", "prefix_padding_ms", "silence_duration_ms"].includes(key))
-        || vad.type !== "server_vad" || typeof vad.threshold !== "number" || !Number.isFinite(vad.threshold) || vad.threshold < 0 || vad.threshold > 1
-        || !integer(vad.prefix_padding_ms, 0, 1000) || !integer(vad.silence_duration_ms, 0, 5000)) return false;
+        || vad.type !== "server_vad"
+        || ("threshold" in vad && (typeof vad.threshold !== "number" || !Number.isFinite(vad.threshold) || vad.threshold < 0 || vad.threshold > 1))
+        || ("prefix_padding_ms" in vad && !integer(vad.prefix_padding_ms, 0, 1000))
+        || ("silence_duration_ms" in vad && !integer(vad.silence_duration_ms, 0, 5000))) return false;
       started = true;
       return true;
     }
     if (!started) return false;
+    if (event.type === "audio.flush" && Object.keys(event).length === 1) return true;
     if (event.type === "session.close" && Object.keys(event).length === 1) { closed = true; return true; }
     if (event.type !== "audio.append" || Object.keys(event).some(key => key !== "type" && key !== "audio") || typeof event.audio !== "string" || !event.audio) return false;
     const bytes = Buffer.from(event.audio, "base64");
@@ -54,6 +66,7 @@ export interface AudioSocketTarget {
   protocols?: string[];
   validateFrame?: (frame: string | Buffer) => boolean;
   maxSessionMs: number;
+  finalizationMs?: number;
   finish: (outcome?: number | "timeout" | "connect_error") => void;
 }
 
@@ -90,6 +103,7 @@ export async function resolveDictationSocket(
     protocols: ["chatgpt-dictation", `openai-bearer.${token}`, "codex-desktop"],
     validateFrame: createDictationFrameValidator(),
     maxSessionMs: DICTATION_SESSION_MAX_MS,
+    finalizationMs: DICTATION_FINALIZATION_MAX_MS,
     finish: finishAudioUpstream(relay),
   };
 }

src/server/index/live-sideband.ts

@@ -306,6 +306,18 @@ export function closeLiveSideband(ws: ServerWebSocket<WsData>, code = 1000, reas
   }
 }
 
+/** Only dictation opts in: a validated close gets a separate bounded drain window. */
+export function beginAudioFinalization(ws: ServerWebSocket<WsData>, raw: string | Buffer): void {
+  if (ws.data.liveFinalizationMs === undefined || ws.data.liveFinalizing || ws.data.liveClosing) return;
+  // The frame validator has already accepted this JSON text frame.
+  if (typeof raw !== "string" || JSON.parse(raw).type !== "session.close") return;
+  ws.data.liveFinalizing = true;
+  if (ws.data.liveSessionTimer !== undefined) clearTimeout(ws.data.liveSessionTimer);
+  ws.data.liveSessionTimer = setTimeout(() => {
+    closeLiveSideband(ws, 1000, "audio finalization expired");
+  }, ws.data.liveFinalizationMs);
+}
+
 /**
  * Dial the upstream sideband and report whether its handshake reached 101.
  *
@@ -505,7 +517,9 @@ export function attachLiveSidebandUpstream(
       ws.data.liveOutcome = "timeout";
       closeLiveSideband(ws, 1011, "audio connection timed out");
     }, 10_000);
-    ws.data.liveSessionTimer = setTimeout(() => closeLiveSideband(ws, 1000, "audio session expired"), ws.data.liveMaxSessionMs);
+    ws.data.liveSessionTimer = setTimeout(() => {
+      closeLiveSideband(ws, 1000, "audio session expired");
+    }, ws.data.liveMaxSessionMs);
   }
 
   upstream.addEventListener("close", (event) => {

src/server/index/serve-options.ts

@@ -1864,6 +1864,7 @@ export function createServeOptions(ctx: ServeOptionsContext) {
               liveUpstreamProtocols: audio?.protocols,
               liveValidateFrame: audio?.validateFrame,
               liveMaxSessionMs: audio?.maxSessionMs,
+              liveFinalizationMs: audio?.finalizationMs,
               liveFinish: finish,
               liveAbortSignal: audioController?.signal,
               livePending: [],

src/server/index/websocket-handler.ts

@@ -10,6 +10,7 @@ import {
   MAX_WS_FRAME_BYTES,
   WEBSOCKET_IDLE_TIMEOUT_SECONDS,
   attachLiveSidebandUpstream,
+  beginAudioFinalization,
   clientCloseForUpstream,
   closeLiveSideband,
   closeLiveSidebandBeforeUpgrade,
@@ -156,6 +157,7 @@ export function createWebsocketHandler(
             closeLiveSideband(ws, 1008, "invalid audio event");
             return;
           }
+          beginAudioFinalization(ws, raw);
           const rawBytes = webSocketFrameBytes(raw);
           if (exceedsLiveSidebandFrameByteLimit(rawBytes)) {
             closeLiveSideband(ws, 1009, "message too large");

src/server/ws-bridge.ts

@@ -53,6 +53,8 @@ export interface WsData {
   liveFinish?: AudioSocketTarget["finish"];
   liveOutcome?: number | "timeout" | "connect_error";
   liveMaxSessionMs?: number;
+  liveFinalizationMs?: number;
+  liveFinalizing?: boolean;
   liveConnectTimer?: ReturnType<typeof setTimeout>;
   liveSessionTimer?: ReturnType<typeof setTimeout>;
   liveAbortSignal?: AbortSignal;

structure/data-planes/inbound-compat.md

@@ -63,8 +63,8 @@ unchanged, on both join credentials described under streaming audio. Coverage li
 `src/server/audio-client.ts` recognizes explicit audio keys before local legacy admission.
 Browser sockets offer opencodex-audio and opencodex-key.<base64url-key>; only the public
 marker is selected downstream. Invalid presented keys cannot become credential-free native calls.
-`src/server/audio-dictation.ts` maps the validated desktop session.start/audio.append/session.close
-protocol to ChatGPT dictation with server-owned credentials and a five-minute lifetime.
+`src/server/audio-dictation.ts` validates session.start/audio.append/audio.flush/session.close for ChatGPT dictation with server-owned credentials, a ten-minute recording budget, and a separate bounded 60-second finalization window after close.
+The forwarded TTL does not set the recording budget. ScribeTray's bounded optional metadata and minimal server VAD preserve PCM checks and refusal of audio after close.
 The `src/server/index.ts` byte relay retains bounded queues, handshake/session deadlines and
 the account/turn lifecycle until its upstream closes. `src/server/ws-bridge.ts` carries the
 in-memory callbacks and signal; handshake credentials are cleared after socket construction.

structure/providers/codex-installation-identity.md

@@ -60,6 +60,15 @@ JSON send, including live's converted multipart session. Live inspects JSON cont
 of Content-Type; non-JSON SDP retains its original bytes. Their selected headers authoritatively
 replace or remove mixed-case provider identity defaults; main and main-pool preserve caller values.
 Vision, audio and sidecar credential selection share materialization.
+Dictation in `src/server/audio-dictation.ts` has a 600,000 ms recording budget and
+opts into a separate 60,000 ms finalization window after a validated `session.close`.
+`src/server/index/live-sideband.ts` replaces the recording timer on that event;
+expiry and cancellation retain the existing authenticated transport cleanup and lease ownership.
+Recording and finalization budget expiry is neutral for account health; an upstream connection timeout remains a timeout.
+The forwarded session TTL does not shorten the recording budget; other audio transports do not opt in.
+Dictation validation also accepts ScribeTray's bounded identifiers, stop-request capability,
+language, asset mode `none`, minimal server VAD, and nonterminal `audio.flush`.
+Optional VAD values retain their numeric bounds; start/close ordering and PCM checks remain enforced.
 `src/server/audio-live.ts` projects the authenticated call's cloned session before JSON or
 multipart serialization and returns a sanitized 503 without sending on identity failure.
 Its socket resolver and `src/server/audio-dictation.ts` carry the selected auth context through