From a530bcd8346129c9cea1b16cb1d54ba97c5d9b29 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:30:51 +0000 Subject: [PATCH 01/15] test(openai): cover list-shaped content in streamed sse deltas Adds tests for thinking models behind OpenAI-compatible gateways that stream delta content as typed parts, covering the transition chunk that carries the closing thought and the first visible words, thinking-only chunks, plain string content, and unknown part types. The transition case guards against a regression where that chunk failed to deserialize and was dropped, truncating the start of the answer. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/sse_tests.rs | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/openai/sse_tests.rs b/crates/tinyinference-llm/src/providers/openai/sse_tests.rs index 53f97fa9..c74f36d4 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse_tests.rs @@ -182,3 +182,213 @@ async fn huge_tool_call_index_is_rejected() { "an out-of-range tool-call index was accepted" ); } + +/// Concatenates the visible-text and reasoning fragments across every streamed +/// `MessageDelta`, as `(text, reasoning)`. +fn streamed_channels(items: &[ModelStreamItem]) -> (String, String) { + let mut text = String::new(); + let mut reasoning = String::new(); + for item in items { + if let ModelStreamItem::MessageDelta(delta) = item { + text.push_str(&delta.text); + reasoning.push_str(&delta.reasoning); + } + } + (text, reasoning) +} + +/// The text of every `Thinking` block on a response, concatenated. +fn thinking_text(response: &ModelResponse) -> String { + response + .message + .content + .iter() + .filter_map(|block| block.as_thinking().map(|(text, _)| text.to_string())) + .collect() +} + +/// A thinking model behind an OpenAI-compatible gateway (Surplus serving +/// `glm-5.3-flash`) streams `delta.content` as a list of typed parts while it +/// thinks, and the chunk that closes the thinking also carries the first +/// visible words. These are the captured frames. That transition chunk used +/// to fail to deserialize and was silently dropped, so the answer lost its +/// opening ("ALPHA BRAVO CHARLIE" arrived as " CHARLIE"). +#[tokio::test] +async fn sse_stream_keeps_the_answer_opening_carried_with_list_content_thinking() { + let items = stream_items(vec![ + br#"data: {"id":"c1","choices":[{"index":0,"delta":{"role":"assistant","content":[{"type":"thinking","thinking":[{"type":"text","text":" user wants"}],"closed":false}]}}]} + +"#, + br#"data: {"id":"c1","choices":[{"index":0,"delta":{"content":[{"type":"thinking","thinking":[{"type":"text","text":"."}],"closed":true},{"type":"text","text":"ALPHA BRAVO"}]}}]} + +"#, + br#"data: {"id":"c1","choices":[{"index":0,"delta":{"content":" CHARLIE"},"finish_reason":"stop"}]} + +"#, + b"data: [DONE]\n\n", + ]) + .await; + + let (text, reasoning) = streamed_channels(&items); + assert_eq!(text, "ALPHA BRAVO CHARLIE"); + assert_eq!(reasoning, " user wants."); + + // Within the transition chunk the closing thought streams before the + // first visible words, so the block order stays thinking -> text. + let position = |wanted: &dyn Fn(&MessageDelta) -> bool| { + items + .iter() + .position(|item| matches!(item, ModelStreamItem::MessageDelta(d) if wanted(d))) + .expect("delta present") + }; + assert!(position(&|d| d.reasoning == ".") < position(&|d| d.text == "ALPHA BRAVO")); + + let response = items + .into_iter() + .find_map(|item| match item { + ModelStreamItem::Completed(response) => Some(response), + _ => None, + }) + .expect("stream ends with Completed"); + assert_eq!(response.text(), "ALPHA BRAVO CHARLIE"); + assert_eq!(thinking_text(&response), " user wants."); + assert!( + matches!( + response.message.content.as_slice(), + [ContentBlock::Thinking { .. }, ContentBlock::Text(_)] + ), + "{:?}", + response.message.content + ); +} + +/// A chunk whose list content holds only thinking parts feeds the reasoning +/// channel and never the visible text. The `thinking` field may be a plain +/// string as well as a list of text parts. +#[tokio::test] +async fn sse_stream_routes_a_thinking_only_list_content_to_reasoning() { + let items = stream_items(vec![ + br#"data: {"choices":[{"delta":{"content":[{"type":"thinking","thinking":[{"type":"text","text":"weigh "}],"closed":false}]}}]} + +"#, + br#"data: {"choices":[{"delta":{"content":[{"type":"thinking","thinking":"it up","closed":true}]},"finish_reason":"stop"}]} + +"#, + b"data: [DONE]\n\n", + ]) + .await; + + let (text, reasoning) = streamed_channels(&items); + assert_eq!(text, ""); + assert_eq!(reasoning, "weigh it up"); + let response = items + .into_iter() + .find_map(|item| match item { + ModelStreamItem::Completed(response) => Some(response), + _ => None, + }) + .expect("stream ends with Completed"); + assert_eq!(response.text(), ""); + assert_eq!(thinking_text(&response), "weigh it up"); +} + +/// Plain string content, the shape every non-thinking reply uses, streams +/// exactly as before: all visible text, no reasoning. +#[tokio::test] +async fn sse_stream_plain_string_content_is_all_visible_text() { + let items = stream_items(vec![ + br#"data: {"choices":[{"delta":{"content":"ALPHA"}}]} + +"#, + br#"data: {"choices":[{"delta":{"content":" BRAVO"},"finish_reason":"stop"}]} + +"#, + b"data: [DONE]\n\n", + ]) + .await; + + assert_eq!( + streamed_channels(&items), + ("ALPHA BRAVO".to_string(), String::new()) + ); +} + +/// A list mixing text, `reasoning` parts (carrying `text`), and a part type +/// this parser does not know: the text parts concatenate in order, the +/// reasoning part goes to the reasoning channel, and the unknown part is +/// skipped without losing the rest of the chunk. +#[tokio::test] +async fn sse_stream_list_content_skips_unknown_parts_and_keeps_the_rest() { + let response = completed(vec![ + br#"data: {"choices":[{"delta":{"content":[{"type":"text","text":"one "},{"type":"image_url","image_url":{"url":"https://example.invalid/x.png"}},{"type":"reasoning","text":"aside"},{"type":"text","text":"two"}]},"finish_reason":"stop"}]} + +"#, + b"data: [DONE]\n\n", + ]) + .await; + + assert_eq!(response.text(), "one two"); + assert_eq!(thinking_text(&response), "aside"); +} + +/// Formatted log output, captured for assertions on what was warned. +#[derive(Clone, Default)] +struct LogBuffer(std::sync::Arc>>); + +impl LogBuffer { + fn contents(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned() + } +} + +impl std::io::Write for LogBuffer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogBuffer { + type Writer = LogBuffer; + + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } +} + +/// A chunk that still does not fit the chunk schema is skipped, as before, +/// but no longer silently: it is warned about with a truncated copy of the +/// payload, and the rest of the stream still arrives. +#[tokio::test] +async fn sse_stream_warns_about_a_chunk_it_cannot_read() { + let logs = LogBuffer::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let unreadable: &'static [u8] = format!( + "data: {{\"choices\":[{{\"index\":\"zero\",\"delta\":{{\"content\":\"{}\"}}}}]}}\n\n", + "x".repeat(4_000) + ) + .leak() + .as_bytes(); + let response = completed(vec![ + b"data: {\"choices\":[{\"delta\":{\"content\":\"kept\"}}]}\n\n", + unreadable, + b"data: {\"choices\":[{\"delta\":{\"content\":\" too\"},\"finish_reason\":\"stop\"}]}\n\n", + b"data: [DONE]\n\n", + ]) + .await; + + assert_eq!(response.text(), "kept too"); + let logs = logs.contents(); + assert!(logs.contains("WARN"), "{logs}"); + assert!(logs.contains(r#"\"index\":\"zero\""#) || logs.contains(r#""index":"zero""#), "{logs}"); + assert!(!logs.contains(&"x".repeat(4_000)), "payload not truncated: {logs}"); +} From bcc8cf9cf128253d1024366af53a001a2a58df3d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:31:04 +0000 Subject: [PATCH 02/15] test(openai): cover list content parts in chat responses Adds a test for parsing chat responses whose message content is a list of typed parts, verifying that text parts form the answer, thinking parts are folded into reasoning after any side-channel reasoning, and unknown part types are ignored. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/mod_tests.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/openai/mod_tests.rs b/crates/tinyinference-llm/src/providers/openai/mod_tests.rs index 8f90e7a1..ee289cf6 100644 --- a/crates/tinyinference-llm/src/providers/openai/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/mod_tests.rs @@ -1180,6 +1180,37 @@ fn parse_chat_response_without_config_leaves_inline_tags_in_text() { assert_eq!(response_reasoning(&response), ""); } +/// A thinking model behind an OpenAI-compatible gateway returns the message +/// `content` as a list of typed parts. Text parts are the answer; thinking +/// parts are reasoning, after any side-channel reasoning, and never visible. +#[test] +fn parse_chat_response_reads_list_content_parts() { + let body = json!({ + "id": "chatcmpl-list", + "choices": [{ + "message": { + "role": "assistant", + "reasoning_content": "side ", + "content": [ + { "type": "thinking", "thinking": [{ "type": "text", "text": "plan" }], "closed": true }, + { "type": "text", "text": "{\"reason\":" }, + { "type": "unknown_part", "data": 1 }, + { "type": "text", "text": "\"ok\"}" } + ] + }, + "finish_reason": "stop" + }] + }); + + let response = parse_chat_response(body, None).unwrap(); + assert_eq!(response.text(), "{\"reason\":\"ok\"}"); + assert_eq!(response_reasoning(&response), "side plan"); + assert!(matches!( + response.message.content.as_slice(), + [ContentBlock::Thinking { .. }, ContentBlock::Text(_)] + )); +} + #[tokio::test] async fn sse_stream_reassembles_multibyte_char_split_across_chunks() { // A 4-byte emoji in the content payload, split down the middle across two From 78369539964da9adf726976dcfd8bc1228920799 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:31:09 +0000 Subject: [PATCH 03/15] test(openai): cover list content completions as reasoning then text Adds a wire test for a non-streaming completion whose content is a list of thinking and text parts, verifying the text parts become the answer and the thinking parts become reasoning instead of failing the call. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/wire_tests.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs index 6c5aaf9e..9a02f690 100644 --- a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs @@ -633,6 +633,42 @@ fn unsupported(parameter: &str) -> String { .to_string() } +/// Over the wire, a non-streaming completion whose `content` is a list of +/// thinking and text parts (as a thinking model behind Surplus returns it) +/// is read rather than failing the call: the answer is the text parts, the +/// thinking parts are reasoning. +#[tokio::test] +async fn a_list_content_completion_is_read_as_reasoning_then_text() { + let body = json!({ + "id": "chatcmpl-list", + "object": "chat.completion", + "model": "glm-5.3-flash", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": [ + { "type": "thinking", "thinking": [{ "type": "text", "text": "." }], "closed": true }, + { "type": "text", "text": "ALPHA BRAVO CHARLIE" } + ] + }, + "finish_reason": "stop" + }] + }) + .to_string(); + let (base, server) = serve_sequence(vec![(200, body)]); + let model = OpenAiModel::new("k").with_base_url(&base); + + let response = model + .invoke(&(), request("list-content-model", 0.2)) + .await + .unwrap(); + server.join().unwrap(); + + assert_eq!(response.text(), "ALPHA BRAVO CHARLIE"); + assert_eq!(response.reasoning(), "."); +} + #[tokio::test] async fn a_rejected_parameter_is_dropped_and_the_request_retried_once() { let (base, server) = serve_sequence(vec![ From d988a45011fc92506659e317850cd5ddbe21a2d8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:31:26 +0000 Subject: [PATCH 04/15] test(openai): assert reasoning block content directly in wire test The list-content completion test now checks the first content block's thinking value instead of the flattened reasoning string, so the assertion reflects how reasoning is actually stored on the message. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/openai/wire_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs index 9a02f690..6ce6b7ec 100644 --- a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs @@ -666,7 +666,10 @@ async fn a_list_content_completion_is_read_as_reasoning_then_text() { server.join().unwrap(); assert_eq!(response.text(), "ALPHA BRAVO CHARLIE"); - assert_eq!(response.reasoning(), "."); + assert_eq!( + response.message.content.first().and_then(|block| block.as_thinking()), + Some((".", None)) + ); } #[tokio::test] From 1351d81e824412db3a2f5c06d8d07cb6a6f0f1fc Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:04 +0000 Subject: [PATCH 05/15] feat(openai): accept typed content parts in responses and stream chunks OpenAI-compatible gateways serving thinking models can send message and delta content as a list of typed parts rather than a plain string, so content is now modelled as an untagged enum covering both shapes. Parts stay raw JSON so an unknown part is skipped on its own instead of failing the whole chunk. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/types.rs | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/openai/types.rs b/crates/tinyinference-llm/src/providers/openai/types.rs index 36f0d3b2..1be42f88 100644 --- a/crates/tinyinference-llm/src/providers/openai/types.rs +++ b/crates/tinyinference-llm/src/providers/openai/types.rs @@ -106,9 +106,10 @@ pub struct ChunkChoiceWire { /// The incremental `delta` object inside a [`ChunkChoiceWire`]. #[derive(Clone, Debug, Default, Deserialize)] pub struct ChunkDeltaWire { - /// Incremental text fragment, when present. + /// Incremental content fragment, when present: a string, or a list of + /// typed parts from a thinking model (see [`ResponseContentWire`]). #[serde(default)] - pub content: Option, + pub content: Option, /// Incremental reasoning/thinking fragment used by some OpenAI-compatible /// providers (for example DeepSeek-compatible streams). #[serde(default)] @@ -388,9 +389,10 @@ pub struct ChoiceWire { /// The assistant message inside a [`ChoiceWire`]. #[derive(Clone, Debug, Deserialize)] pub struct ResponseMessageWire { - /// Textual content, when present. + /// Message content, when present: a string, or a list of typed parts + /// from a thinking model (see [`ResponseContentWire`]). #[serde(default)] - pub content: Option, + pub content: Option, /// Reasoning/thinking content on non-streaming responses, when a compatible /// provider exposes it. Kept off visible assistant text by the translator. #[serde(default)] @@ -403,6 +405,30 @@ pub struct ResponseMessageWire { pub tool_calls: Vec, } +/// The `content` of a [`ResponseMessageWire`] or a streamed [`ChunkDeltaWire`]. +/// +/// OpenAI sends a plain string. Some OpenAI-compatible gateways serving a +/// thinking model (Surplus with `glm-5.3-flash`, for one) send a list of typed +/// parts instead, and the chunk that closes the thinking also carries the +/// first visible text: +/// +/// ```json +/// [{"type": "thinking", "thinking": [{"type": "text", "text": "."}], "closed": true}, +/// {"type": "text", "text": "ALPHA BRAVO"}] +/// ``` +/// +/// `text` parts are visible text; `thinking` and `reasoning` parts are +/// reasoning and never visible. Parts stay raw JSON so a part of an unknown +/// type or shape is skipped on its own instead of failing the whole chunk. +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum ResponseContentWire { + /// Plain-string content: all of it visible text. + Text(String), + /// A list of typed content parts. + Parts(Vec), +} + /// Token usage reported by OpenAI. #[derive(Clone, Debug, Default, Deserialize)] pub struct UsageWire { From 75610236698d68c6734389f7401c77ae5a3147c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:11 +0000 Subject: [PATCH 06/15] feat(llm): split openai response content into text and reasoning Add a helper that separates visible text from reasoning when an OpenAI-compatible response or delta carries content as a list of parts, concatenating text parts in order and normalizing thinking or reasoning parts the same way reasoning_content is handled. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/convert.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/openai/convert.rs b/crates/tinyinference-llm/src/providers/openai/convert.rs index ad29e9d0..b2815fd5 100644 --- a/crates/tinyinference-llm/src/providers/openai/convert.rs +++ b/crates/tinyinference-llm/src/providers/openai/convert.rs @@ -644,6 +644,43 @@ pub(super) fn reasoning_value_text(value: Value) -> Option { } } +/// Splits a response or delta `content` into `(visible text, reasoning)`. +/// +/// A string is all visible text. In a list of parts, `text` parts are visible +/// text, concatenated in order; `thinking` / `reasoning` parts are reasoning, +/// read from their `thinking` field (a string or a list of text parts) or else +/// their `text` field, and normalized like `reasoning_content`. Parts of any +/// other type, and anything that is not an object, are skipped. +pub(super) fn split_response_content(content: ResponseContentWire) -> (String, String) { + let parts = match content { + ResponseContentWire::Text(text) => return (text, String::new()), + ResponseContentWire::Parts(parts) => parts, + }; + let mut text = String::new(); + let mut reasoning = String::new(); + for part in &parts { + match part.get("type").and_then(Value::as_str) { + Some("text") => { + if let Some(fragment) = part.get("text").and_then(Value::as_str) { + text.push_str(fragment); + } + } + Some("thinking" | "reasoning") => { + if let Some(fragment) = part + .get("thinking") + .or_else(|| part.get("text")) + .cloned() + .and_then(reasoning_value_text) + { + reasoning.push_str(&fragment); + } + } + _ => {} + } + } + (text, reasoning) +} + /// Extracts the reasoning/thinking text from a streamed delta, accepting the /// common OpenAI-compatible aliases. pub(super) fn delta_reasoning_text(delta: &mut ChunkDeltaWire) -> String { From 7277745514abeb24859601df1617b50b2a56b14a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:18 +0000 Subject: [PATCH 07/15] fix(openai): extract reasoning from list content parts Responses whose content arrives as a list of parts now have their thinking segments split out and appended to the reasoning side channel before inline tag extraction runs on the visible text. Previously those parts were dropped, so reasoning was lost for providers that return structured content. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/convert.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/openai/convert.rs b/crates/tinyinference-llm/src/providers/openai/convert.rs index b2815fd5..8eeba275 100644 --- a/crates/tinyinference-llm/src/providers/openai/convert.rs +++ b/crates/tinyinference-llm/src/providers/openai/convert.rs @@ -408,11 +408,16 @@ pub(super) fn parse_chat_response( } } + // Thinking parts of a list `content` follow the side-channel reasoning. + let (text, content_reasoning) = choice + .message + .content + .map(split_response_content) + .unwrap_or_default(); + reasoning.push_str(&content_reasoning); + // Inline `` extraction on the visible content, when enabled. - let visible = match ( - choice.message.content.filter(|t| !t.is_empty()), - reasoning_tags, - ) { + let visible = match (Some(text).filter(|t| !t.is_empty()), reasoning_tags) { (Some(text), Some(config)) => { let (visible, inline) = extract_reasoning(config, &text); if !inline.is_empty() { From f7f25de987def74dc958903e6505a5b963b7d655 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:20 +0000 Subject: [PATCH 08/15] fix(openai): extract reasoning from list content in stream deltas List-shaped content in a stream delta can mix thinking and text parts, and the reasoning portion was being dropped. Split those parts so reasoning is emitted before text and block ordering stays thinking then text. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/openai/sse.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/openai/sse.rs b/crates/tinyinference-llm/src/providers/openai/sse.rs index d3ff5b3d..6d245093 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse.rs @@ -236,7 +236,17 @@ impl OpenAiStreamAcc { if let Some(reason) = choice.finish_reason { self.finish_reason = Some(reason); } - let reasoning = delta_reasoning_text(&mut choice.delta); + // A list `content` can carry thinking and text parts in one chunk; + // its reasoning streams before its text, after any side-channel + // reasoning, so the block order stays thinking -> text. + let (content, content_reasoning) = choice + .delta + .content + .take() + .map(split_response_content) + .unwrap_or_default(); + let mut reasoning = delta_reasoning_text(&mut choice.delta); + reasoning.push_str(&content_reasoning); if !reasoning.is_empty() { self.reasoning.push_str(&reasoning); self.push_block_delta(BlockRequest::Reasoning, &reasoning, pending); From 639fb89ae1e37e7b8e00289b37e3975a026b34ec Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:24 +0000 Subject: [PATCH 09/15] fix(llm): emit non-empty content deltas from openai streams The content delta check now tests the accumulated content buffer instead of the incoming delta, so non-empty deltas are no longer dropped when the buffer already holds text. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/openai/sse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/openai/sse.rs b/crates/tinyinference-llm/src/providers/openai/sse.rs index 6d245093..61857796 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse.rs @@ -256,7 +256,7 @@ impl OpenAiStreamAcc { tool_call: None, })); } - if let Some(content) = choice.delta.content.filter(|c| !c.is_empty()) { + if !content.is_empty() { // Retain the raw content so the terminal response can recompute // an authoritative inline-tag split (see `into_response`). self.text.push_str(&content); From 8cfb65112eeb56caca5848ccba6bba87b9acca9e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:31 +0000 Subject: [PATCH 10/15] fix(openai): log chunks skipped by sse schema mismatch Chunks that fail to deserialize into ChatCompletionChunk are still dropped, but now emit a warning with provider, model, error, and a payload preview so silent data loss is visible. A dropped chunk previously cost every third answer its opening words without any trace. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/sse.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/openai/sse.rs b/crates/tinyinference-llm/src/providers/openai/sse.rs index 61857796..9dac5f41 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse.rs @@ -720,7 +720,23 @@ impl SseState { self.acc.keep_usage_extensions(usage); } } - if let Ok(chunk) = serde_json::from_value::(value) { + let chunk = match serde_json::from_value::(value) { + Ok(chunk) => Some(chunk), + Err(error) => { + // Still skipped, but never silently: a dropped chunk loses + // whatever text it carried, which once cost every third + // answer its opening words. + tracing::warn!( + provider = %self.provider, + model = %self.model, + %error, + payload = %log_preview(payload), + "openai sse: skipped a chunk that does not match the chunk schema" + ); + None + } + }; + if let Some(chunk) = chunk { // `resolve_slot` grows the slot vector up to the wire index, so an // out-of-range index is rejected before it can size an allocation. if let Some(index) = chunk From 6b58a04c7f4a0563b76753f710cab7905a4d791e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:39 +0000 Subject: [PATCH 11/15] feat(openai): add bounded log preview helper for sse payloads Add a log_preview helper that truncates an SSE payload to its first 512 characters, appending an ellipsis when cut, so that logging a payload keeps log lines bounded while remaining UTF-8 safe. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/openai/sse.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/openai/sse.rs b/crates/tinyinference-llm/src/providers/openai/sse.rs index 9dac5f41..06388f4f 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse.rs @@ -831,6 +831,18 @@ impl SseState { } } +/// Characters of an SSE payload kept when one is logged. +const LOG_PREVIEW_CHARS: usize = 512; + +/// The first [`LOG_PREVIEW_CHARS`] characters of `payload` (UTF-8 safe), +/// marked with `...` when cut, so a log line stays bounded. +fn log_preview(payload: &str) -> String { + match payload.char_indices().nth(LOG_PREVIEW_CHARS) { + Some((end, _)) => format!("{}...", &payload[..end]), + None => payload.to_string(), + } +} + /// Advances the SSE [`SseState`] by one item for [`futures::stream::unfold`]. pub(super) async fn sse_next(mut state: SseState) -> Option<(ModelStreamItem, SseState)> { loop { From 8edf163464fc43d07149b1d73f8cf512c2474b8b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:32:53 +0000 Subject: [PATCH 12/15] test(openai): update null tool_calls assertion for typed content The test now compares the delta content against the typed ResponseContentWire::Text variant instead of a plain string, matching the current wire representation. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/openai/mod_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/openai/mod_tests.rs b/crates/tinyinference-llm/src/providers/openai/mod_tests.rs index ee289cf6..a3383226 100644 --- a/crates/tinyinference-llm/src/providers/openai/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/mod_tests.rs @@ -3066,7 +3066,10 @@ fn a_null_tool_calls_delta_is_read_as_no_fragments() { })) .expect("a null tool_calls must not fail the chunk"); assert!(chunk.choices[0].delta.tool_calls.is_empty()); - assert_eq!(chunk.choices[0].delta.content.as_deref(), Some("Hi")); + assert_eq!( + chunk.choices[0].delta.content, + Some(ResponseContentWire::Text("Hi".into())) + ); } /// Unwraps a [`ModelStreamItem::BlockStart`], panicking with the actual item From b677ca025387b7b11a467cca754e1048383c2019 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:33:04 +0000 Subject: [PATCH 13/15] test(openai): reformat assertions to satisfy line width Wrapped two long assertion expressions in the SSE and wire tests so they fit the formatter's line width. No test behaviour changed. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/openai/sse_tests.rs | 10 ++++++++-- .../src/providers/openai/wire_tests.rs | 6 +++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/openai/sse_tests.rs b/crates/tinyinference-llm/src/providers/openai/sse_tests.rs index c74f36d4..c055e802 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse_tests.rs @@ -389,6 +389,12 @@ async fn sse_stream_warns_about_a_chunk_it_cannot_read() { assert_eq!(response.text(), "kept too"); let logs = logs.contents(); assert!(logs.contains("WARN"), "{logs}"); - assert!(logs.contains(r#"\"index\":\"zero\""#) || logs.contains(r#""index":"zero""#), "{logs}"); - assert!(!logs.contains(&"x".repeat(4_000)), "payload not truncated: {logs}"); + assert!( + logs.contains(r#"\"index\":\"zero\""#) || logs.contains(r#""index":"zero""#), + "{logs}" + ); + assert!( + !logs.contains(&"x".repeat(4_000)), + "payload not truncated: {logs}" + ); } diff --git a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs index 6ce6b7ec..2aee13d5 100644 --- a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs @@ -667,7 +667,11 @@ async fn a_list_content_completion_is_read_as_reasoning_then_text() { assert_eq!(response.text(), "ALPHA BRAVO CHARLIE"); assert_eq!( - response.message.content.first().and_then(|block| block.as_thinking()), + response + .message + .content + .first() + .and_then(|block| block.as_thinking()), Some((".", None)) ); } From ac27805860d6a37ff2d2b24a21199d7728060a3d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:33:56 +0000 Subject: [PATCH 14/15] chore(openai): reword sse chunk drop comment Reworded the comment explaining why malformed SSE chunks are skipped, clarifying that a silent drop would hide a truncated answer rather than citing a past incident. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/openai/sse.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/openai/sse.rs b/crates/tinyinference-llm/src/providers/openai/sse.rs index 06388f4f..8fb51e8a 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse.rs @@ -724,8 +724,8 @@ impl SseState { Ok(chunk) => Some(chunk), Err(error) => { // Still skipped, but never silently: a dropped chunk loses - // whatever text it carried, which once cost every third - // answer its opening words. + // whatever text it carried, and a silent drop hides that the + // answer arrived truncated. tracing::warn!( provider = %self.provider, model = %self.model, From 3c4aa886c9348a71966ac1d6409ca6f17b04c29c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 11 Nov 2026 21:46:27 +0000 Subject: [PATCH 15/15] fix(openai): keep the skipped chunk's payload off the WARN line A chunk that fails to deserialize can carry answer or reasoning text. The WARN keeps provider, model and the serde error; the truncated payload moves to a separate DEBUG event. Review feedback on #150. Co-authored-by: Medulla --- .../src/providers/openai/sse.rs | 8 ++++-- .../src/providers/openai/sse_tests.rs | 26 ++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/openai/sse.rs b/crates/tinyinference-llm/src/providers/openai/sse.rs index 8fb51e8a..afc870f1 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse.rs @@ -725,14 +725,18 @@ impl SseState { Err(error) => { // Still skipped, but never silently: a dropped chunk loses // whatever text it carried, and a silent drop hides that the - // answer arrived truncated. + // answer arrived truncated. The payload can hold answer or + // reasoning text, so only DEBUG carries it, truncated. tracing::warn!( provider = %self.provider, model = %self.model, %error, - payload = %log_preview(payload), "openai sse: skipped a chunk that does not match the chunk schema" ); + tracing::debug!( + payload = %log_preview(payload), + "openai sse: skipped chunk payload" + ); None } }; diff --git a/crates/tinyinference-llm/src/providers/openai/sse_tests.rs b/crates/tinyinference-llm/src/providers/openai/sse_tests.rs index c055e802..beae8656 100644 --- a/crates/tinyinference-llm/src/providers/openai/sse_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/sse_tests.rs @@ -361,14 +361,16 @@ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogBuffer { } /// A chunk that still does not fit the chunk schema is skipped, as before, -/// but no longer silently: it is warned about with a truncated copy of the -/// payload, and the rest of the stream still arrives. +/// but no longer silently: it is warned about (provider, model, error), and the +/// rest of the stream still arrives. The payload can hold answer or reasoning +/// text, so it stays off the WARN line; a truncated copy is logged at DEBUG. #[tokio::test] async fn sse_stream_warns_about_a_chunk_it_cannot_read() { let logs = LogBuffer::default(); let subscriber = tracing_subscriber::fmt() .with_writer(logs.clone()) .with_ansi(false) + .with_max_level(tracing::Level::DEBUG) .finish(); let _guard = tracing::subscriber::set_default(subscriber); @@ -388,13 +390,19 @@ async fn sse_stream_warns_about_a_chunk_it_cannot_read() { assert_eq!(response.text(), "kept too"); let logs = logs.contents(); - assert!(logs.contains("WARN"), "{logs}"); - assert!( - logs.contains(r#"\"index\":\"zero\""#) || logs.contains(r#""index":"zero""#), - "{logs}" - ); + let line = |level: &str| { + logs.lines() + .find(|line| line.contains(level) && line.contains("openai sse: skipped")) + .unwrap_or_else(|| panic!("no {level} line: {logs}")) + }; + let warn = line("WARN"); + assert!(warn.contains("model=gpt-4.1-mini"), "{warn}"); + assert!(warn.contains("error="), "{warn}"); + assert!(!warn.contains("xxxx"), "payload on the WARN line: {warn}"); + let debug = line("DEBUG"); + assert!(debug.contains(r#""index":"zero""#), "{debug}"); assert!( - !logs.contains(&"x".repeat(4_000)), - "payload not truncated: {logs}" + !debug.contains(&"x".repeat(4_000)), + "payload not truncated: {debug}" ); }