Skip to content

fix(translation): preserve Responses refusal streams #1300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dagardner-nv merged 1 commit into from
Nov 10, 2026
Merged

fix(translation): preserve Responses refusal streams #1300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 42 additions & 6 deletions crates/switchyard-translation/src/codecs/responses/stream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,13 @@ fn decode_responses_stream(
model: state.model.clone(),
}]
}
Some("response.output_text.delta") => event
Some("response.output_text.delta" | "response.refusal.delta") => event
.get("delta")
.and_then(Value::as_str)
.map(|text| {
if event_type == Some("response.refusal.delta") {
state.stop_reason = Some("refusal".to_string());
}
let index = event
.get("output_index")
.and_then(Value::as_u64)
Expand All @@ -181,6 +184,31 @@ fn decode_responses_stream(
}]
})
.unwrap_or_default(),
Some("response.refusal.done") => {
state.stop_reason = Some("refusal".to_string());
let Some(text) = event.get("refusal").and_then(Value::as_str) else {
return Vec::new();
};
let index = event
.get("output_index")
.and_then(Value::as_u64)
.unwrap_or(0) as usize;
let content_index = event
.get("content_index")
.and_then(Value::as_u64)
.unwrap_or(0) as usize;
let decoded = state
.decoded_response_text
.entry(index)
.or_default()
.entry(content_index)
.or_default();
match snapshot_suffix(decoded, text) {
Ok(Some(text)) => vec![LlmResponseChunk::TextDelta { index, text }],
Ok(None) => Vec::new(),
Err(error) => vec![error],
}
}
Some("response.reasoning_text.delta") | Some("response.reasoning_summary_text.delta") => {
event
.get("delta")
Expand Down Expand Up @@ -314,7 +342,10 @@ fn decode_responses_stream(
// use for such output; the stream must too, or stop-reason-driven tool loops
// (Anthropic `tool_use`, Chat `tool_calls`) stop without running the tool.
// Carries the Anthropic spelling because every encoder already maps it.
let reason = state.decoded_tool_call.then(|| "tool_use".to_string());
let reason = state
.stop_reason
.clone()
.or_else(|| state.decoded_tool_call.then(|| "tool_use".to_string()));
out.push(LlmResponseChunk::MessageStop { reason });
out
}
Expand Down Expand Up @@ -751,10 +782,15 @@ fn decode_responses_completed_item(
let mut out = Vec::new();
if let Some(content) = item.get("content").and_then(Value::as_array) {
for (content_index, part) in content.iter().enumerate() {
if part.get("type").and_then(Value::as_str) != Some("output_text") {
continue;
}
if let Some(text) = part.get("text").and_then(Value::as_str) {
let text = match part.get("type").and_then(Value::as_str) {
Some("output_text") => part.get("text"),
Some("refusal") => {
state.stop_reason = Some("refusal".to_string());
part.get("refusal")
}
_ => continue,
};
if let Some(text) = text.and_then(Value::as_str) {
let decoded = state
.decoded_response_text
.entry(index)
Expand Down
131 changes: 92 additions & 39 deletions crates/switchyard-translation/tests/stream_translation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,60 +589,113 @@ fn openai_chat_stream_event_translates_to_anthropic_message_events() -> TestResu
Ok(())
}

// Verifies Chat Completions refusal deltas remain visible and terminate as refusals.
// Refusal text is emitted once, whether supplied as deltas or a completed snapshot.
#[test]
fn openai_chat_refusal_stream_translates_to_anthropic() -> TestResult {
fn refusal_stream_translates_to_anthropic() -> TestResult {
let engine = TranslationEngine::default();
let mut state =
StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::AnthropicMessages);
let refusal = json!({
let refusal = "I cannot help with that request.";
let chat_delta = json!({
"id": "chatcmpl-refusal",
"object": "chat.completion.chunk",
"model": "gpt-4o",
"choices": [{
"index": 0,
"delta": {"refusal": "I cannot help with that request."},
"delta": {"refusal": refusal},
"finish_reason": null
}]
});

let mut events = engine.translate_event(
&mut state,
WireFormat::OpenAiChat,
WireFormat::AnthropicMessages,
&refusal,
)?;
let text_delta = events
.iter()
.find(|event| event["type"] == "content_block_delta")
.ok_or("missing refusal text delta")?;
assert_eq!(
text_delta["delta"]["text"],
"I cannot help with that request."
);

let terminal = json!({
let chat_terminal = json!({
"id": "chatcmpl-refusal",
"object": "chat.completion.chunk",
"model": "gpt-4o",
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
});
events.extend(engine.translate_event(
&mut state,
WireFormat::OpenAiChat,
WireFormat::AnthropicMessages,
&terminal,
)?);
events.extend(engine.finish_stream(&mut state, WireFormat::AnthropicMessages)?);
let message_delta = events
.iter()
.find(|event| event["type"] == "message_delta")
.ok_or("missing Anthropic terminal delta")?;
assert_eq!(message_delta["delta"]["stop_reason"], "refusal");
assert_eq!(
message_delta["delta"]["stop_details"],
json!({"type": "refusal", "category": null, "explanation": null})
);
let part = json!({"type": "refusal", "refusal": refusal});
let message = json!({"id": "msg_refusal", "type": "message", "role": "assistant",
"status": "completed", "content": [part]});
let response = json!({"id": "resp_refusal", "model": "fixture",
"status": "completed", "output": [message]});
let completed = json!({"type": "response.completed", "response": response});
let refusal_done = json!({"type": "response.refusal.done", "item_id": "msg_refusal",
"output_index": 0, "content_index": 0, "refusal": refusal});

for (case, source, input, expected_fragments) in [
(
"chat_deltas",
WireFormat::OpenAiChat,
vec![chat_delta, chat_terminal],
vec![refusal],
),
(
"responses_snapshot_only",
WireFormat::OpenAiResponses,
vec![completed.clone()],
vec![refusal],
),
(
"responses_refusal_done_only",
WireFormat::OpenAiResponses,
vec![
refusal_done.clone(),
json!({"type": "response.completed", "response": {}}),
],
vec![refusal],
),
(
"responses_full_lifecycle",
WireFormat::OpenAiResponses,
vec![
json!({"type": "response.created", "response": {"id": "resp_refusal",
"model": "fixture", "status": "in_progress", "output": []}}),
json!({"type": "response.output_item.added", "output_index": 0,
"item": {"id": "msg_refusal", "type": "message", "role": "assistant",
"status": "in_progress", "content": []}}),
json!({"type": "response.content_part.added", "item_id": "msg_refusal",
"output_index": 0, "content_index": 0,
"part": {"type": "refusal", "refusal": ""}}),
json!({"type": "response.refusal.delta", "item_id": "msg_refusal",
"output_index": 0, "content_index": 0, "delta": "I cannot "}),
json!({"type": "response.refusal.delta", "item_id": "msg_refusal",
"output_index": 0, "content_index": 0, "delta": "help with that request."}),
refusal_done,
json!({"type": "response.content_part.done", "item_id": "msg_refusal",
"output_index": 0, "content_index": 0, "part": part}),
json!({"type": "response.output_item.done", "output_index": 0, "item": message}),
completed,
],
vec!["I cannot ", "help with that request."],
),
] {
let target = WireFormat::AnthropicMessages;
let mut decoder = StreamTranslationState::new(source, target);
let mut encoder = StreamTranslationState::new(source, target);
let mut events = Vec::new();
for event in input {
let decoded = engine.decode_stream_event(&mut decoder, source, event)?;
events.extend(engine.encode_stream_event(&mut encoder, target, decoded)?);
}
events.extend(engine.finish_stream(&mut encoder, target)?);
let fragments: Vec<_> = events
.iter()
.filter(|event| event["type"] == "content_block_delta")
.filter_map(|event| event["delta"]["text"].as_str())
.collect();
assert_eq!(fragments, expected_fragments, "{case}");
let terminal_deltas: Vec<_> = events
.iter()
.filter(|event| event["type"] == "message_delta")
.map(|event| event["delta"].clone())
.collect();
assert_eq!(
terminal_deltas,
vec![json!({
"stop_reason": "refusal",
"stop_sequence": null,
"stop_details": {"type": "refusal", "category": null, "explanation": null}
})],
"{case}"
);
}
Ok(())
}

Expand Down
Loading