Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs-site/src/content/docs/reference/adapters.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@ transport; it does not infer subscription attribution from the inbound protocol.
Adjacent text blocks share inline-tag parsing; typed thinking resets it.
Unsupported or malformed array blocks return an upstream error. Other hosts
keep their existing behavior. This does not add Mistral signature replay.
- Native `/v1/chat/completions` passthrough to `https://api.mistral.ai/v1` (key auth)
converts the same arrays into Chat strings: thinking arrives as `reasoning_content`
and text as `content`, streamed in source order. A non-streaming reply joins each
into `message.reasoning_content` and `message.content`. Unsupported blocks return
an upstream error.
- Suppresses bare `<tool_call>` text when it duplicates a structured call, and collapses two
immediately adjacent identical blocks when exactly one structured call agrees with their function
and input. A doubled `input` is reduced to one copy, joined either directly or by one newline,
Expand Down
3 changes: 2 additions & 1 deletion scripts/test-layout/layout-chat.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
"client-error-body.test.ts": "lib",
"chat-error-body-normalization.test.ts": "responses",
"chat-error-collector-verdicts.test.ts": "responses",
"server-chat-error-body-evidence.test.ts": "server"
"server-chat-error-body-evidence.test.ts": "server",
"chat-native-mistral-content.test.ts": "responses"
}
98 changes: 93 additions & 5 deletions src/server/chat-native-sse.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { chatContentEvents } from "../adapters/openai-chat/response-events";
import { ownedChatErrorPayload, canonicalSseErrorBlock } from "../lib/client-error-body";
import { legacyFailureVerdict, type FailureVerdictSink } from "../chat/failure-verdict";
import { chatCompletionsErrorBody, type ChatCompletionsCollectorSource } from "../chat/outbound";
Expand Down Expand Up @@ -114,6 +115,82 @@ function normalizedChunk(value: Rec, requestedModel: string): Rec {
};
}

type ChatStringField = "content" | "reasoning_content";

/** Mistral typed blocks as Chat string runs in source order; null when a block is unsupported. */
function mistralContentRuns(content: unknown[], baseUrl: string): { field: ChatStringField; text: string }[] | null {
const runs: { field: ChatStringField; text: string }[] = [];
for (const event of chatContentEvents(content, baseUrl)) {
if (event.type === "error") return null;
const field = event.type === "text_delta" ? "content" : "reasoning_content";
const last = runs.at(-1);
if (last?.field === field) last.text += event.text;
else runs.push({ field, text: event.text });
}
return runs;
}

/**
* Mistral reasoning models stream `delta.content` as typed `thinking`/`text` blocks, while native
* Chat clients parse strings. Each run becomes its own chunk in source order, thinking as
* `reasoning_content` and text as `content`. `role` rides the first chunk; the last keeps the
* remaining delta fields, `finish_reason` and `usage`. Every emitted choice carries its index.
* Chunks without typed content pass through unchanged.
*/
export function mistralTypedContentChunks(value: Rec, baseUrl: string): Rec[] | null {
const choices = Array.isArray(value.choices) ? value.choices : [];
if (!choices.some(choice => isRec(choice) && isRec(choice.delta) && Array.isArray(choice.delta.content))) return [value];
const { choices: _choices, usage, ...chunk } = value;
const split: unknown[] = [];
for (const [position, choice] of choices.entries()) {
if (!isRec(choice) || !isRec(choice.delta) || !Array.isArray(choice.delta.content)) {
split.push(isRec(choice) && typeof choice.index !== "number" ? { ...choice, index: position } : choice);
continue;
}
const runs = mistralContentRuns(choice.delta.content, baseUrl);
if (!runs) return null;
const { content: _content, role, ...delta } = choice.delta;
const index = typeof choice.index === "number" ? choice.index : position;
const first = role === undefined ? {} : { role };
const last = runs.pop() ?? { field: "content", text: "" };
runs.forEach((run, ordinal) => split.push({
index, delta: { ...(ordinal === 0 ? first : {}), [run.field]: run.text }, finish_reason: null,
}));
const prior = typeof delta[last.field] === "string" ? delta[last.field] as string : "";
split.push({ ...choice, index, delta: { ...(runs.length === 0 ? first : {}), ...delta, [last.field]: prior + last.text } });
}
return split.map((choice, position) => ({
...chunk,
choices: [choice],
...(position === split.length - 1 && usage !== undefined ? { usage } : {}),
}));
}

/** Buffered form of {@link mistralTypedContentChunks}: text joins `content`, thinking joins `reasoning_content`. */
export function mistralTypedContentCompletion(value: Rec, baseUrl: string): Rec | null {
if (!Array.isArray(value.choices)) return value;
let changed = false;
const choices: unknown[] = [];
for (const choice of value.choices) {
if (!isRec(choice) || !isRec(choice.message) || !Array.isArray(choice.message.content)) {
choices.push(choice);
continue;
}
const runs = mistralContentRuns(choice.message.content, baseUrl);
if (!runs) return null;
const joined = (field: ChatStringField) => runs.filter(run => run.field === field).map(run => run.text).join("");
const reasoning = joined("reasoning_content");
const prior = typeof choice.message.reasoning_content === "string" ? choice.message.reasoning_content : "";
choices.push({
...choice,
// `null` for a thinking- or tool-only reply, matching the streamed collector's message.
message: { ...choice.message, content: joined("content") || null, ...(reasoning ? { reasoning_content: prior + reasoning } : {}) },
});
changed = true;
}
return changed ? { ...value, choices } : value;
}

export function jsonCompletionSse(value: Rec, requestedModel: string, budget?: TranslatorBudget): string {
const id = typeof value.id === "string" ? value.id : `chatcmpl-${crypto.randomUUID().replace(/-/g, "").slice(0, 24)}`;
const created = typeof value.created === "number" ? value.created : Math.floor(Date.now() / 1000);
Expand Down Expand Up @@ -188,6 +265,8 @@ interface NativeChatSseOptions {
* `stallTimeoutSec` then resolves to disabled so a slow local model is not cut mid-turn.
*/
localUpstream?: boolean;
/** Canonical Mistral Chat base URL; set only for that destination, it converts typed content blocks. */
mistralBaseUrl?: string;
clientDeclaredEmptyToolCatalog?: boolean;
declaredLegacyFunctionNames?: ReadonlySet<string>;
onFirstOutput?: () => void;
Expand Down Expand Up @@ -397,6 +476,13 @@ export function nativeChatSse(
fail(controller, UNDECLARED_CHAT_TOOL_MESSAGE, UNDECLARED_TOOL_CALL_ERROR_CODE);
return;
}
const normalized = normalizedChunk(parsed, options.requestedModel);
// Replay capture above keeps reading the upstream chunk; only the client copy is converted.
const clientChunks = options.mistralBaseUrl ? mistralTypedContentChunks(normalized, options.mistralBaseUrl) : [normalized];
if (!clientChunks) {
fail(controller, "unsupported Mistral content block", "upstream_sse_invalid");
return;
}
const choices = Array.isArray(parsed.choices) ? parsed.choices : [];
const finished = choices.some(choice => isRec(choice) && typeof choice.finish_reason === "string" && choice.finish_reason.length > 0);
if (hasPendingToolName() && choices.some((choice, position) => isRec(choice)
Expand All @@ -406,7 +492,7 @@ export function nativeChatSse(
return;
}
if (finished) sawFinish = true;
const hasText = choices.some(choice => {
const hasText = clientChunks.flatMap(chunk => Array.isArray(chunk.choices) ? chunk.choices : []).some(choice => {
if (!isRec(choice) || !isRec(choice.delta)) return false;
const delta = choice.delta;
return (typeof delta.content === "string" && delta.content.length > 0)
Expand Down Expand Up @@ -445,10 +531,12 @@ export function nativeChatSse(
options.collectorSource?.retain(JSON.stringify(payload), parsed);
relay(controller, (eventName === "error" ? "event: error\n" : "") + canonicalSseErrorBlock(payload));
} else {
const normalized = normalizedChunk(parsed, options.requestedModel);
const choiceError = choices.some(choice => isRec(choice) && choice.error !== undefined && choice.error !== null);
relay(controller, choiceError ? canonicalSseErrorBlock(normalizeChatChoiceErrors(normalized))
: replaceSseDataPayload(block, JSON.stringify(normalized)) + delimiter);
for (const chunk of clientChunks) {
const chunkChoices = Array.isArray(chunk.choices) ? chunk.choices : [];
const choiceError = chunkChoices.some(choice => isRec(choice) && choice.error !== undefined && choice.error !== null);
relay(controller, choiceError ? canonicalSseErrorBlock(normalizeChatChoiceErrors(chunk))
: replaceSseDataPayload(block, JSON.stringify(chunk)) + delimiter);
}
}
};

Expand Down
16 changes: 12 additions & 4 deletions src/server/chat-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,10 @@ import {
recordAttemptCredentialSource,
type RequestLogContext,
} from "./request-log";
import { chatResponseHasUndeclaredToolCalls, UNDECLARED_CHAT_TOOL_MESSAGE, jsonCompletionSse, nativeChatSse, structuredError, usageFromChat } from "./chat-native-sse";
import {
chatResponseHasUndeclaredToolCalls, UNDECLARED_CHAT_TOOL_MESSAGE, jsonCompletionSse, mistralTypedContentCompletion, nativeChatSse,
structuredError, usageFromChat,
} from "./chat-native-sse";
import { beginInferenceAttempt, type InferenceAttempt } from "./inference/attempt";
import { createFinalRequestLog } from "./inference/final-log";
import { registerTurn, unregisterTurn } from "./lifecycle";
Expand All @@ -92,7 +95,7 @@ import type { ComboProtocolSource } from "./responses/core-combo-native";
import type { ProtocolEnvelope } from "../protocols/envelope";
import { protocolTraceSuppressed } from "../protocols/trace";
import { nativeChatReplayScope, bindNativeChatReplayScope, nativeChatReasoningCapture } from "./chat-native-reasoning";
import { resolveMistralTransport } from "../providers/mistral-transport";
import { isCanonicalMistralChat, resolveMistralTransport } from "../providers/mistral-transport";

export { isNativeChatRouteEligible, nativeChatDeclineReason } from "./chat-native-eligibility";

Expand Down Expand Up @@ -805,6 +808,7 @@ export async function runNativeChatAttempt(

const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
const reasoningCapture = nativeChatReasoningCapture(activeProvider, route.modelId, replayScope, translatorBudget);
const mistralBaseUrl = isCanonicalMistralChat(activeProvider) ? activeProvider.baseUrl : undefined;
if (contentType.includes("text/event-stream") && response.body) {
if (requestedStream) transferTurnToStream();
let terminalStatus: number | undefined;
Expand Down Expand Up @@ -845,6 +849,7 @@ export async function runNativeChatAttempt(
signal: upstream.signal,
stallTimeoutSec: config.stallTimeoutSec,
localUpstream,
mistralBaseUrl,
clientDeclaredEmptyToolCatalog,
declaredLegacyFunctionNames,
onFirstOutput,
Expand Down Expand Up @@ -956,11 +961,14 @@ export async function runNativeChatAttempt(
if (clientDeclaredEmptyToolCatalog && chatResponseHasUndeclaredToolCalls(completion, declaredLegacyFunctionNames)) {
return failUpstream(502, UNDECLARED_CHAT_TOOL_MESSAGE, "upstream_error", UNDECLARED_TOOL_CALL_ERROR_CODE);
}
// Replay capture above keeps reading the upstream body; only the client copy is converted.
const clientCompletion = mistralBaseUrl ? mistralTypedContentCompletion(completion, mistralBaseUrl) : completion;
if (!clientCompletion) return failUpstream(502, "unsupported Mistral content block", "upstream_error");
onFirstOutput?.();
try {
const serialized = requestedStream
? jsonCompletionSse(completion, requestedModel, translatorBudget)
: JSON.stringify(completion);
? jsonCompletionSse(clientCompletion, requestedModel, translatorBudget)
: JSON.stringify(clientCompletion);
if (!requestedStream) translatorBudget.chargeRetained(Buffer.byteLength(serialized) * 2, { kind: "live_transient" });
finishLog(200);
return new Response(serialized, {
Expand Down
3 changes: 2 additions & 1 deletion structure/providers/chat-compat.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@ Shared parsing and streaming follow the [request-copy](../transports/byte-accoun
## Reasoning and tool-result compatibility

For `api.mistral.ai`, `src/adapters/openai-chat/response-events.ts` preserves typed text/thinking arrays in both response paths and rejects unsupported blocks. Inline-tag parsing crosses adjacent text blocks but resets at typed reasoning. It adds no signature replay.
Native Chat passthrough reuses that converter for canonical key-auth Mistral (`isCanonicalMistralChat`), because its clients parse Chat strings. `src/server/chat-native-sse.ts` splits a typed stream delta into one chunk per run in source order, thinking as `reasoning_content` and text as `content`; the last chunk keeps the remaining delta fields, `finish_reason` and `usage`. A buffered `message.content` joins text into `content` and thinking into `reasoning_content`. An unsupported block fails the turn with a 502. Native reasoning replay capture still reads the upstream chunk, so request history is unchanged.
On that same host, `src/adapters/openai-chat/messages.ts` reconstructs historical thinking as native `content` chunks before assistant text, for every hosted model family. An explicit `preserveReasoningContentModels` list, including `[]`, still controls enrollment; the default preserves supplied thinking. The shared prompt projection counts that thinking. Replay can recover actual text from the existing scoped tool-call cache, but never invents placeholders or forwards generic IR signatures. Original chunk metadata and exact chunk boundaries are not retained by this text-only reconstruction. Native Chat passthrough and other destinations retain their existing behavior.
Tests: `tests/adapters/openai/openai-chat-mistral-content.test.ts`; byte budgets retain their existing limits.
Tests: `tests/adapters/openai/openai-chat-mistral-content.test.ts`, `tests/responses/chat-native-mistral-content.test.ts`; byte budgets retain their existing limits.

### Inline think-tag recovery

Expand Down
1 change: 1 addition & 0 deletions tests/fixtures/test-layout-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,7 @@
"chat-media-translation.test.ts": "responses", "chat-native-developer-role.test.ts": "responses",
"chat-native-reasoning-replay.test.ts": "responses",
"chat-native-reasoning-scope.test.ts": "responses",
"chat-native-mistral-content.test.ts": "responses",
"chat-native-image-normalization.test.ts": "responses", "chat-native-spend.test.ts": "responses",
"chat-refusal-scope.test.ts": "responses", "chat-refusal.test.ts": "responses",
"chat-responses-control-integration.test.ts": "responses", "chat-responses-control-scope.test.ts": "responses",
Expand Down
Loading
Loading