Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions src/adapters/anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ function applyCacheControlToLastText(blocks: Array<Record<string, unknown>>, cc:
type PromptCachingOptions = {
maxExplicitBreakpoints?: number;
skipLastUser?: boolean;
/** The last system block is the caller's uncached per-turn tail; mark the block before it. */
uncachedSystemTail?: boolean;
};

/** Place explicit cache_control breakpoints on the built Anthropic body. */
Expand Down Expand Up @@ -158,7 +160,8 @@ function applyPromptCaching(
// 2. system
const system = body.system as Array<Record<string, unknown>> | undefined;
if (system && system.length > 0) {
applyCacheControlToLast(system, cc);
const i = system.length - (options.uncachedSystemTail && system.length > 1 ? 2 : 1);
system[i] = { ...system[i], cache_control: cc };
used++;
}
if (used >= explicitLimit || !messages) return;
Expand Down Expand Up @@ -759,7 +762,7 @@ function messagesToAnthropicFormat(
parsed: OcxParsedRequest,
toolNames: { toWire: (name: string) => string },
serverReplay?: ReadonlyMap<string, readonly Record<string, unknown>[]>,
): { system: string | undefined; messages: unknown[] } {
): { system: string[]; messages: unknown[] } {
// The client's server-tool exchanges go back ahead of the block they preceded. Their anchors are
// unique in the caller's history (replayedAnthropicServerBlocks drops repeated ones); each is used once.
const pendingReplay = new Map(serverReplay);
Expand Down Expand Up @@ -787,9 +790,14 @@ function messagesToAnthropicFormat(
tool => toolNames.toWire(namespacedToolName(tool.namespace, tool.name)),
);
const systemParts = [...(parsed.context.systemPrompt ?? []), ...(toolCatalogNudge ? [toolCatalogNudge] : [])];
const system = systemParts.length
? identifyRoutedModel(systemParts.join("\n\n"), parsed.modelId) || undefined
: undefined;
// End a block at the caller's cached prefix so its breakpoint skips the per-turn tail.
const prefix = parsed._anthropicSystemCachePrefix
? identifyRoutedModel(parsed._anthropicSystemCachePrefix, parsed.modelId) : undefined;
const head = prefix && systemParts[0] ? identifyRoutedModel(systemParts[0], parsed.modelId) : systemParts[0];
const texts = prefix && head?.startsWith(`${prefix}\n\n`)
? [prefix, [head.slice(prefix.length + 2), ...systemParts.slice(1)].join("\n\n")]
: [systemParts.join("\n\n")];
const system = texts.map(text => identifyRoutedModel(text, parsed.modelId)).filter(Boolean);
const messages: unknown[] = [];

for (let i = 0; i < parsed.context.messages.length; i++) {
Expand Down Expand Up @@ -1078,10 +1086,10 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti
// Claude OAuth (Pro/Max) requires the first system block to be the Claude Code identity.
body.system = [
{ type: "text", text: CLAUDE_CODE_SYSTEM_INSTRUCTION },
...(system ? [{ type: "text", text: system }] : []),
...system.map(text => ({ type: "text", text })),
];
} else if (system) {
body.system = [{ type: "text", text: system }];
} else if (system.length > 0) {
body.system = system.map(text => ({ type: "text", text }));
}
if (tools) body.tools = tools;
if (parsed.options.temperature !== undefined) body.temperature = parsed.options.temperature;
Expand Down Expand Up @@ -1277,6 +1285,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti
applyPromptCaching(body, cc, {
maxExplicitBreakpoints: explicitLimit,
skipLastUser: !!automaticPromptCaching,
uncachedSystemTail: system.length > 1,
});
enforceCacheControlLimit(body, explicitLimit);
normalizeTtlOrdering(body);
Expand Down
12 changes: 12 additions & 0 deletions src/claude/inbound-content-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,18 @@ export function systemToInstructions(system: unknown): string | undefined {
return undefined;
}

/**
* The `instructions` prefix ending at the caller's last cache-marked system block, when blocks
* follow it. Claude Code ends its system with a per-turn env block; joining everything into one
* Anthropic system block moved the breakpoint onto those changing bytes, so system and every
* later breakpoint missed on each turn. The Anthropic adapter splits its system here instead.
*/
export function systemCachePrefix(system: unknown): string | undefined {
if (!Array.isArray(system)) return undefined;
const last = system.findLastIndex(block => isRec(block) && block.cache_control != null);
return last >= 0 && last < system.length - 1 ? systemToInstructions(system.slice(0, last + 1)) : undefined;
}

export function toolsToResponses(tools: unknown): Rec[] | undefined {
if (!Array.isArray(tools) || tools.length === 0) return undefined;
// The shared hosted-search lane cannot enforce these constraints on every destination or
Expand Down
3 changes: 2 additions & 1 deletion src/server/claude-messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ import { stripOneMillionMarker } from "../claude/context-windows";
import { captureAdmissionPolicy } from "./capture-admission-policy";
import type { CapturedAdmissionPolicy } from "../types/admission-policy";
import { captureClaudeInbound } from "../claude/inbound-debug";
import { declaredAnthropicServerTools, replayedAnthropicServerBlocks } from "../claude/inbound-content-options";
import { declaredAnthropicServerTools, replayedAnthropicServerBlocks, systemCachePrefix } from "../claude/inbound-content-options";
import { claudeCodeForIngress } from "../claude/intercept/model-bindings";
import { classifyInterceptClient } from "../claude/intercept/client-class";
import { analyzeClaudeCompatibility, isClaudeCompatibilityMode } from "../claude/compatibility";
Expand Down Expand Up @@ -1286,6 +1286,7 @@ async function handleClaudeMessagesWithBudget(
// Out of body like the replay above: the core hands these to an Anthropic destination only.
anthropicServerTools: declaredAnthropicServerTools(anthropicBody),
anthropicServerReplay: replayedAnthropicServerBlocks(anthropicBody),
anthropicSystemCachePrefix: isRec(anthropicBody) ? systemCachePrefix(anthropicBody.system) : undefined,
// Preserve the original catalog through late fallback; only the physical adapter projects it.
anthropicToolLoadingSource: cacheKeySource === "metadata" && isRec(anthropicBody)
&& Array.isArray(cc?.appendOnlyToolLoadingModels) && cc.appendOnlyToolLoadingModels.length > 0
Expand Down
2 changes: 2 additions & 0 deletions src/server/responses/core-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ export interface HandleResponsesOptions {
anthropicServerTools?: readonly Record<string, unknown>[];
/** The Claude client's replayable server-tool exchanges (`replayedAnthropicServerBlocks`); same scope. */
anthropicServerReplay?: ReadonlyMap<string, readonly Record<string, unknown>[]>;
/** The caller's cached system prefix (`systemCachePrefix`); same scope. */
anthropicSystemCachePrefix?: string;
/** Metadata-qualified Messages discovery evidence, never caller JSON or a preselected destination. */
anthropicToolLoadingSource?: Readonly<Record<string, unknown>>;
/** Droid's per-request effort default; each concrete combo or policy target applies it only if its ladder allows it. */
Expand Down
1 change: 1 addition & 0 deletions src/server/responses/request-prepare.ts
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@ export async function prepareResponsesRequest(
parsed._nativeReasoningMint = options.nativeReasoningMint;
parsed._anthropicServerTools = options.anthropicServerTools;
parsed._anthropicServerReplay = options.anthropicServerReplay;
parsed._anthropicSystemCachePrefix = options.anthropicSystemCachePrefix;
if (parsed.options.textFormat?.type === "json_schema") parsed._structuredOutputConformance = true;
}
parsed._promptCacheKeyIsSharedCohort = options.promptCacheKeyIsSharedCohort;
Expand Down
5 changes: 5 additions & 0 deletions src/types/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ export interface OcxParsedRequest {
* followed them (`call:<tool_use id>` / `text:<text>`). Same audience as `_anthropicServerTools`.
*/
_anthropicServerReplay?: ReadonlyMap<string, readonly Record<string, unknown>[]>;
/**
* The caller's `instructions` prefix through its last cache-marked system block. The anthropic
* adapter ends a system block there so the breakpoint stays off the per-turn tail.
*/
_anthropicSystemCachePrefix?: string;
/**
* Boundary between replayed history and this turn's newly appended input. Usually the
* items the proxy restored from local previous_response_id state; also set when the
Expand Down
Loading
Loading