From f3972c963050453766c979b04014cfc6dbdf8f0f Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 4 Nov 2026 10:26:17 -0800 Subject: [PATCH 1/4] feat(observability): account for upstream attempts and retry costs Signed-off-by: Bryan Bednarski --- crates/libsy-llm-client/src/attempt.rs | 531 ++++++++++++++++++ crates/libsy-llm-client/src/client.rs | 172 ++++-- crates/libsy-llm-client/src/lib.rs | 1 + crates/libsy-llm-client/src/observability.rs | 4 +- crates/libsy-llm-client/src/run.rs | 19 +- .../libsy-llm-client/tests/observability.rs | 152 +++++ crates/switchyard-nemo-relay-plugin/README.md | 23 + .../examples/attempt_costs.py | 93 +++ .../scripts/test_attempt_accounting.py | 181 ++++++ .../src/attempt_tests.rs | 386 +++++++++++++ .../src/span_capture.rs | 115 +++- .../src/codecs/anthropic/buffered.rs | 2 +- .../src/codecs/anthropic/mod.rs | 2 + .../switchyard-translation/src/codecs/mod.rs | 13 + .../src/codecs/openai_chat/mod.rs | 2 + .../src/codecs/responses/buffered.rs | 2 +- .../src/codecs/responses/mod.rs | 2 + .../src/codecs/responses/stream.rs | 46 +- crates/switchyard-translation/src/lib.rs | 1 + .../tests/failure_usage.rs | 55 ++ docs/integrations/nemo_relay.md | 76 ++- .../req3_attempt_accounting_implementation.md | 84 +++ 22 files changed, 1833 insertions(+), 129 deletions(-) create mode 100644 crates/libsy-llm-client/src/attempt.rs create mode 100644 crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py create mode 100644 crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py create mode 100644 crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs create mode 100644 crates/switchyard-translation/tests/failure_usage.rs create mode 100644 docs/internal/req3_attempt_accounting_implementation.md diff --git a/crates/libsy-llm-client/src/attempt.rs b/crates/libsy-llm-client/src/attempt.rs new file mode 100644 index 0000000000..ea3fa78dc2 --- /dev/null +++ b/crates/libsy-llm-client/src/attempt.rs @@ -0,0 +1,531 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Accounting retained until an HTTP attempt's response is consumed or abandoned. + +use std::pin::Pin; +use std::task::{Context, Poll}; + +use futures::Stream; +use http::HeaderMap; +use switchyard_protocol::{ + LlmClientError, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ModelId, Usage, + WireFormat, +}; +use tracing::Span; + +use crate::observability::{llm_client_error_type, record_gen_ai_usage}; + +pub(crate) struct AttemptGuard { + span: Span, + usage: Usage, + accounting: bool, + final_usage: bool, + model_finished: bool, + finished: bool, +} + +impl AttemptGuard { + pub(crate) fn new( + model: &ModelId, + format: WireFormat, + ordinal: u64, + max_attempts: u64, + accounting: bool, + ) -> Self { + if accounting { + Span::current().record("switchyard.accounting.unit", "summary"); + Span::current().record("switchyard.accounting.attempts", ordinal); + } + let span = tracing::debug_span!(target: "libsy", "libsy.upstream_attempt", + model = %model, gen_ai.request.model = %model, wire_format = %format, + attempt = ordinal, max_attempts, retry = ordinal > 1, + openinference.span.kind = "CHAIN", + outcome = tracing::field::Empty, status_code = tracing::field::Empty, + will_retry = tracing::field::Empty, retry_delay_ms = tracing::field::Empty, + error.type = tracing::field::Empty, otel.status_code = tracing::field::Empty, + gen_ai.response.id = tracing::field::Empty, gen_ai.response.model = tracing::field::Empty, + gen_ai.usage.input_tokens = tracing::field::Empty, + gen_ai.usage.output_tokens = tracing::field::Empty, + gen_ai.usage.cache_read.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty, + gen_ai.usage.reasoning.output_tokens = tracing::field::Empty, + switchyard.usage.total_tokens = tracing::field::Empty, + switchyard.usage.provider_cost_usd = tracing::field::Empty, + switchyard.accounting.charge_source = tracing::field::Empty, + switchyard.accounting.usage_status = tracing::field::Empty, + switchyard.accounting.unit = if accounting { "completion_attempt" } else { "diagnostic" }, + ); + Self { + span, + usage: Usage::default(), + accounting, + final_usage: false, + model_finished: false, + finished: false, + } + } + + pub(crate) fn span(&self) -> &Span { + &self.span + } + + pub(crate) fn headers(&mut self, headers: &HeaderMap) { + if let Some(cost) = headers + .get("x-litellm-response-cost") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| value.is_finite() && *value >= 0.0) + { + self.usage.provider_cost = Some(cost); + self.span + .record("switchyard.accounting.charge_source", "header"); + } + } + + pub(crate) fn record_usage(&mut self, usage: &Usage, final_report: bool) { + macro_rules! update { + ($($field:ident),+ $(,)?) => {$(if usage.$field.is_some() { self.usage.$field = usage.$field; })+}; + } + update!(input_tokens, output_tokens, total_tokens, reasoning_tokens); + if let Some(cache) = &usage.cache { + let target = self.usage.cache.get_or_insert_with(Default::default); + if cache.cached_input_tokens.is_some() { + target.cached_input_tokens = cache.cached_input_tokens; + } + if cache.cache_creation_input_tokens.is_some() { + target.cache_creation_input_tokens = cache.cache_creation_input_tokens; + } + if cache.cache_creation_1h_input_tokens.is_some() { + target.cache_creation_1h_input_tokens = cache.cache_creation_1h_input_tokens; + } + } + if let Some(cost) = usage + .provider_cost + .filter(|value| value.is_finite() && *value >= 0.0) + { + self.usage.provider_cost = Some(cost); + self.span + .record("switchyard.accounting.charge_source", "provider"); + } + self.final_usage |= final_report; + } + + pub(crate) fn response(&mut self, body: &serde_json::Value, format: WireFormat) { + if let Some(id) = body.get("id").and_then(serde_json::Value::as_str) { + self.span.record("gen_ai.response.id", id); + } + if let Some(model) = body.get("model").and_then(serde_json::Value::as_str) { + self.span.record("gen_ai.response.model", model); + } + self.record_usage(&switchyard_translation::response_usage(body, format), true); + } + + pub(crate) fn observe( + &mut self, + item: &Result, + ) -> bool { + let event = match item { + Ok(event) => event, + Err(error) => { + self.error(error); + return true; + } + }; + self.observe_usage(event); + for chunk in event.normalized() { + match chunk { + LlmResponseChunk::MessageStart { id, model } => { + if let Some(id) = id { + self.span.record("gen_ai.response.id", id.as_str()); + } + if let Some(model) = model { + self.span.record("gen_ai.response.model", model.as_str()); + } + } + LlmResponseChunk::MessageStop { .. } => self.model_finished = true, + LlmResponseChunk::StreamError { .. } => { + self.finish("error", Some("provider_stream")); + return true; + } + LlmResponseChunk::DecodeError { .. } => { + self.finish("error", Some("response_translation")); + return true; + } + _ => (), + } + } + false + } + + pub(crate) fn observe_usage(&mut self, event: &LlmResponseStreamEvent) { + // One provider event can contain both its final usage and a terminal error. + let terminal_report = event.preservation().is_some_and(|raw| { + matches!( + raw.raw().get("type").and_then(serde_json::Value::as_str), + Some("response.completed" | "response.failed" | "response.incomplete") + ) + }); + for chunk in event.normalized() { + if let LlmResponseChunk::Usage(usage) = chunk { + self.record_usage(usage, terminal_report); + } + } + } + + pub(crate) fn error(&mut self, error: &LlmClientError) { + self.finish("error", Some(llm_client_error_type(error).as_ref())); + } + + pub(crate) fn success(&mut self) { + self.finish("ok", None); + } + + fn finish(&mut self, outcome: &str, error: Option<&str>) { + if self.finished { + return; + } + self.finished = true; + self.span.record("outcome", outcome); + self.span.record( + "otel.status_code", + if outcome == "ok" { "OK" } else { "ERROR" }, + ); + if let Some(error) = error { + self.span.record("error.type", error); + } + if !self.accounting { + return; + } + record_gen_ai_usage(&self.span, &self.usage); + if let Some(total) = self.usage.total_tokens { + self.span.record("switchyard.usage.total_tokens", total); + } + if let Some(cost) = self.usage.provider_cost { + self.span.record("switchyard.usage.provider_cost_usd", cost); + } + let known = self.usage.input_tokens.is_some() + || self.usage.output_tokens.is_some() + || self.usage.cached_input_tokens().is_some() + || self.usage.cache_creation_input_tokens().is_some() + || self + .usage + .cache + .as_ref() + .is_some_and(|cache| cache.cache_creation_1h_input_tokens.is_some()) + || self.usage.total_tokens.is_some() + || self.usage.reasoning_tokens.is_some(); + let complete = self.final_usage + && self.usage.input_tokens.is_some() + && self.usage.output_tokens.is_some(); + self.span.record( + "switchyard.accounting.usage_status", + if complete { + "complete" + } else if known { + "partial" + } else { + "missing" + }, + ); + } + + pub(crate) fn wrap(self, stream: LlmResponseStream) -> LlmResponseStream { + Box::pin(AttemptStream { + inner: stream, + guard: Some(self), + dispatch: tracing::dispatcher::get_default(Clone::clone), + }) + } +} + +impl Drop for AttemptGuard { + fn drop(&mut self) { + self.finish( + if self.model_finished { + "ok" + } else { + "cancelled" + }, + None, + ); + } +} + +struct AttemptStream { + inner: LlmResponseStream, + guard: Option, + dispatch: tracing::Dispatch, +} + +impl Stream for AttemptStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let Self { + inner, + guard, + dispatch, + } = self.get_mut(); + tracing::dispatcher::with_default(dispatch, || match inner.as_mut().poll_next(cx) { + Poll::Ready(Some(item)) => { + if guard.as_mut().is_some_and(|guard| guard.observe(&item)) { + guard.take(); + } + Poll::Ready(Some(item)) + } + Poll::Ready(None) => { + if let Some(mut guard) = guard.take() { + guard.final_usage |= guard.model_finished; + guard.success(); + } + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + }) + } +} + +impl Drop for AttemptStream { + fn drop(&mut self) { + let Self { + inner, + guard, + dispatch, + } = self; + // Registry parent references can close recursively when the last child + // drops. Keep that close on the subscriber that owns the response. + tracing::dispatcher::with_default(dispatch, || { + drop(guard.take()); + drop(std::mem::replace(inner, Box::pin(futures::stream::empty()))); + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::{StreamExt, stream}; + use serde_json::{Map, Value}; + use std::sync::{Arc, Mutex}; + use tracing::Subscriber; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::{ + Layer, + layer::{Context as LayerContext, SubscriberExt}, + registry::LookupSpan, + }; + + type Records = Arc>>>; + struct Collector(Records); + + impl LookupSpan<'a>> Layer for Collector { + fn on_new_span( + &self, + attrs: &tracing::span::Attributes<'_>, + id: &tracing::Id, + ctx: LayerContext<'_, S>, + ) { + let mut fields = Map::::new(); + attrs.record( + &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { + let text = format!("{value:?}"); + fields.insert( + field.name().into(), + serde_json::from_str(&text).unwrap_or(Value::String(text)), + ); + }, + ); + ctx.span(id).unwrap().extensions_mut().insert(fields); + } + fn on_record( + &self, + id: &tracing::Id, + values: &tracing::span::Record<'_>, + ctx: LayerContext<'_, S>, + ) { + let span = ctx.span(id).unwrap(); + let mut extensions = span.extensions_mut(); + let fields = extensions.get_mut::>().unwrap(); + values.record( + &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| { + let text = format!("{value:?}"); + fields.insert( + field.name().into(), + serde_json::from_str(&text).unwrap_or(Value::String(text)), + ); + }, + ); + } + fn on_close(&self, id: tracing::Id, ctx: LayerContext<'_, S>) { + let fields = ctx + .span(&id) + .unwrap() + .extensions_mut() + .remove::>() + .unwrap(); + self.0.lock().unwrap().push(fields); + } + } + + fn usage(input: Option, output: Option) -> LlmResponseChunk { + LlmResponseChunk::Usage(Usage { + input_tokens: input, + output_tokens: output, + ..Usage::default() + }) + } + + async fn observe( + events: Vec, + take: Option, + ) -> Map { + observe_results(events.into_iter().map(Ok).collect(), take).await + } + + async fn observe_results( + events: Vec>, + take: Option, + ) -> Map { + let records = Records::default(); + let subscriber = tracing_subscriber::registry().with(Collector(records.clone())); + async { + let guard = + AttemptGuard::new(&ModelId::from("model"), WireFormat::OpenAiChat, 1, 1, true); + let mut response = guard.wrap(stream::iter(events).boxed()); + if let Some(take) = take { + for _ in 0..take { + response.next().await; + } + } else { + while response.next().await.is_some() {} + } + drop(response); + } + .with_subscriber(subscriber) + .await; + let records = records.lock().unwrap(); + assert_eq!(records.len(), 1); + records[0].clone() + } + + #[tokio::test] + async fn usage_in_a_terminal_error_event_survives_regardless_of_chunk_order() { + let fields = observe( + vec![LlmResponseStreamEvent::new(vec![ + LlmResponseChunk::StreamError { + message: "private failure".into(), + }, + usage(Some(11), Some(3)), + ])], + None, + ) + .await; + assert_eq!(fields["outcome"], "error"); + assert_eq!(fields["gen_ai.usage.input_tokens"], 11); + assert_eq!(fields["gen_ai.usage.output_tokens"], 3); + assert!(!fields.contains_key("error")); + } + + #[tokio::test] + async fn cumulative_snapshots_preserve_missing_fields_and_final_corrections() { + let fields = observe( + vec![ + usage(Some(11), Some(2)).into(), + usage(None, Some(3)).into(), + usage(Some(9), Some(1)).into(), + LlmResponseChunk::MessageStop { + reason: Some("stop".into()), + } + .into(), + ], + None, + ) + .await; + assert_eq!(fields["gen_ai.usage.input_tokens"], 9); + assert_eq!(fields["gen_ai.usage.output_tokens"], 1); + assert_eq!(fields["switchyard.accounting.usage_status"], "complete"); + } + + #[tokio::test] + async fn drop_after_finish_is_success_with_partial_accounting() { + let fields = observe( + vec![ + usage(Some(11), Some(3)).into(), + LlmResponseChunk::MessageStop { + reason: Some("stop".into()), + } + .into(), + usage(Some(12), Some(4)).into(), + ], + Some(2), + ) + .await; + assert_eq!(fields["outcome"], "ok"); + assert_eq!(fields["gen_ai.usage.input_tokens"], 11); + assert_eq!(fields["switchyard.accounting.usage_status"], "partial"); + } + + #[tokio::test] + async fn missing_usage_is_distinct_from_explicit_zero() { + for (counts, status) in [(None, "missing"), (Some(0), "complete")] { + let fields = observe( + vec![ + LlmResponseChunk::MessageStop { + reason: Some("stop".into()), + } + .into(), + usage(counts, counts).into(), + ], + None, + ) + .await; + assert_eq!(fields["switchyard.accounting.usage_status"], status); + assert_eq!( + fields + .get("gen_ai.usage.input_tokens") + .and_then(Value::as_u64), + counts + ); + } + } + + #[tokio::test] + async fn trailing_usage_after_model_finish_is_not_lost_or_added_twice() { + let fields = observe( + vec![ + usage(Some(11), Some(3)).into(), + LlmResponseChunk::MessageStop { + reason: Some("stop".into()), + } + .into(), + usage(Some(12), Some(4)).into(), + usage(Some(12), Some(4)).into(), + ], + None, + ) + .await; + assert_eq!(fields["gen_ai.usage.input_tokens"], 12); + assert_eq!(fields["gen_ai.usage.output_tokens"], 4); + assert_eq!(fields["switchyard.accounting.usage_status"], "complete"); + } + + #[tokio::test] + async fn timeout_after_usage_retains_partial_consumption_and_error() { + let fields = observe_results( + vec![ + Ok(usage(Some(11), Some(3)).into()), + Err(LlmClientError::Timeout { + source: Box::new(std::io::Error::from(std::io::ErrorKind::TimedOut)), + }), + ], + None, + ) + .await; + assert_eq!(fields["gen_ai.usage.input_tokens"], 11); + assert_eq!(fields["outcome"], "error"); + assert_eq!(fields["error.type"], "timeout"); + assert_eq!(fields["switchyard.accounting.usage_status"], "partial"); + } +} diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index deb4d9a128..dc0c97610b 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -26,6 +26,7 @@ use switchyard_translation::{ use tokio::time::Instant; use tracing::Instrument; +use crate::attempt::AttemptGuard; use crate::backend::{Backend, openai_url}; use crate::error::{LlmClientError, Result}; use crate::metrics; @@ -381,7 +382,15 @@ impl TranslatingLlmClient { record_gen_ai_request(&url, model, streaming); let result = self - .send_with_retries(&url, backend, &body, metadata, model, streaming) + .send_with_retries( + &url, + backend, + &body, + metadata, + model, + streaming, + matches!(endpoint, UpstreamEndpoint::Completion), + ) .await; // Keep quota failures request-local when forwarding auth or returning 429 to callers. if let Some(until) = unavailable_until @@ -398,6 +407,7 @@ impl TranslatingLlmClient { } // Sends the encoded body, retrying retryable failures within the backend's retry budget. + #[allow(clippy::too_many_arguments)] async fn send_with_retries( &self, url: &str, @@ -406,6 +416,7 @@ impl TranslatingLlmClient { metadata: Option<&Metadata>, model: &ModelId, streaming: bool, + accounting: bool, ) -> Result { let max_retries = u64::from(backend.max_retries()); let max_attempts = max_retries @@ -436,40 +447,34 @@ impl TranslatingLlmClient { { return Err(deadline_error(timeout)); } - let span = tracing::debug_span!( - target: "libsy", - "libsy.upstream_attempt", - model = %model, - wire_format = %backend.wire_format(), - attempt = attempt + 1 + u64::from(template_retry), - max_attempts, - retry = attempt > 0 || template_retry, - openinference.span.kind = "CHAIN", - outcome = tracing::field::Empty, - status_code = tracing::field::Empty, - will_retry = tracing::field::Empty, - retry_delay_ms = tracing::field::Empty, - ); - let mut attempt_started = false; + let request = self.build_request(url, backend, body.as_ref(), metadata)?; + let mut guard = None; let result = tokio::select! { biased; - timeout = &mut deadline => { - if attempt_started { - metrics::record_upstream_attempt(None); - } - span.record("outcome", "error"); - span.record("will_retry", false); - return Err(deadline_error(timeout)); - } + _ = &mut deadline => None, result = async { - attempt_started = true; - self.send_once(url, backend, body.as_ref(), metadata, model, streaming).await - }.instrument(span.clone()) => result, + let attempt_guard = guard.insert(AttemptGuard::new(model, backend.wire_format(), attempt + 1 + u64::from(template_retry), max_attempts, accounting)); + let span = attempt_guard.span().clone(); + self.send_once(request, backend, metadata, model, streaming, attempt_guard).instrument(span).await + } => Some(result), + }; + let Some(result) = result else { + let timeout = expires_at + .expect("only a configured deadline can resolve") + .1; + let error = deadline_error(timeout); + if let Some(guard) = &mut guard { + metrics::record_upstream_attempt(None); + guard.span().record("will_retry", false); + guard.error(&error); + } + return Err(error); }; + let mut guard = guard.expect("a completed HTTP attempt has accounting state"); + let span = guard.span().clone(); // The retained handle updates this same attempt span with its outcome. match result { Ok(response) => { - span.record("outcome", "ok"); span.record("status_code", response.status()); span.record("will_retry", false); if attempt > 0 || template_retry { @@ -482,10 +487,25 @@ impl TranslatingLlmClient { upstream_headers, } => EncodedResponse::Streaming { status, - chunks: stream_with_deadline(chunks, expires_at), + chunks: guard.wrap(stream_with_deadline(chunks, expires_at)), upstream_headers, }, - buffered => buffered, + EncodedResponse::Buffered { + status, + body, + upstream_headers, + .. + } => { + if !accounting { + guard.success(); + } + EncodedResponse::Buffered { + status, + body, + upstream_headers, + attempt: accounting.then_some(guard), + } + } }); } Err(failure) => { @@ -495,7 +515,7 @@ impl TranslatingLlmClient { && rejects_template_options(&failure.error); let will_retry = retry_without_template || (attempt < max_retries && failure.is_retryable()); - span.record("outcome", "error"); + guard.error(&failure.error); if let Some(status) = failure.status { span.record("status_code", status.as_u16()); } @@ -516,6 +536,7 @@ impl TranslatingLlmClient { span.record("retry_delay_ms", duration_millis(delay)); // Close the attempt span before sleeping so backoff is not attempt latency. drop(span); + drop(guard); tokio::select! { biased; timeout = &mut deadline => return Err(deadline_error(timeout)), @@ -527,16 +548,14 @@ impl TranslatingLlmClient { } } - // Performs one HTTP attempt and retains the retry metadata alongside any error. - async fn send_once( + // Validate the request before opening a physical attempt. + fn build_request( &self, url: &str, backend: &Backend, body: &Value, metadata: Option<&Metadata>, - model: &ModelId, - streaming: bool, - ) -> std::result::Result { + ) -> Result { let client = if backend.is_forwarding_auth() { &self.forward_auth_client } else { @@ -547,9 +566,24 @@ impl TranslatingLlmClient { let builder = backend.apply_forwarded_auth(builder, metadata); let builder = apply_extra_headers(builder, backend); let builder = backend.apply_auth(builder); + builder.build().map_err(convert_reqwest_error) + } + async fn send_once( + &self, + request: reqwest::Request, + backend: &Backend, + metadata: Option<&Metadata>, + model: &ModelId, + streaming: bool, + guard: &mut AttemptGuard, + ) -> std::result::Result { + let client = if backend.is_forwarding_auth() { + &self.forward_auth_client + } else { + &self.client + }; let response = match async { - let request = builder.build()?; if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) { crate::payload_capture::record("request", || String::from_utf8_lossy(body)); } @@ -568,10 +602,12 @@ impl TranslatingLlmClient { } }; let status = response.status(); + guard.span().record("status_code", status.as_u16()); + guard.headers(response.headers()); if status.is_success() { if streaming { let upstream_headers = response.headers().clone(); - let chunks = match prepare_response_stream(response, backend, model).await { + let chunks = match prepare_response_stream(response, backend, model, guard).await { Ok(chunks) => chunks, Err(error) => { metrics::record_upstream_attempt(None); @@ -607,6 +643,7 @@ impl TranslatingLlmClient { status: status.as_u16(), body: body.to_vec(), upstream_headers, + attempt: None, }); } @@ -622,6 +659,9 @@ impl TranslatingLlmClient { }); } }; + if let Ok(value) = serde_json::from_str::(&body) { + guard.response(&value, backend.wire_format()); + } let body = redact_forwarded_headers(body, metadata, backend.is_forwarding_auth()); metrics::record_upstream_attempt(Some(status.as_u16())); let error = @@ -653,6 +693,8 @@ impl TranslatingLlmClient { request: Request, model_name: Option<&ModelId>, ) -> Result { + tracing::Span::current().record("switchyard.accounting.unit", "summary"); + tracing::Span::current().record("switchyard.accounting.attempts", 0_u64); let Request { mut llm_request, metadata, @@ -702,16 +744,22 @@ impl TranslatingLlmClient { EncodedResponse::Buffered { body, upstream_headers, + mut attempt, .. } => { - let body = serde_json::from_slice::(&body).map_err(|error| { - LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}")) - })?; - // Map a provider's failed generation to 502, even under HTTP 200. - // Redact forwarded credentials before returning the provider error. - let mut agg = - observed_translation::decode_response(&body, wire_format).map_err(|error| { - match error { + let result = (|| { + let body = serde_json::from_slice::(&body).map_err(|error| { + LlmClientError::ResponseTranslation(format!( + "invalid upstream JSON: {error}" + )) + })?; + if let Some(guard) = &mut attempt { + guard.response(&body, wire_format); + } + // Map a provider's failed generation to 502, even under HTTP 200. + // Redact forwarded credentials before returning the provider error. + let mut agg = observed_translation::decode_response(&body, wire_format) + .map_err(|error| match error { TranslationError::UpstreamFailure { error } => { LlmClientError::UpstreamHttp { status: StatusCode::BAD_GATEWAY, @@ -723,17 +771,24 @@ impl TranslatingLlmClient { } } error => LlmClientError::ResponseTranslation(error.to_string()), - } - })?; - // The gateway may report a buffered charge only in its response header. - if agg.usage.provider_cost.is_none() { - agg.usage.provider_cost = upstream_headers - .get("x-litellm-response-cost") - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.trim().parse::().ok()) - .filter(|usd| usd.is_finite() && *usd >= 0.0); + })?; + // The gateway may report a buffered charge only in its response header. + if agg.usage.provider_cost.is_none() { + agg.usage.provider_cost = upstream_headers + .get("x-litellm-response-cost") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .filter(|usd| usd.is_finite() && *usd >= 0.0); + } + Ok((LlmResponse::Agg(agg), upstream_headers)) + })(); + if let Some(guard) = &mut attempt { + match &result { + Ok(_) => guard.success(), + Err(error) => guard.error(error), + } } - (LlmResponse::Agg(agg), upstream_headers) + result? } }; @@ -849,6 +904,7 @@ enum EncodedResponse { status: u16, body: Vec, upstream_headers: HeaderMap, + attempt: Option, }, Streaming { status: u16, @@ -926,6 +982,7 @@ async fn prepare_response_stream( response: reqwest::Response, backend: &Backend, model: &ModelId, + guard: &mut AttemptGuard, ) -> Result { let bytes = response.bytes_stream().map(|chunk| { chunk @@ -942,6 +999,9 @@ async fn prepare_response_stream( Some(first) => { // An in-band context overflow skips retries and advances to another candidate. if let Some(message) = first_event_overflow(&first, backend) { + if let Ok(event) = &first { + guard.observe_usage(event); + } return Err(LlmClientError::ContextWindowExceeded { model: model.clone(), message, diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index 3710ad150d..d216e68350 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -20,6 +20,7 @@ //! answer call from the routing outcome when needed. A host that just wants the answer does not //! have to drive the step stream itself. +mod attempt; pub mod backend; pub mod client; pub mod error; diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs index 26a916343c..695f4bcb92 100644 --- a/crates/libsy-llm-client/src/observability.rs +++ b/crates/libsy-llm-client/src/observability.rs @@ -113,7 +113,7 @@ fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) { } /// Records semantic token totals and cache-usage breakdowns on a client-call span. -fn record_gen_ai_usage(span: &Span, usage: &Usage) { +pub(crate) fn record_gen_ai_usage(span: &Span, usage: &Usage) { let cache_read = usage.cached_input_tokens(); let cache_creation = usage.cache_creation_input_tokens(); let cache_creation_1h = usage @@ -207,7 +207,7 @@ fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> { } } -fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> { +pub(crate) fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> { match error { LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"), LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"), diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index d660f8c987..13adae8e59 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -421,6 +421,8 @@ impl CallPhase { gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty, gen_ai.usage.reasoning.output_tokens = tracing::field::Empty, + switchyard.accounting.unit = tracing::field::Empty, + switchyard.accounting.attempts = tracing::field::Empty, outcome = tracing::field::Empty, otel.status_code = tracing::field::Empty, error.type = tracing::field::Empty, @@ -455,23 +457,30 @@ async fn call_one( let started = Instant::now(); let mut routing_usage = None; let result = async { - let mut response = client?.call(request).await?; + let response = async { client?.call(request).await } + .await + .map_err(|source| LibsyError::client_call(model_id.clone(), source))?; + let mut response = observability::observe_client_call(Ok(response))?; if is_routing && let LlmResponse::Stream(chunks) = response.llm_response { - let (chunks, usage) = buffer_routing_stream(chunks).await?; + let (chunks, usage) = buffer_routing_stream(chunks) + .await + .map_err(|source| LibsyError::client_call(model_id.clone(), source))?; routing_usage = usage; response.llm_response = LlmResponse::Stream(chunks); } Ok(response) } - .await - .map_err(|source| LibsyError::client_call(model_id.clone(), source)); + .await; let duration = started.elapsed(); let result = result.map(|mut response| { response.set_served_model(model_id); response }); - let result = observability::observe_client_call(result); + let result = match result { + Ok(response) => Ok(response), + Err(error) => observability::observe_client_call(Err(error)), + }; let result = if is_routing { result } else { diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 558b7cb5e4..39a842c46e 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -2295,3 +2295,155 @@ async fn in_flight_gauge_clears_when_a_run_is_abandoned() -> switchyard_libsy::R ); Ok(()) } + +#[tokio::test] +async fn routing_buffer_failure_preserves_real_attempt_and_parent_usage() +-> Result<(), Box> { + use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; + use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; + + let _guard = serialize_test().lock().await; + let (store, _, _, _, _) = telemetry(); + const MODEL: &str = "obs-routing-failure-usage"; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_raw(concat!( + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"r1\"}}\n\n", + "data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"message\":\"synthetic failure\"},\"usage\":{\"input_tokens\":11,\"output_tokens\":3}}}\n\n" + ), "text/event-stream")) + .expect(1) + .mount(&server).await; + let client = Arc::new(TranslatingLlmClient::new(&[ModelConfig::new( + MODEL, + Backend::OpenAiResponses(HttpBackendConfig { + base_url: server.uri(), + api_key: None, + forward_auth: false, + extra_headers: Default::default(), + extra_body: Default::default(), + omit_body_fields: Default::default(), + reasoning_effort: None, + max_retries: 1, + failure_cooldown: Duration::ZERO, + timeout: None, + }), + None, + )])?); + let algorithm = Arc::new(RoutingCallAlgo { + name: "obs-routing-failure-usage-algo".into(), + target: MODEL.into(), + }); + let mut request = request_with_metadata("routing-failure", "routing-failure"); + request.llm_request.stream = true; + assert!(run(algorithm, client, request).await.is_err()); + let spans = store.spans(); + for (name, field) in [ + ("libsy.client_call", "selected_model"), + ("libsy.upstream_attempt", "model"), + ] { + let span = find_span(&spans, name, field, MODEL); + assert_eq!( + span.fields + .get("gen_ai.usage.input_tokens") + .map(String::as_str), + Some("11") + ); + assert_eq!( + span.fields + .get("gen_ai.usage.output_tokens") + .map(String::as_str), + Some("3") + ); + assert_eq!( + span.fields.get("outcome").map(String::as_str), + Some("error") + ); + } + assert_eq!( + spans + .iter() + .filter(|s| s.name == "libsy.upstream_attempt" + && s.fields.get("model").is_some_and(|m| m == MODEL)) + .count(), + 1 + ); + Ok(()) +} + +#[tokio::test] +async fn retries_and_fallback_keep_separate_models_and_attempt_ordinals() +-> Result<(), Box> { + use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; + use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; + + let _guard = serialize_test().lock().await; + let (store, _, _, _, _) = telemetry(); + const WEAK: &str = "obs-attempt-fallback-weak"; + const STRONG: &str = "obs-attempt-fallback-strong"; + let server = MockServer::start().await; + Mock::given(method("POST")).respond_with(|request: &wiremock::Request| { + let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap(); + if body["model"] == WEAK { + ResponseTemplate::new(503).insert_header("retry-after", "0") + .set_body_json(json!({"error":{"message":"synthetic failure"},"usage":{"prompt_tokens":11,"completion_tokens":3,"cost":0.02}})) + } else { + ResponseTemplate::new(200).set_body_json(json!({ + "id":"fallback", "model":STRONG, + "choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":7,"completion_tokens":2,"cost":0.03} + })) + } + }).expect(3).mount(&server).await; + let backend = Backend::OpenAiChat(HttpBackendConfig { + base_url: server.uri(), + api_key: None, + forward_auth: false, + extra_headers: Default::default(), + extra_body: Default::default(), + omit_body_fields: Default::default(), + reasoning_effort: None, + max_retries: 1, + failure_cooldown: Duration::ZERO, + timeout: None, + }); + let client = Arc::new(TranslatingLlmClient::new(&[ + ModelConfig::new(WEAK, backend.clone(), None), + ModelConfig::new(STRONG, backend, None), + ])?); + let algorithm = Arc::new(SingleCallAlgo { + name: "obs-attempt-fallback-algo".into(), + target_set: vec![WEAK.into(), STRONG.into()], + metadata: None, + }); + let (model, response) = run( + algorithm, + client, + request_with_metadata("fallback", "fallback"), + ) + .await?; + assert_eq!(model.as_str(), WEAK); + assert_eq!(response.served_model().unwrap().as_str(), STRONG); + let spans = store.spans(); + for (model, count, cost) in [(WEAK, 2, "0.02"), (STRONG, 1, "0.03")] { + let attempts: Vec<_> = spans + .iter() + .filter(|s| { + s.name == "libsy.upstream_attempt" + && s.fields.get("model").is_some_and(|m| m == model) + }) + .collect(); + assert_eq!(attempts.len(), count); + let mut ordinals = Vec::new(); + for attempt in attempts { + ordinals.push(attempt.fields["attempt"].parse::()?); + assert_eq!(attempt.fields["switchyard.usage.provider_cost_usd"], cost); + } + ordinals.sort(); + assert_eq!(ordinals, (1..=count).collect::>()); + assert_eq!( + find_span(&spans, "libsy.client_call", "selected_model", model).fields["switchyard.accounting.attempts"], + count.to_string() + ); + } + Ok(()) +} diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 69b8c46bb3..13fa0298de 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -317,3 +317,26 @@ that guarantee. behavior. The plugin does not maintain a separate trusted-default target or rerun routing after an execution failure. Failures outside the shared runner, including response translation failures, are returned to Relay. + +## Validate attempt accounting + +After building and packaging the native library as described above, run the +local acceptance fixture with Relay 0.10: + +```sh +uv run --with nemo-relay==0.10.0 python \ + crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py \ + --manifest /absolute/path/to/bundle/relay-plugin.toml \ + --output /tmp/switchyard-attempt-accounting +``` + +The fixture uses a loopback HTTP provider and fixed synthetic prices. It checks +failed and successful attempts, reported-charge precedence, token-price +estimates, missing usage, unknown prices, zero charges, duplicate delivery, +missing records, and workflow isolation. It writes ATOF events and separate +cost summaries. Relay also loads system plugin configuration when present. + +The [cost-summary example](examples/attempt_costs.py) selects unique attempt +records and uses Relay's codec to obtain costs. These summaries cover Switchyard +completion work. They do not change ATIF `final_metrics` or include separate +decision-model usage. diff --git a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py new file mode 100644 index 0000000000..7c965178b9 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Price unique Switchyard attempts from one workflow's ATOF events using Relay.""" + +import argparse +import json +import math +from pathlib import Path + +from nemo_relay import codecs + + +def summarize(events, workflow_uuid): + """Return observed spending; a missing total means coverage or pricing is incomplete. + + Configure Relay's pricing component before calling this function to enable + estimates. Provider-reported charges work without a pricing catalog. + This covers instrumented completion attempts, including routing calls. + """ + scopes = {e["uuid"]: e for e in events if e.get("kind") == "scope"} + + def belongs(event): + seen = set() + while event["uuid"] not in seen: + if event["uuid"] == workflow_uuid: + return True + seen.add(event["uuid"]) + event = scopes.get(event.get("parent_uuid")) + if event is None: + return False + return False + + ends = { + e["uuid"]: e + for e in events + if e.get("kind") == "scope" and e.get("scope_category") == "end" and belongs(e) + } + calls = [e for e in scopes.values() if e.get("name") == "libsy.client_call" and belongs(e)] + attempts = [ + e for e in ends.values() + if e.get("metadata", {}).get("switchyard.accounting.unit") == "completion_attempt" + ] + coverage = bool(calls) and workflow_uuid in ends + for call in calls: + meta = ends.get(call["uuid"], {}).get("metadata", {}) + children = [e for e in attempts if e.get("parent_uuid") == call["uuid"]] + expected = meta.get("switchyard.accounting.attempts") + coverage &= ( + meta.get("switchyard.accounting.unit") == "summary" + and isinstance(expected, int) + and expected >= 0 + and expected == meta.get("switchyard.accounting.attempt_records") == len(children) + and sorted(e["metadata"]["attempt"] for e in children) == list(range(1, expected + 1)) + ) + coverage &= all(e.get("parent_uuid") in {c["uuid"] for c in calls} for e in attempts) + records = [] + for event in attempts: + meta = event["metadata"] + usage = codecs.AnthropicMessagesCodec().decode_response(event["data"]).usage or {} + cost = usage.get("cost", {}) + amount = cost.get("total") + priced = isinstance(amount, (float, int)) and math.isfinite(amount) and amount >= 0 + complete = priced and ( + cost.get("source") == "provider_reported" + or meta.get("switchyard.accounting.usage_status") == "complete" + ) + records.append({ + "uuid": event["uuid"], "model": event["data"].get("model"), + "outcome": meta.get("outcome"), + "usage_status": meta.get("switchyard.accounting.usage_status"), + "cost": cost if priced else None, "cost_complete": complete, + }) + subtotal = math.fsum(r["cost"]["total"] for r in records if r["cost"] is not None) + complete = coverage and all(r["cost_complete"] for r in records) + return { + "workflow_uuid": workflow_uuid, "attempts": records, + "coverage_complete": bool(coverage), "cost_status": "complete" if complete else "incomplete", + "known_cost_subtotal_usd": subtotal, "total_cost_usd": subtotal if complete else None, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("events", type=Path, help="ATOF JSONL containing the workflow's scope tree") + parser.add_argument("--workflow-uuid", required=True) + args = parser.parse_args() + events = [json.loads(line) for line in args.events.read_text().splitlines() if line.strip()] + print(json.dumps(summarize(events, args.workflow_uuid), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py new file mode 100644 index 0000000000..0c5ae3983f --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise a packaged native plugin with Relay 0.10 and a synthetic provider.""" + +import argparse +import asyncio +import json +import math +import platform +import sys +import tempfile +import threading +import tomllib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib.metadata import version +from pathlib import Path + +import nemo_relay as relay +from nemo_relay import codecs, model_pricing, plugin, subscribers + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "examples")) +from attempt_costs import summarize # noqa: E402 + + +class Provider(BaseHTTPRequestHandler): + scenario = "reported" + calls = 0 + + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + first = Provider.calls % 2 == 0 + Provider.calls += 1 + usage = {"prompt_tokens": 11 if first else 7, "completion_tokens": 3 if first else 2} + if self.scenario in {"reported", "missing"}: + usage["cost"] = 0.02 if first else 0.03 + if self.scenario == "zero": + usage = {"cost": 0.0} + if self.scenario == "missing" and first: + usage = {} + body = {"model": request["model"], "usage": usage} + if first: + body["error"] = {"message": "synthetic failure"} + else: + body.update(id="r1", choices=[{ + "index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", + }]) + raw = json.dumps(body).encode() + self.send_response(503 if first else 200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.send_header("Retry-After", "0") + # A body charge must override this valid header fallback. + if self.scenario == "reported": + self.send_header("x-litellm-response-cost", "1.0") + self.end_headers() + self.wfile.write(raw) + + def log_message(self, *_args): + pass + + +async def run(manifest, output): + server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + events = [] + subscribers.register("attempt-accounting-test", lambda e: events.append(e.to_dict())) + activation = None + try: + deployment = output / "switchyard.toml" + deployment.write_text(f'''schema_version = 1 +[llm_clients.local] +format = "openai_chat" +base_url = "http://127.0.0.1:{server.server_port}/v1" +max_retries = 1 +failure_cooldown_ms = 0 +[targets.default] +id = "test/model" +llm_client = "local" +[targets.unpriced] +id = "test/unpriced" +llm_client = "local" +[routes.default] +id = "switchyard/default" +type = "passthrough" +target = "default" +[routes.unpriced] +id = "switchyard/unpriced" +type = "passthrough" +target = "unpriced" +''') + config = output / "plugins.toml" + config.write_text(f'''[[plugins.dynamic]] +manifest = {json.dumps(str(manifest))} +[plugins.dynamic.config] +switchyard_config_path = {json.dumps(str(deployment))} +[plugins.policy.overrides."nvidia.switchyard"] +attestation = "integrity_only" +''') + prices = model_pricing.ComponentSpec(model_pricing.PricingConfig(sources=[ + model_pricing.InlineSource(model_pricing.PricingCatalog(entries=[ + model_pricing.ModelPricing( + provider="anthropic", model_id="test/model", pricing_as_of="2026-11-04", + pricing_source="synthetic", rates=model_pricing.TokenPricingRates( + input_per_million=1000, output_per_million=2000, + ), + ), + ])), + ])) + activation = await plugin.initialize(plugin.PluginConfig(components=[prices]), config) + (output / "environment.json").write_text(json.dumps({ + "relay_version": version("nemo-relay"), "platform": platform.platform(), + "bundle_sha256": tomllib.loads(manifest.read_text())["integrity"]["sha256"], + }, indent=2)) + + async def forbidden(_request): + raise AssertionError("native plugin did not intercept the request") + + for scenario, expected_subtotal, expected_total in [ + ("reported", 0.05, 0.05), ("missing", 0.03, None), + ("estimated", 0.028, 0.028), ("unpriced", 0.0, None), ("zero", 0.0, 0.0), + ]: + Provider.scenario = scenario + Provider.calls = 0 + route = "unpriced" if scenario == "unpriced" else "default" + with relay.scope.scope(scenario, relay.ScopeType.Agent): + await relay.llm.execute( + "openai.chat_completions", + relay.LLMRequest({}, {"model": f"switchyard/{route}", "messages": [{"role": "user", "content": "synthetic test"}]}), + forbidden, codec=codecs.OpenAIChatCodec(), response_codec=codecs.OpenAIChatCodec(), + ) + await subscribers.flush_async() + workflow = next(e["uuid"] for e in events if e.get("name") == scenario and e.get("scope_category") == "start") + report = summarize(events, workflow) + assert Provider.calls == 2 + assert len(report["attempts"]) == 2 + assert report["attempts"][0]["outcome"] == "error" + assert report["coverage_complete"] + assert math.isclose(report["known_cost_subtotal_usd"], expected_subtotal) + if expected_total is None: + assert report["total_cost_usd"] is None + else: + assert math.isclose(report["total_cost_usd"], expected_total) + # ATOF replay, missing records, and mixed workflows must not inflate totals. + assert summarize(events + events, workflow) == report + dropped = report["attempts"][0]["uuid"] + assert summarize([e for e in events if e["uuid"] != dropped], workflow)["total_cost_usd"] is None + if scenario == "reported": + assert all(a["cost"]["source"] == "provider_reported" for a in report["attempts"]) + if scenario == "estimated": + assert all(a["cost"]["source"] == "model_pricing" for a in report["attempts"]) + if scenario == "zero": + assert all(a["usage_status"] == "missing" for a in report["attempts"]) + (output / f"{scenario}.json").write_text(json.dumps(report, indent=2)) + print(f"{scenario}: subtotal={report['known_cost_subtotal_usd']}, total={report['total_cost_usd']}") + (output / "events.jsonl").write_text("".join(json.dumps(e) + "\n" for e in events)) + finally: + if activation is not None: + await activation.close() + subscribers.deregister("attempt-accounting-test") + server.shutdown() + server.server_close() + thread.join() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.output: + args.output.mkdir(parents=True, exist_ok=True) + asyncio.run(run(args.manifest.resolve(), args.output.resolve())) + else: + with tempfile.TemporaryDirectory() as directory: + asyncio.run(run(args.manifest.resolve(), Path(directory))) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs b/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs new file mode 100644 index 0000000000..dbd621f001 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs @@ -0,0 +1,386 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc, +}; + +use futures_util::StreamExt; +use switchyard_protocol::{Request, WireFormat, text_request}; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; + +use crate::{config::SwitchyardConfig, runtime::SwitchyardRuntime}; + +fn runtime(server: &MockServer, format: &str, retries: u32) -> SwitchyardRuntime { + let config = serde_json::from_value::(serde_json::json!({ + "switchyard_config": { + "schema_version": 1, + "llm_clients": {"local": {"format": format, "base_url": format!("{}/v1",server.uri()), "max_retries": retries, "failure_cooldown_ms": 0}}, + "targets": {"default": {"id": "test/model", "llm_client": "local"}}, + "routes": {"default": {"id": "switchyard/default", "type": "passthrough", "target": "default"}} + } + })).unwrap(); + SwitchyardRuntime::new(config).unwrap() +} + +fn request(stream: bool) -> Request { + let mut llm_request = text_request(Some("switchyard/default".into()), "test"); + llm_request.stream = stream; + Request { + llm_request, + raw_request: None, + metadata: None, + } +} + +fn recorder(payloads: bool) -> (SpanCapture, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(); + ( + SpanCapture::new(payloads.then_some(8192), move |span| tx.send(span).unwrap()), + rx, + ) +} + +fn success() -> ResponseTemplate { + ResponseTemplate::new(200).insert_header("x-litellm-response-cost", "1.0").set_body_json(serde_json::json!({ + "id":"response", "model":"test/model", "choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":7,"completion_tokens":2,"total_tokens":9,"cost":0.03} + })) +} + +#[tokio::test] +async fn retry_usage_and_reported_charges_survive_failure_without_parent_charge() { + let server = MockServer::start().await; + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + Mock::given(method("POST")).respond_with(move |_: &wiremock::Request| { + if seen.fetch_add(1, Ordering::SeqCst) == 0 { + ResponseTemplate::new(503).insert_header("retry-after", "0").set_body_json(serde_json::json!({ + "error":{"message":"failed"}, "usage":{"prompt_tokens":11,"completion_tokens":3,"cost":0.02} + })) + } else { success() } + }).mount(&server).await; + let runtime = runtime(&server, "openai_chat", 1); + let (capture, rx) = recorder(false); + let result = capture + .capture(runtime.execute_buffered(WireFormat::OpenAiChat, request(false))) + .await; + assert!(result.result.is_ok()); + let call = rx.try_recv().unwrap(); + assert_eq!(call.children.len(), 2); + assert_eq!(call.fields["switchyard.accounting.attempts"], 2); + let (kind, metadata, _, output) = call.projection(); + assert!(matches!(kind, ScopeType::Custom)); + assert!(output.is_none()); + assert_eq!(metadata["switchyard.accounting.attempt_records"], 2); + for (attempt, outcome, input, cost) in [ + (&call.children[0], "error", 11, 0.02), + (&call.children[1], "ok", 7, 0.03), + ] { + let (kind, metadata, input_data, output) = attempt.projection(); + assert!(matches!(kind, ScopeType::Llm)); + assert_eq!(metadata["outcome"], outcome); + assert_eq!(metadata["switchyard.accounting.usage_status"], "complete"); + assert_eq!(input_data.unwrap()["model"], "test/model"); + let output = output.unwrap(); + assert_eq!(output["usage"]["input_tokens"], input); + assert_eq!(output["usage"]["cost"]["total"], cost); + } + assert_eq!( + call.children[1].fields["switchyard.accounting.charge_source"], + "provider" + ); + assert_eq!(calls.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn missing_retry_usage_is_not_zero() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503).insert_header("retry-after", "0")) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_chat", 1); + let (capture, rx) = recorder(false); + assert!( + capture + .capture(runtime.execute_buffered(WireFormat::OpenAiChat, request(false))) + .await + .result + .is_err() + ); + let call = rx.try_recv().unwrap(); + assert_eq!(call.children.len(), 2); + for attempt in &call.children { + let (_, fields, _, output) = attempt.projection(); + assert_eq!(fields["switchyard.accounting.usage_status"], "missing"); + assert!(output.unwrap().get("usage").is_none()); + } +} + +#[tokio::test] +async fn buffered_failed_generation_keeps_usage_without_retrying_content_failure() { + let server = MockServer::start().await; + Mock::given(method("POST")).respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "id":"failed", "model":"test/model", "status":"failed", "error":{"message":"generation failed"}, + "usage":{"input_tokens":11,"output_tokens":3,"total_tokens":14} + }))).expect(1).mount(&server).await; + let runtime = runtime(&server, "openai_responses", 1); + let (capture, rx) = recorder(false); + assert!( + capture + .capture(runtime.execute_buffered(WireFormat::OpenAiResponses, request(false))) + .await + .result + .is_err() + ); + let call = rx.try_recv().unwrap(); + let [attempt] = call.children.as_slice() else { + panic!("expected one HTTP attempt") + }; + assert_eq!(attempt.fields["outcome"], "error"); + assert_eq!(attempt.fields["gen_ai.usage.input_tokens"], 11); + assert_eq!( + attempt.fields["switchyard.accounting.usage_status"], + "complete" + ); +} + +#[tokio::test] +async fn failed_stream_terminal_frame_keeps_usage_and_error() { + let server = MockServer::start().await; + let body = concat!( + "data: {\"type\":\"response.created\",\"response\":{\"id\":\"r1\",\"model\":\"test/model\"}}\n\n", + "data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"message\":\"failed\"},\"usage\":{\"input_tokens\":11,\"output_tokens\":3}}}\n\n" + ); + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(body), + ) + .expect(1) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_responses", 1); + let (capture, rx) = recorder(false); + capture + .capture(async { + let mut events = runtime + .execute_stream(WireFormat::OpenAiResponses, request(true), Arc::new(|_| {})) + .await + .result + .unwrap(); + while events.next().await.is_some() {} + }) + .await; + let call = rx.try_recv().unwrap(); + let [attempt] = call.children.as_slice() else { + panic!("expected one HTTP attempt") + }; + assert_eq!(attempt.fields["outcome"], "error"); + assert_eq!(attempt.fields["gen_ai.usage.output_tokens"], 3); + assert_eq!( + attempt.fields["switchyard.accounting.usage_status"], + "complete" + ); +} + +#[tokio::test] +async fn abandoning_stream_preserves_partial_usage() { + let server = MockServer::start().await; + let body = concat!( + "data: {\"id\":\"r1\",\"model\":\"test/model\",\"choices\":[],\"usage\":{\"prompt_tokens\":11,\"completion_tokens\":3}}\n\n", + "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"}}]}\n\n" + ); + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(body), + ) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_chat", 0); + let (capture, rx) = recorder(false); + capture + .capture(async { + let mut events = runtime + .execute_stream(WireFormat::OpenAiChat, request(true), Arc::new(|_| {})) + .await + .result + .unwrap(); + while let Some(event) = events.next().await { + if event.unwrap().to_string().contains("partial") { + break; + } + } + drop(events); + }) + .await; + let call = rx.try_recv().unwrap(); + let attempt = &call.children[0]; + assert_eq!(attempt.fields["outcome"], "cancelled"); + assert_eq!(attempt.fields["gen_ai.usage.input_tokens"], 11); + assert_eq!( + attempt.fields["switchyard.accounting.usage_status"], + "partial" + ); +} + +#[test] +fn debug_payloads_cannot_replace_usage_and_summary_is_not_priceable() { + let fields = serde_json::json!({"gen_ai.request.model":"model", "gen_ai.usage.input_tokens":12, "gen_ai.usage.output_tokens":0, "switchyard.accounting.unit":"completion_attempt"}).as_object().unwrap().clone(); + let mut attempt = CapturedSpan { + name: UPSTREAM_ATTEMPT, + fields, + payloads: Payloads { + request: Some(serde_json::json!({"body":"{}","truncated":false})), + response: Some(serde_json::json!({"body":null,"truncated":true})), + }, + started_at: SystemTime::now(), + ended_at: SystemTime::now(), + children: Vec::new(), + }; + let (_, _, input, output) = attempt.projection(); + assert_eq!(input.unwrap()["model"], "model"); + let output = output.unwrap(); + assert_eq!(output["usage"]["output_tokens"], 0); + assert_eq!(output["truncated"], true); + attempt.fields.insert( + "switchyard.accounting.unit".into(), + Json::String("summary".into()), + ); + let (kind, _, _, output) = attempt.projection(); + assert!(matches!(kind, ScopeType::Custom)); + assert!(output.is_none()); +} + +#[tokio::test] +async fn header_only_charge_is_preserved_on_an_http_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).insert_header("x-litellm-response-cost", "0")) + .expect(1) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_chat", 0); + let (capture, rx) = recorder(false); + assert!( + capture + .capture(runtime.execute_buffered(WireFormat::OpenAiChat, request(false))) + .await + .result + .is_err() + ); + let call = rx.try_recv().unwrap(); + let [attempt] = call.children.as_slice() else { + panic!("expected one attempt") + }; + let (_, meta, _, output) = attempt.projection(); + assert_eq!(meta["outcome"], "error"); + assert_eq!(meta["switchyard.accounting.usage_status"], "missing"); + assert_eq!(meta["switchyard.accounting.charge_source"], "header"); + assert_eq!(output.unwrap()["usage"]["cost"]["total"], 0.0); +} + +#[tokio::test] +async fn expired_deadline_does_not_create_an_attempt() { + let server = MockServer::start().await; + use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; + use tracing::Instrument; + let client = TranslatingLlmClient::new(&[ModelConfig::new( + "test/model", + Backend::OpenAiChat(HttpBackendConfig { + base_url: server.uri(), + api_key: None, + forward_auth: false, + extra_headers: Default::default(), + extra_body: Default::default(), + omit_body_fields: Default::default(), + reasoning_effort: None, + max_retries: 0, + failure_cooldown: std::time::Duration::ZERO, + timeout: Some(std::time::Duration::ZERO), + }), + None, + )]) + .unwrap(); + let (capture, rx) = recorder(false); + let result = capture + .capture(async { + client + .call_rewrite_model(request(false), Some(&"test/model".into())) + .instrument(tracing::debug_span!(target: "libsy", "libsy.client_call", + switchyard.accounting.unit = tracing::field::Empty, + switchyard.accounting.attempts = tracing::field::Empty)) + .await + }) + .await; + assert!(result.is_err()); + let call = rx.try_recv().unwrap(); + assert!(call.children.is_empty()); + assert_eq!(call.fields["switchyard.accounting.attempts"], 0); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[tokio::test] +async fn concurrent_calls_keep_their_own_attempts() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(success().set_delay(std::time::Duration::from_millis(10))) + .expect(2) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_chat", 0); + let (capture, rx) = recorder(false); + let first = capture.capture(runtime.execute_buffered(WireFormat::OpenAiChat, request(false))); + let second = capture.capture(runtime.execute_buffered(WireFormat::OpenAiChat, request(false))); + let (first, second) = tokio::join!(first, second); + assert!(first.result.is_ok()); + assert!(second.result.is_ok()); + let calls: Vec<_> = rx.try_iter().collect(); + assert_eq!(calls.len(), 2); + for call in calls { + assert_eq!(call.children.len(), 1); + assert_eq!(call.fields["switchyard.accounting.attempts"], 1); + assert_eq!(call.children[0].fields["gen_ai.usage.input_tokens"], 7); + } +} + +#[tokio::test] +async fn prefetched_context_rejection_keeps_usage_and_specific_error_type() { + let server = MockServer::start().await; + let body = "data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"message\":\"maximum context length exceeded\"},\"usage\":{\"input_tokens\":11,\"output_tokens\":0}}}\n\n"; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_raw(body, "text/event-stream")) + .expect(1) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_responses", 1); + let (capture, rx) = recorder(false); + assert!( + capture + .capture(runtime.execute_stream( + WireFormat::OpenAiResponses, + request(true), + Arc::new(|_| {}) + )) + .await + .result + .is_err() + ); + let call = rx.try_recv().unwrap(); + let [attempt] = call.children.as_slice() else { + panic!("expected one attempt") + }; + assert_eq!(attempt.fields["gen_ai.usage.input_tokens"], 11); + assert_eq!(attempt.fields["error.type"], "context_window_exceeded"); + assert_eq!( + attempt.fields["switchyard.accounting.usage_status"], + "complete" + ); +} diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs index 59db88c4bc..9b38c3634e 100644 --- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs +++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs @@ -200,28 +200,10 @@ impl CapturedSpan { } pub(crate) fn emit(self, runtime: &PluginRuntime) -> Result<(), String> { - let mut metadata = Json::Object(self.fields); - let output = self.payloads.response.or_else(|| { - if self.name == CLIENT_CALL { - response_data(&metadata) - } else { - None - } - }); - if let Some(session_id) = metadata.get("gen_ai.conversation.id").cloned() { - metadata["session_id"] = session_id; - } - if let Some(status) = span_status(&metadata) { - metadata["otel.status_code"] = status["otel.status_code"].clone(); - } - let input = self.payloads.request.or_else(|| { - metadata - .get("gen_ai.request.model") - .map(|model| serde_json::json!({"model": model})) - }); + let (scope_type, metadata, input, output) = self.projection(); let mut scope = runtime.scope_at( self.name, - relay_scope_type(self.name), + scope_type, None, Some(&metadata), input.as_ref(), @@ -234,6 +216,69 @@ impl CapturedSpan { let close_result = scope.close_at(output.as_ref(), Some(&metadata), self.ended_at); child_result.and(close_result) } + + fn projection(&self) -> (ScopeType, Json, Option, Option) { + let mut metadata = Json::Object(self.fields.clone()); + if let Some(session) = metadata.get("gen_ai.conversation.id").cloned() { + metadata["session_id"] = session; + } + if let Some(status) = span_status(&metadata) { + metadata["otel.status_code"] = status["otel.status_code"].clone(); + } + let unit = metadata + .get("switchyard.accounting.unit") + .and_then(Json::as_str); + let summary = unit == Some("summary"); + let attempt = unit == Some("completion_attempt"); + let scope_type = if summary { + ScopeType::Custom + } else if attempt { + ScopeType::Llm + } else { + relay_scope_type(self.name) + }; + let input = request_data(&metadata, self.payloads.request.as_ref()); + let output = if summary { + metadata["switchyard.accounting.attempt_records"] = Json::from( + self.children + .iter() + .filter(|child| { + child + .fields + .get("switchyard.accounting.unit") + .and_then(Json::as_str) + == Some("completion_attempt") + }) + .count(), + ); + None + } else if attempt || self.name == CLIENT_CALL { + // Debug payloads supplement accounting, rather than replacing its usage. + let mut output = response_data(&metadata); + if let Some(payload) = &self.payloads.response { + let value = output.get_or_insert_with(|| serde_json::json!({})); + if let (Some(target), Some(source)) = (value.as_object_mut(), payload.as_object()) { + target.extend(source.clone()); + } + } + output + } else { + self.payloads.response.clone() + }; + (scope_type, metadata, input, output) + } +} + +// Preserve the requested model alongside an optional debug payload. +fn request_data(fields: &Json, payload: Option<&Json>) -> Option { + let mut input = payload.cloned().unwrap_or_else(|| serde_json::json!({})); + if let Some(model) = fields + .get("gen_ai.request.model") + .or_else(|| fields.get("selected_model")) + { + input["model"] = model.clone(); + } + (!input.as_object()?.is_empty()).then_some(input) } fn redact_captured_json(value: Json, redactor: &ProviderKeyRedactor, depth: usize) -> Option { @@ -291,9 +336,17 @@ fn response_data(fields: &Json) -> Option { output.insert(target.into(), value.clone()); } } + if !output.contains_key("model") + && let Some(model) = fields + .get("gen_ai.request.model") + .or_else(|| fields.get("selected_model")) + { + output.insert("model".into(), model.clone()); + } let mut usage = Map::new(); for (source, target) in [ ("gen_ai.usage.output_tokens", "output_tokens"), + ("gen_ai.usage.reasoning.output_tokens", "reasoning_tokens"), ( "gen_ai.usage.cache_read.input_tokens", "cache_read_input_tokens", @@ -307,6 +360,12 @@ fn response_data(fields: &Json) -> Option { usage.insert(target.into(), value.clone()); } } + if let Some(cost) = fields.get("switchyard.usage.provider_cost_usd") { + usage.insert( + "cost".into(), + serde_json::json!({"total": cost, "currency": "USD", "source": "provider_reported"}), + ); + } if let Some(value) = fields.get("gen_ai.usage.cache_creation_1h.input_tokens") { usage.insert( "cache_creation".into(), @@ -336,8 +395,8 @@ fn response_data(fields: &Json) -> Option { if output.is_empty() { return None; } - // Relay decodes scope output with provider codecs. A content-free Messages - // response preserves separate cache-read and cache-creation token counts. + // A consumer can decode this content-free Messages response with Relay's + // codec, preserving separate cache-read and cache-creation token counts. output.insert("type".into(), Json::String("message".into())); output.insert("role".into(), Json::String("assistant".into())); output.insert("content".into(), Json::Array(Vec::new())); @@ -521,7 +580,13 @@ fn relay_scope_type(name: &str) -> ScopeType { fn is_safe_field(name: &str) -> bool { matches!( name, - "algorithm" + "switchyard.accounting.unit" + | "switchyard.accounting.attempts" + | "switchyard.accounting.usage_status" + | "switchyard.accounting.charge_source" + | "switchyard.usage.total_tokens" + | "switchyard.usage.provider_cost_usd" + | "algorithm" | "switchyard.candidate" | "switchyard.candidate_count" | "switchyard.call_phase" @@ -1002,3 +1067,7 @@ mod tests { assert_eq!(attempt.fields["outcome"], "cancelled"); } } + +#[cfg(test)] +#[path = "attempt_tests.rs"] +mod attempt_tests; diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 18658bf69a..330aac3f05 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -1259,7 +1259,7 @@ fn encode_anthropic_tool_choice(choice: &ToolChoice) -> Value { } // Normalizes Anthropic usage fields. -fn decode_anthropic_usage(value: Option<&Value>) -> Usage { +pub(crate) fn decode_anthropic_usage(value: Option<&Value>) -> Usage { let Some(value) = value.and_then(Value::as_object) else { return Usage::default(); }; diff --git a/crates/switchyard-translation/src/codecs/anthropic/mod.rs b/crates/switchyard-translation/src/codecs/anthropic/mod.rs index d990cd988c..61701d33ff 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/mod.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/mod.rs @@ -8,3 +8,5 @@ mod stream; pub use buffered::AnthropicMessagesCodec; pub use stream::AnthropicMessagesStreamCodec; + +pub(crate) use buffered::decode_anthropic_usage; diff --git a/crates/switchyard-translation/src/codecs/mod.rs b/crates/switchyard-translation/src/codecs/mod.rs index 3b36a0bb82..96babe04bb 100644 --- a/crates/switchyard-translation/src/codecs/mod.rs +++ b/crates/switchyard-translation/src/codecs/mod.rs @@ -68,3 +68,16 @@ pub trait FormatCodec: Send + Sync { policy: &TranslationPolicy, ) -> Result; } + +/// Extracts reported usage independently of generation success or content validity. +/// +/// Only the provider's top-level usage object is inspected. Missing or invalid +/// counters remain unknown; the response is not accepted as a successful result. +pub fn response_usage(body: &Value, format: crate::WireFormat) -> crate::Usage { + let usage = body.get("usage"); + match format { + crate::WireFormat::OpenAiChat => openai_chat::decode_openai_usage(usage), + crate::WireFormat::OpenAiResponses => responses::decode_responses_usage(usage), + crate::WireFormat::AnthropicMessages => anthropic::decode_anthropic_usage(usage), + } +} diff --git a/crates/switchyard-translation/src/codecs/openai_chat/mod.rs b/crates/switchyard-translation/src/codecs/openai_chat/mod.rs index 766f312c6f..590fb45749 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/mod.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/mod.rs @@ -10,3 +10,5 @@ pub use buffered::OpenAiChatCodec; pub use stream::OpenAiChatStreamCodec; pub(crate) use buffered::{decode_file_source, decode_image_source}; + +pub(crate) use buffered::decode_openai_usage; diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index a3558c1477..dc47c48bdd 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -2065,7 +2065,7 @@ fn json_string_python_style(value: &Value) -> String { } // Normalizes Responses usage fields. -fn decode_responses_usage(value: Option<&Value>) -> Usage { +pub(crate) fn decode_responses_usage(value: Option<&Value>) -> Usage { let Some(value) = value.and_then(Value::as_object) else { return Usage::default(); }; diff --git a/crates/switchyard-translation/src/codecs/responses/mod.rs b/crates/switchyard-translation/src/codecs/responses/mod.rs index 8f18e26fb0..8094b249ee 100644 --- a/crates/switchyard-translation/src/codecs/responses/mod.rs +++ b/crates/switchyard-translation/src/codecs/responses/mod.rs @@ -76,3 +76,5 @@ fn decode_anthropic_thinking(payload: &str) -> Option<(String, String)> { pub fn is_anthropic_thinking_envelope(encrypted_content: &str) -> bool { encrypted_content.starts_with(ANTHROPIC_THINKING_PREFIX) } + +pub(crate) use buffered::decode_responses_usage; diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs index b726bea71a..2be6af0d91 100644 --- a/crates/switchyard-translation/src/codecs/responses/stream.rs +++ b/crates/switchyard-translation/src/codecs/responses/stream.rs @@ -304,19 +304,35 @@ fn decode_responses_stream( out.push(LlmResponseChunk::MessageStop { reason }); out } - // Carries the Anthropic spelling because every encoder already maps it. - Some("response.incomplete") => vec![LlmResponseChunk::MessageStop { - reason: Some("max_tokens".to_string()), - }], - Some("response.failed") => vec![LlmResponseChunk::StreamError { - message: event + Some(kind @ ("response.incomplete" | "response.failed")) => { + let mut out = Vec::new(); + if let Some(value) = event .get("response") - .and_then(|response| response.get("error")) - .and_then(|error| error.get("message")) - .and_then(Value::as_str) - .unwrap_or("unknown Responses stream error") - .to_string(), - }], + .and_then(|response| response.get("usage")) + .and_then(Value::as_object) + { + let usage = responses_usage(value); + state.usage = usage.clone(); + state.saw_backend_usage = true; + out.push(LlmResponseChunk::Usage(usage)); + } + out.push(if kind == "response.incomplete" { + LlmResponseChunk::MessageStop { + reason: Some("max_tokens".into()), + } + } else { + LlmResponseChunk::StreamError { + message: event + .get("response") + .and_then(|response| response.get("error")) + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .unwrap_or("unknown Responses stream error") + .to_string(), + } + }); + out + } Some("error") => vec![LlmResponseChunk::StreamError { message: event .get("message") @@ -1176,7 +1192,11 @@ fn responses_usage(usage: &serde_json::Map) -> Usage { total_tokens: usage .get("total_tokens") .and_then(Value::as_u64) - .or_else(|| Some(aggregate_input_tokens.unwrap_or(0) + output_tokens.unwrap_or(0))), + .or_else(|| { + aggregate_input_tokens + .zip(output_tokens) + .map(|(input, output)| input + output) + }), reasoning_tokens: usage .get("output_tokens_details") .and_then(|details| details.get("reasoning_tokens")) diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs index 3b23417cc9..7067dc30d4 100644 --- a/crates/switchyard-translation/src/lib.rs +++ b/crates/switchyard-translation/src/lib.rs @@ -16,6 +16,7 @@ mod sse; pub mod stream; pub mod util; +pub use codecs::response_usage; pub use codecs::responses::is_anthropic_thinking_envelope; pub use switchyard_protocol::stream::{ LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError, diff --git a/crates/switchyard-translation/tests/failure_usage.rs b/crates/switchyard-translation/tests/failure_usage.rs new file mode 100644 index 0000000000..afd76686c7 --- /dev/null +++ b/crates/switchyard-translation/tests/failure_usage.rs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use futures::{StreamExt, stream}; +use serde_json::json; +use switchyard_translation::{LlmResponseChunk, WireFormat, decode_stream, response_usage}; + +#[test] +fn failure_usage_reuses_provider_normalization_and_preserves_unknowns() { + let body = json!({"status":"failed", "error":{"message":"failed"}, "usage":{"input_tokens":20,"input_tokens_details":{"cached_tokens":3,"cache_write_tokens":2},"output_tokens":4,"cost":0.0}}); + let usage = response_usage(&body, WireFormat::OpenAiResponses); + assert_eq!(usage.input_tokens, Some(15)); + assert_eq!(usage.cached_input_tokens(), Some(3)); + assert_eq!(usage.cache_creation_input_tokens(), Some(2)); + assert_eq!(usage.total_tokens, Some(24)); + assert_eq!(usage.provider_cost, Some(0.0)); + assert_eq!( + response_usage( + &json!({"error":{"usage":{"input_tokens":10}}}), + WireFormat::OpenAiResponses + ), + Default::default() + ); + assert_eq!( + response_usage(&json!({"usage":{}}), WireFormat::AnthropicMessages), + Default::default() + ); +} + +#[test] +fn responses_terminal_failures_preserve_usage_and_original_event() { + futures::executor::block_on(async { + for kind in ["response.failed", "response.incomplete"] { + let raw = json!({"type":kind,"response":{"id":"r1","status":kind.strip_prefix("response.").unwrap(),"error":{"message":"failed"},"usage":{"input_tokens":11,"output_tokens":3}}}); + let bytes = format!("data: {raw}\n\n").into_bytes(); + let mut events = decode_stream( + stream::iter(vec![Ok(bytes)]).boxed(), + WireFormat::OpenAiResponses, + ) + .unwrap(); + let event = events.next().await.unwrap().unwrap(); + assert_eq!(event.preservation().unwrap().raw(), &raw); + let [LlmResponseChunk::Usage(usage), terminal] = event.normalized() else { + panic!("usage must precede terminal bookkeeping: {event:?}") + }; + assert_eq!(usage.input_tokens, Some(11)); + assert_eq!(usage.output_tokens, Some(3)); + assert!(matches!( + (kind, terminal), + ("response.failed", LlmResponseChunk::StreamError { .. }) + | ("response.incomplete", LlmResponseChunk::MessageStop { .. }) + )); + } + }); +} diff --git a/docs/integrations/nemo_relay.md b/docs/integrations/nemo_relay.md index ccce9e0860..dd32c7560b 100644 --- a/docs/integrations/nemo_relay.md +++ b/docs/integrations/nemo_relay.md @@ -58,34 +58,54 @@ A routed request can spend tokens choosing a model and generating the answer. Some routes call a judge or classifier first. The selected model, or a fallback, then answers the request. -Relay records the request and answer seen by the agent. It can record a -provider-reported cost or estimate one when model pricing is configured. -Switchyard separately records model calls, available token usage and timing, -fallbacks, and failures. - -The integration does not calculate a combined cost or savings figure. Relay can -price the answer returned to the agent, while Switchyard reports internal -routing usage separately. To estimate the observed cost for a workload, use: - -```text -Relay answer cost + cost of Switchyard tokens where call_role = "routing" +Switchyard exports one `libsy.upstream_attempt` scope for each dispatched +completion attempt, including retries, routing calls, and fallback candidates. +Each scope retains available usage and supported provider charges when the +attempt fails or its stream is dropped. Local validation failures and deadlines +that expire before dispatch create no attempt. + +Use these scopes as the accounting source for Switchyard completion work: + +1. Select end events with `switchyard.accounting.unit = "completion_attempt"` + in the workflow's scope tree. Count each event UUID once. +2. Decode each scope's content-free Messages-shaped output with Relay's + `AnthropicMessagesCodec`. Relay uses reported charges first and configured + model prices otherwise. Price each attempt with its own model. +3. Exclude `libsy.client_call` summaries, token metrics, and Relay's outer answer + from this total. They describe work already represented by the attempts. +4. Check each summary's expected `switchyard.accounting.attempts` against + `switchyard.accounting.attempt_records` and the received attempt scopes. + Missing records leave coverage incomplete. + +The [ATOF cost-summary example](../../crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py) +implements this selection and uses Relay's existing codec and pricing resolver. +Import `summarize(events, workflow_uuid)` into a host with Relay model pricing +configured to obtain estimates. For recorded provider charges, it also works +as a command: + +```sh +python crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py events.jsonl \ + --workflow-uuid ``` -Apply the price of each `target_model` to those routing tokens. Do not add -`call_role = "answer"` again because Relay already counted that answer. Do not -sum every `token_type`; `total` is a rollup, and cache or reasoning values may -already be included in broader counts. - -Relay does not copy request identity into exported metric attributes. Use this -calculation for a workload or time window, not to reconcile one trace. - -Relay does not include a model-price catalog. Follow its -[model-pricing guide](https://docs.nvidia.com/nemo/relay/configure-plugins/model-pricing) -to supply and validate model rates. Run the same representative workload once -with a fixed model and once with routing. Compare cost, latency, fallback rate, -and task success. Missing usage, failed attempts, internal HTTP retries, and -incomplete streams can leave some cost unknown. An absent value means unknown, -not zero. +The result separates `known_cost_subtotal_usd` from nullable `total_cost_usd`. +For example, a failed `$0.02` attempt followed by a successful `$0.03` attempt +has a `$0.05` total. If the failed attempt has unknown consumption, the known +subtotal is `$0.03` and the complete total is absent. A reported `$0` charge is +valid even when token usage is missing. + +`switchyard.accounting.usage_status` is `complete`, `partial`, or `missing`. +Cumulative usage snapshots replace known fields; repeated snapshots are not +added together. Cache and reasoning fields can overlap other token counts. +Token coverage and charge coverage are separate: a known provider charge can +cover an attempt whose token usage is incomplete. + +Relay's [model-pricing guide](https://docs.nvidia.com/nemo/relay/configure-plugins/model-pricing) +explains how to supply model rates. The cost-summary example covers instrumented +completion attempts. Separate decision-model calls are not yet exported with +per-call usage. ATIF `final_metrics` still describes Relay's outer interaction; +it does not automatically include this retry total. Provider work after a +disconnect and missing telemetry can leave costs unknown. ## How Requests Flow @@ -305,8 +325,8 @@ can receive these records, but each output format presents them differently. | Output | What the current integration shows | | --- | --- | | ATIF | The request and response seen by the agent. Internal routing is not added as separate steps or included in `final_metrics`. Relay-managed ATIF files may retain the raw records under `extra.observed_events`. | -| OpenTelemetry traces, including OpenInference | Relay's outer LLM span, nested `libsy.client_call` and `libsy.upstream_attempt` duration scopes, plus eligible routing marks. | -| OpenTelemetry metrics | Switchyard request, routing-call, latency, token, and failure measurements. Combine the reported routing usage with Relay's answer cost to estimate the observed routed cost. | +| OpenTelemetry traces, including OpenInference | Relay's outer LLM span, nested `libsy.client_call` summaries and `libsy.upstream_attempt` accounting scopes, plus eligible routing marks. | +| OpenTelemetry metrics | Switchyard request, routing-call, latency, token, and failure measurements. Aggregate model-call measurements; use attempt scopes for retry cost accounting. | | OTLP logs | Non-metric Switchyard marks that meet the configured severity threshold. | The plugin captures two existing Switchyard spans. Each routing or answer LLM diff --git a/docs/internal/req3_attempt_accounting_implementation.md b/docs/internal/req3_attempt_accounting_implementation.md new file mode 100644 index 0000000000..bcbbd4a996 --- /dev/null +++ b/docs/internal/req3_attempt_accounting_implementation.md @@ -0,0 +1,84 @@ +# Per-attempt accounting implementation + +Implemented on `bbednarski/req3-attempt-accounting`, based on upstream main +`cf92c1135`. The changes are pending review and commit. + +The client retains accounting until each dispatched completion attempt ends. +Its guard preserves usage across HTTP failures, decoding, routing buffering, +streaming errors, timeouts, and cancellation. Usage is cumulative by field. +Missing values remain unknown. Local failures before dispatch have no attempt. + +The native plugin exports priceable attempt scopes, with content-free usage +and bounded error categories. Logical client calls become summary scopes with +expected and captured attempt counts. Existing payload capture remains optional +and supplements the accounting data. + +## Scope + +```mermaid +flowchart TB + Core["libsy / runner algorithms"] --> Client + Protocol["Existing protocol Usage and Response types"] -.-> Guard + Transport["Existing HTTP transport"] <--> Client + subgraph LlmClient["switchyard-llm-client"] + Client["Augment client.rs / run.rs / observability.rs"] --> Guard["NEW attempt.rs guard and stream ownership"] + end + Client <--> Translation["Augment switchyard-translation
Preserve supported failure usage"] + Guard --> Capture["Augment native plugin span_capture.rs
Usage, outcome, summary counts, shared projections"] + Capture --> Relay["Existing Relay SDK and pricing codecs"] + Relay --> Consumer["NEW ATOF consumer example
Unique attempts, priced subtotal, nullable total"] + Consumer --> Fixture["NEW native acceptance fixture"] + classDef unchanged fill:#dcfce7,stroke:#15803d,color:#14532d; + classDef augmented fill:#fef3c7,stroke:#b45309,color:#78350f; + classDef added fill:#fee2e2,stroke:#b91c1c,color:#7f1d1d; + class Core,Protocol,Transport,Relay unchanged; + class Client,Translation,Capture augmented; + class Guard,Consumer,Fixture added; +``` + +Three production crates change. The translation changes reuse provider usage +parsers and retain usage in Responses failed/incomplete terminal frames. +Protocol types, retry policy, and decision transports stay unchanged. + +## Relay integration result + +Tested a packaged macOS arm64 native bundle against `nemo-relay==0.10.0` and +native SDK `0.10.0`, with a loopback provider and fixed synthetic rates. + +Generic native scopes retain their raw output, but do not acquire a normalized +pricing annotation automatically. Relay's outer answer also retains the final +response charge. The smallest follow-up is an opt-in ATOF consumer example that +selects unique attempt end events, decodes their output with Relay's existing +codec, and checks declared record coverage. It adds no price catalog or pricing +formula. No Relay source changes are required for this output. + +Relay 0.10 ignores the scalar `usage.provider_cost` field. The plugin uses the +supported detailed `usage.cost` object so reported charges take precedence over +model-price estimates. + +| Fixture | Known subtotal | Complete total | +| --- | --- | --- | +| Failed reported $0.02, successful reported $0.03 | $0.05 | $0.05 | +| Failed usage missing, successful reported $0.03 | $0.03 | Absent | +| Both attempts priced by configured token rates | $0.028 | $0.028 | +| Usage known, model price missing | $0 | Absent | +| Both attempts report $0, tokens missing | $0 | $0 | + +The fixture also verifies duplicate-event delivery, missing attempt records, +reported-charge precedence over headers, and separation between workflows. +The Rust suites cover routing buffer failure, terminal stream errors, trailing +and repeated usage snapshots, timeout, abandonment, pre-dispatch deadlines, +concurrent requests, debug payload compatibility, and credential redaction. + +## Validation and remaining boundaries + +The full workspace Rust tests, formatting, and Clippy pass. The repository +pre-commit hooks pass. Optional non-blocking mypy/pyright commands are unavailable +in this environment; those hooks report that they did not execute. +The new Python scripts also pass a direct Ruff check and native acceptance. +See the plugin README for the reproducible native fixture command. + +These totals cover observed instrumented completion attempts. Separate decision +calls remain Req 4. ATIF `final_metrics` is unchanged; use the supplied ATOF +summary for retry totals. Provider work continuing after disconnection, process +termination, and wholly absent telemetry cannot be reconciled from these records. From f48197a002fb1ffe75462b25b51dc641a3bd3260 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 4 Nov 2026 10:50:07 -0800 Subject: [PATCH 2/4] fix(relay): exclude non-USD costs from attempt totals Signed-off-by: Bryan Bednarski --- .../examples/attempt_costs.py | 5 +- .../scripts/test_attempt_costs.py | 73 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py diff --git a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py index 7c965178b9..1f4167208b 100644 --- a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py +++ b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py @@ -60,7 +60,10 @@ def belongs(event): usage = codecs.AnthropicMessagesCodec().decode_response(event["data"]).usage or {} cost = usage.get("cost", {}) amount = cost.get("total") - priced = isinstance(amount, (float, int)) and math.isfinite(amount) and amount >= 0 + priced = ( + cost.get("currency", "USD").upper() == "USD" + and isinstance(amount, (float, int)) and math.isfinite(amount) and amount >= 0 + ) complete = priced and ( cost.get("source") == "provider_reported" or meta.get("switchyard.accounting.usage_status") == "complete" diff --git a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py new file mode 100644 index 0000000000..2abccd73c0 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Check USD accounting with Relay's response codec, without loading a plugin.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "examples")) +from attempt_costs import summarize # noqa: E402 + + +def report(*costs): + events = [ + {"kind": "scope", "uuid": "workflow", "scope_category": "end"}, + { + "kind": "scope", "uuid": "call", "parent_uuid": "workflow", + "name": "libsy.client_call", "scope_category": "end", + "metadata": { + "switchyard.accounting.unit": "summary", + "switchyard.accounting.attempts": len(costs), + "switchyard.accounting.attempt_records": len(costs), + }, + }, + ] + for ordinal, cost in enumerate(costs, 1): + events.append({ + "kind": "scope", "uuid": f"attempt-{ordinal}", "parent_uuid": "call", + "scope_category": "end", + "metadata": { + "attempt": ordinal, "switchyard.accounting.unit": "completion_attempt", + "switchyard.accounting.usage_status": "complete", "outcome": "ok", + }, + "data": { + "type": "message", "role": "assistant", "content": [], "model": "test/model", + "usage": {"input_tokens": 1, "output_tokens": 1, "cost": cost}, + }, + }) + return summarize(events, "workflow") + + +class CurrencyTests(unittest.TestCase): + def test_usd_and_missing_currency_keep_complete_total(self): + for currency in ["USD", "usd", None]: + with self.subTest(currency=currency): + cost = {"total": 0.02} + if currency is not None: + cost["currency"] = currency + result = report(cost) + self.assertEqual(result["known_cost_subtotal_usd"], 0.02) + self.assertEqual(result["total_cost_usd"], 0.02) + self.assertEqual(result["cost_status"], "complete") + + def test_non_usd_amounts_do_not_contribute_to_usd_totals(self): + for currency, amount in [("EUR", 0.02), ("JPY", 3), ("EUR", 0.0)]: + with self.subTest(currency=currency, amount=amount): + result = report({"total": amount, "currency": currency}) + self.assertTrue(result["coverage_complete"]) + self.assertEqual(result["known_cost_subtotal_usd"], 0.0) + self.assertIsNone(result["total_cost_usd"]) + self.assertEqual(result["cost_status"], "incomplete") + self.assertFalse(result["attempts"][0]["cost_complete"]) + + def test_mixed_currencies_keep_only_the_usd_subtotal(self): + result = report({"total": 0.03, "currency": "USD"}, {"total": 0.02, "currency": "EUR"}) + self.assertEqual(result["known_cost_subtotal_usd"], 0.03) + self.assertIsNone(result["total_cost_usd"]) + self.assertEqual(result["cost_status"], "incomplete") + + +if __name__ == "__main__": + unittest.main() From 9f1853a21cddb4674521e8e17de9aaa17dabd094 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 4 Nov 2026 17:15:16 -0800 Subject: [PATCH 3/4] feat(relay): export decision usage and workflow attribution Signed-off-by: Bryan Bednarski --- crates/libsy-llm-client/src/attempt.rs | 34 +-- crates/libsy-llm-client/src/observability.rs | 35 +++ crates/libsy-llm-client/src/run.rs | 114 ++++++++- .../libsy-llm-client/tests/observability.rs | 138 +++++++++++ crates/switchyard-nemo-relay-plugin/README.md | 17 +- .../examples/attempt_costs.py | 34 ++- .../scripts/test_attempt_accounting.py | 186 ++++++++++++++- .../scripts/test_attempt_costs.py | 58 ++++- .../src/decision_tests.rs | 218 ++++++++++++++++++ .../src/span_capture.rs | 26 ++- docs/integrations/nemo_relay.md | 49 +++- ...req4_decision_accounting_implementation.md | 122 ++++++++++ 12 files changed, 961 insertions(+), 70 deletions(-) create mode 100644 crates/switchyard-nemo-relay-plugin/src/decision_tests.rs create mode 100644 docs/internal/req4_decision_accounting_implementation.md diff --git a/crates/libsy-llm-client/src/attempt.rs b/crates/libsy-llm-client/src/attempt.rs index ea3fa78dc2..6537ebfff0 100644 --- a/crates/libsy-llm-client/src/attempt.rs +++ b/crates/libsy-llm-client/src/attempt.rs @@ -14,7 +14,7 @@ use switchyard_protocol::{ }; use tracing::Span; -use crate::observability::{llm_client_error_type, record_gen_ai_usage}; +use crate::observability::{llm_client_error_type, record_accounting_usage}; pub(crate) struct AttemptGuard { span: Span, @@ -198,37 +198,7 @@ impl AttemptGuard { if !self.accounting { return; } - record_gen_ai_usage(&self.span, &self.usage); - if let Some(total) = self.usage.total_tokens { - self.span.record("switchyard.usage.total_tokens", total); - } - if let Some(cost) = self.usage.provider_cost { - self.span.record("switchyard.usage.provider_cost_usd", cost); - } - let known = self.usage.input_tokens.is_some() - || self.usage.output_tokens.is_some() - || self.usage.cached_input_tokens().is_some() - || self.usage.cache_creation_input_tokens().is_some() - || self - .usage - .cache - .as_ref() - .is_some_and(|cache| cache.cache_creation_1h_input_tokens.is_some()) - || self.usage.total_tokens.is_some() - || self.usage.reasoning_tokens.is_some(); - let complete = self.final_usage - && self.usage.input_tokens.is_some() - && self.usage.output_tokens.is_some(); - self.span.record( - "switchyard.accounting.usage_status", - if complete { - "complete" - } else if known { - "partial" - } else { - "missing" - }, - ); + record_accounting_usage(&self.span, &self.usage, self.final_usage); } pub(crate) fn wrap(self, stream: LlmResponseStream) -> LlmResponseStream { diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs index 695f4bcb92..08814d7359 100644 --- a/crates/libsy-llm-client/src/observability.rs +++ b/crates/libsy-llm-client/src/observability.rs @@ -147,6 +147,41 @@ pub(crate) fn record_gen_ai_usage(span: &Span, usage: &Usage) { } } +/// Records normalized consumption and whether its final report is available. +pub(crate) fn record_accounting_usage(span: &Span, usage: &Usage, final_report: bool) { + record_gen_ai_usage(span, usage); + if let Some(total) = usage.total_tokens { + span.record("switchyard.usage.total_tokens", total); + } + if let Some(cost) = usage + .provider_cost + .filter(|cost| cost.is_finite() && *cost >= 0.0) + { + span.record("switchyard.usage.provider_cost_usd", cost); + } + let known = usage.input_tokens.is_some() + || usage.output_tokens.is_some() + || usage.cached_input_tokens().is_some() + || usage.cache_creation_input_tokens().is_some() + || usage + .cache + .as_ref() + .is_some_and(|cache| cache.cache_creation_1h_input_tokens.is_some()) + || usage.total_tokens.is_some() + || usage.reasoning_tokens.is_some(); + let complete = final_report && usage.input_tokens.is_some() && usage.output_tokens.is_some(); + span.record( + "switchyard.accounting.usage_status", + if complete { + "complete" + } else if known { + "partial" + } else { + "missing" + }, + ); +} + // OpenTelemetry integer attributes are signed; token counts are unsigned in the IR. fn otel_int(value: u64) -> i64 { value.min(i64::MAX as u64) as i64 diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index dacbf1a2c8..47274655ac 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -63,6 +63,7 @@ pub async fn run( let algorithm_name = algorithm.name().to_string(); let run_started = Instant::now(); let routing_clients = clients.clone(); + let decision_context = DecisionContext::from_request(&request); // This says if we have an observer, put Some(..) in routing_observations. // No observer means we don't want any routing_observations. let routing_observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new()))); @@ -71,7 +72,14 @@ pub async fn run( None => { drive(Arc::clone(&algorithm), request, models, { let routing_observations = routing_observations.clone(); - move |call| serve(routing_clients.clone(), call, routing_observations.clone()) + move |call| { + serve( + routing_clients.clone(), + call, + routing_observations.clone(), + decision_context.clone(), + ) + } }) .await } @@ -202,11 +210,17 @@ pub async fn decide( models: Arc, ) -> Result { let routing_clients = clients.clone(); + let decision_context = DecisionContext::from_request(&request); let mut outcome = match clients.stored_state_owner(&request) { Some(owner) => continue_on(owner, algorithm.name(), request), None => { drive(algorithm, request, models, move |call| { - serve(routing_clients.clone(), call, None) + serve( + routing_clients.clone(), + call, + None, + decision_context.clone(), + ) }) .await? } @@ -220,23 +234,110 @@ pub async fn decide( Ok(outcome) } +#[derive(Clone, Default)] +struct DecisionContext { + session_id: Option, + task_id: Option, + turn_id: Option, + correlation_id: Option, +} + +impl DecisionContext { + fn from_request(request: &Request) -> Self { + request + .metadata + .as_ref() + .map_or_else(Self::default, |metadata| Self { + session_id: metadata.session_id.clone(), + task_id: metadata.task_id.clone(), + turn_id: metadata.turn_id.clone(), + correlation_id: metadata.correlation_id.clone(), + }) + } + + fn record(&self, span: &tracing::Span) { + for (field, value) in [ + ("gen_ai.conversation.id", &self.session_id), + ("task_id", &self.task_id), + ("turn_id", &self.turn_id), + ("correlation_id", &self.correlation_id), + ] { + if let Some(value) = value { + span.record(field, value.as_str()); + } + } + } +} + #[tracing::instrument( + target = "libsy", name = "libsy.decision_client_call", skip_all, - fields(algorithm = %call.algorithm, selected_model = %call.model, outcome = tracing::field::Empty), + fields( + algorithm = %call.algorithm, selected_model = %call.model, gen_ai.request.model = %call.model, + switchyard.accounting.unit = "decision_call", switchyard.accounting.usage_status = "missing", + gen_ai.conversation.id = tracing::field::Empty, task_id = tracing::field::Empty, + turn_id = tracing::field::Empty, correlation_id = tracing::field::Empty, + gen_ai.response.id = tracing::field::Empty, gen_ai.response.model = tracing::field::Empty, + gen_ai.usage.input_tokens = tracing::field::Empty, gen_ai.usage.output_tokens = tracing::field::Empty, + gen_ai.usage.cache_read.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, + gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty, + gen_ai.usage.reasoning.output_tokens = tracing::field::Empty, + switchyard.usage.total_tokens = tracing::field::Empty, + switchyard.usage.provider_cost_usd = tracing::field::Empty, + switchyard.accounting.charge_source = tracing::field::Empty, + outcome = tracing::field::Empty, error.type = tracing::field::Empty, otel.status_code = tracing::field::Empty, + ), )] async fn serve_decision( clients: &ClientRouter, call: CallDecision, observations: &Option>>>, + context: &DecisionContext, ) -> Result<()> { + let span = tracing::Span::current(); + context.record(&span); let client = clients.route_decision(&call.model); let model = call.model.clone(); let request = call.request.clone(); call.respond(async { let started = Instant::now(); let result = async { client?.call(request).await }.await; - tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" }); + span.record("outcome", if result.is_ok() { "ok" } else { "error" }); + span.record( + "otel.status_code", + if result.is_ok() { "OK" } else { "ERROR" }, + ); + match &result { + Ok(response) => { + for (field, value) in [ + ("gen_ai.response.id", response.id.as_deref()), + ( + "gen_ai.response.model", + response.model.as_ref().map(ModelId::as_str), + ), + ] { + if let Some(value) = value { + span.record(field, value); + } + } + observability::record_accounting_usage(&span, &response.usage, true); + if response + .usage + .provider_cost + .is_some_and(|cost| cost.is_finite() && cost >= 0.0) + { + span.record("switchyard.accounting.charge_source", "provider"); + } + } + Err(error) => { + span.record( + "error.type", + observability::llm_client_error_type(error).as_ref(), + ); + } + } if let Some(observations) = observations { observations .lock() @@ -283,10 +384,13 @@ async fn serve( clients: ClientRouter, call: Call, observations: Option>>>, + decision_context: DecisionContext, ) -> Result<()> { let call = match call { Call::Model(call) => *call, - Call::Decision(call) => return serve_decision(&clients, *call, &observations).await, + Call::Decision(call) => { + return serve_decision(&clients, *call, &observations, &decision_context).await; + } }; let observe = |observation| { if let Some(observations) = &observations { diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 5602338820..1e4a6462c9 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -2586,3 +2586,141 @@ async fn target_aliases_preserve_provider_model_metric_labels() -> switchyard_li } Ok(()) } + +#[tokio::test] +async fn host_decision_accounting_records_invocations_instead_of_questions() +-> switchyard_libsy::Result<()> { + use switchyard_protocol::{ + DecisionKind, DecisionQuestion, DecisionRequest, DecisionResponse, RoutedDecisionClient, + }; + const MODEL: &str = "obs-host-decision-model"; + const ALGORITHM: &str = "obs-host-decision-algo"; + struct DecisionAlgo; + #[async_trait] + impl Algorithm for DecisionAlgo { + fn name(&self) -> &str { + ALGORITHM + } + async fn route( + self: Arc, + driver: Driver, + request: Request, + ) -> switchyard_libsy::Result { + for _ in 0..2 { + let questions = ["first", "second"] + .into_iter() + .map(|id| { + ( + id.into(), + DecisionQuestion { + instructions: json!("private question"), + kind: DecisionKind::Boolean { + true_description: None, + false_description: None, + }, + }, + ) + }) + .collect(); + driver + .call_decision( + DecisionRequest { + model: None, + context: json!("private context"), + questions, + }, + MODEL.into(), + ) + .await?; + } + Ok(RoutingOutcome::route_to("answer".into(), vec![], request)) + } + } + struct DecisionClient; + #[async_trait] + impl RoutedDecisionClient for DecisionClient { + async fn call(&self, request: DecisionRequest) -> Result { + assert_eq!(request.questions.len(), 2); + Ok(DecisionResponse { + id: Some("host-decision-id".into()), + model: None, + answers: Default::default(), + usage: Usage { + input_tokens: Some(6), + output_tokens: Some(2), + provider_cost: Some(0.01), + ..Default::default() + }, + }) + } + } + let _guard = serialize_test().lock().await; + let (store, exporter, provider, _, _) = telemetry(); + let clients = ClientRouter::single_with_decision_clients( + Arc::new(UsageClient { + usage: Usage::default(), + }), + [( + ModelId::from(MODEL), + Arc::new(DecisionClient) as Arc, + )] + .into(), + ); + let observations = Arc::new(Mutex::new(Vec::new())); + let seen = observations.clone(); + switchyard_llm_client::run( + Arc::new(DecisionAlgo), + clients, + request_with_metadata("host-decision-session", "host-decision-correlation"), + Arc::new(RuntimeModels::default()), + Some(Arc::new(move |event| seen.lock().push(event))), + ) + .await?; + let spans = store.spans(); + let host: Vec<_> = spans + .iter() + .filter(|s| { + s.name == "libsy.decision_client_call" + && s.fields + .get("selected_model") + .is_some_and(|model| model == MODEL) + }) + .collect(); + assert_eq!(host.len(), 2); + for span in host { + assert_eq!(span.fields["switchyard.accounting.unit"], "decision_call"); + assert_eq!( + span.fields["switchyard.accounting.usage_status"], + "complete" + ); + assert_eq!(span.fields["gen_ai.usage.input_tokens"], "6"); + assert_eq!(span.fields["switchyard.usage.provider_cost_usd"], "0.01"); + assert_eq!( + span.fields["gen_ai.conversation.id"], + "host-decision-session" + ); + assert!(!format!("{span:?}").contains("private question")); + } + assert_eq!( + observations + .lock() + .iter() + .filter(|o| matches!(o, RunObservation::DecisionCall(_))) + .count(), + 2 + ); + let snapshots = flushed_metrics(exporter, provider); + assert_eq!( + u64_counter_value( + &snapshots, + "switchyard.decision_calls", + &[ + ("algorithm", ALGORITHM), + ("selected_model", MODEL), + ("outcome", "ok") + ] + ), + Some(2) + ); + Ok(()) +} diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 13fa0298de..76f0d4ffb4 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -338,5 +338,18 @@ cost summaries. Relay also loads system plugin configuration when present. The [cost-summary example](examples/attempt_costs.py) selects unique attempt records and uses Relay's codec to obtain costs. These summaries cover Switchyard -completion work. They do not change ATIF `final_metrics` or include separate -decision-model usage. +completion and typed decision work. They do not change ATIF `final_metrics`. + +Add `--include-decisions` to the fixture command to test explicit zero-rate and +paid decisions, missing usage and prices, reported zero charges, repeated +invocations, concurrent workflows, and native cancellation. Each workflow uses +its own Relay scope stack. The test checks actual native parent scopes, +including workflows without identity headers, and keeps decision marks as +coverage checks rather than charges. + +The focused consumer tests require Relay and no native bundle: + +```sh +uv run --with nemo-relay==0.10.0 python \ + crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py +``` diff --git a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py index 1f4167208b..95d58d7d72 100644 --- a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py +++ b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Price unique Switchyard attempts from one workflow's ATOF events using Relay.""" +"""Price Switchyard completion attempts and decision calls within one ATOF workflow.""" import argparse import json @@ -16,14 +16,14 @@ def summarize(events, workflow_uuid): Configure Relay's pricing component before calling this function to enable estimates. Provider-reported charges work without a pricing catalog. - This covers instrumented completion attempts, including routing calls. + This covers instrumented completion attempts and typed decision calls. """ scopes = {e["uuid"]: e for e in events if e.get("kind") == "scope"} - def belongs(event): + def belongs(event, roots=(workflow_uuid,)): seen = set() while event["uuid"] not in seen: - if event["uuid"] == workflow_uuid: + if event["uuid"] in roots: return True seen.add(event["uuid"]) event = scopes.get(event.get("parent_uuid")) @@ -41,7 +41,25 @@ def belongs(event): e for e in ends.values() if e.get("metadata", {}).get("switchyard.accounting.unit") == "completion_attempt" ] - coverage = bool(calls) and workflow_uuid in ends + decisions = [ + e for e in ends.values() + if e.get("metadata", {}).get("switchyard.accounting.unit") == "decision_call" + ] + known_workflows = {workflow_uuid} | {e["uuid"] for e in scopes.values() if e.get("category") == "agent"} + unattributed = { + e["uuid"]: {"uuid": e["uuid"], "context_status": "unattributed"} + for e in events if e.get("kind") == "scope" and e.get("scope_category") == "end" + and e.get("metadata", {}).get("switchyard.accounting.unit") == "decision_call" + and not belongs(e, known_workflows) + } + # Existing decision marks establish record coverage, but are never priced. + decision_marks = { + e["uuid"]: e for e in events + if e.get("kind") == "mark" and e.get("name") == "switchyard.routing.decision_call" and belongs(e) + } + coverage = bool(calls or decisions) and workflow_uuid in ends + coverage &= len(decisions) == len(decision_marks) + coverage &= not unattributed for call in calls: meta = ends.get(call["uuid"], {}).get("metadata", {}) children = [e for e in attempts if e.get("parent_uuid") == call["uuid"]] @@ -55,7 +73,7 @@ def belongs(event): ) coverage &= all(e.get("parent_uuid") in {c["uuid"] for c in calls} for e in attempts) records = [] - for event in attempts: + for event in attempts + decisions: meta = event["metadata"] usage = codecs.AnthropicMessagesCodec().decode_response(event["data"]).usage or {} cost = usage.get("cost", {}) @@ -72,12 +90,14 @@ def belongs(event): "uuid": event["uuid"], "model": event["data"].get("model"), "outcome": meta.get("outcome"), "usage_status": meta.get("switchyard.accounting.usage_status"), + "context_status": "bound", "cost": cost if priced else None, "cost_complete": complete, }) subtotal = math.fsum(r["cost"]["total"] for r in records if r["cost"] is not None) complete = coverage and all(r["cost_complete"] for r in records) return { - "workflow_uuid": workflow_uuid, "attempts": records, + "workflow_uuid": workflow_uuid, "attempts": records[:len(attempts)], "decisions": records[len(attempts):], + "unattributed_decisions": list(unattributed.values()), "coverage_complete": bool(coverage), "cost_status": "complete" if complete else "incomplete", "known_cost_subtotal_usd": subtotal, "total_cost_usd": subtotal if complete else None, } diff --git a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py index 0c5ae3983f..864d300104 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py @@ -7,10 +7,12 @@ import asyncio import json import math +import os import platform import sys import tempfile import threading +import time import tomllib from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from importlib.metadata import version @@ -26,13 +28,37 @@ class Provider(BaseHTTPRequestHandler): scenario = "reported" calls = 0 + cancel_started = threading.Event() def do_POST(self): request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if self.path in {"/decisions", "/systemone"}: + if self.scenario == "decision_cancel": + Provider.cancel_started.set() + time.sleep(0.2) + if request["model"].endswith("error"): + self.reply(503, {"error": "synthetic private failure"}) + return + time.sleep(0.02) + answer = {"type": "choice", "name": "route", "choice": "capable_first", "confidence": 1.0, + "probabilities": [{"value": "capable_first", "probability": 1.0}, {"value": "efficient_first", "probability": 0.0}]} + body = {"model": request["model"], "answers": [answer], "usage": { + "input_tokens": 6, "output_tokens": 2, "total_tokens": 8, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }} + if self.path == "/systemone": + body = {"answers": {"route": {"type": "choice", "choice": "capable_first", "confidence": 1.0, "probabilities": {"capable_first": 1.0, "efficient_first": 0.0}}}} + if request["model"].endswith("reported_zero"): + body["usage"] = {"provider_cost": 0.0} + self.reply(200, body) + return first = Provider.calls % 2 == 0 Provider.calls += 1 + if self.scenario.startswith("decision_"): + first = False usage = {"prompt_tokens": 11 if first else 7, "completion_tokens": 3 if first else 2} - if self.scenario in {"reported", "missing"}: + if self.scenario in {"reported", "missing"} or self.scenario.startswith("decision_"): usage["cost"] = 0.02 if first else 0.03 if self.scenario == "zero": usage = {"cost": 0.0} @@ -45,8 +71,11 @@ def do_POST(self): body.update(id="r1", choices=[{ "index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", }]) + self.reply(503 if first else 200, body) + + def reply(self, status, body): raw = json.dumps(body).encode() - self.send_response(503 if first else 200) + self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(raw))) self.send_header("Retry-After", "0") @@ -54,19 +83,23 @@ def do_POST(self): if self.scenario == "reported": self.send_header("x-litellm-response-cost", "1.0") self.end_headers() - self.wfile.write(raw) + try: + self.wfile.write(raw) + except (BrokenPipeError, ConnectionResetError): + pass def log_message(self, *_args): pass -async def run(manifest, output): +async def run(manifest, output, include_decisions=False): server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() events = [] subscribers.register("attempt-accounting-test", lambda e: events.append(e.to_dict())) activation = None + previous_key = os.environ.get("SWITCHYARD_REQ4_FIXTURE_KEY") try: deployment = output / "switchyard.toml" deployment.write_text(f'''schema_version = 1 @@ -89,6 +122,35 @@ async def run(manifest, output): id = "switchyard/unpriced" type = "passthrough" target = "unpriced" +''') + if include_decisions: + os.environ["SWITCHYARD_REQ4_FIXTURE_KEY"] = "synthetic-decision-key" + with deployment.open("a") as stream: + for client, wire, endpoint in [("judge", "openai_decisions", "decisions"), ("system", "system_one", "systemone")]: + stream.write(f'''\n[decision_clients.{client}] +format = "{wire}" +endpoint = "http://127.0.0.1:{server.server_port}/{endpoint}" +api_key_env = "SWITCHYARD_REQ4_FIXTURE_KEY" +timeout_ms = 2000 +''') + for name in ["free", "paid", "unpriced", "missing", "reported_zero", "error"]: + client = "system" if name in {"missing", "reported_zero"} else "judge" + stream.write(f'''\n[decision_targets.judge_{name}] +id = "test/decision-{name}" +decision_client = "{client}" +[routes.decision_{name}] +id = "switchyard/decision_{name}" +type = "llm_classifier" +classifier_target = "judge_{name}" +strong_target = "default" +weak_target = "unpriced" +fail_open = false +[routes.decision_{name}.decision] +cutoff = 0.5 +candidates = {{ capable_first = "default", efficient_first = "unpriced" }} +criteria = {{ capable_first = "Capable", efficient_first = "Efficient" }} +instructions = {{ task = "Choose" }} +evidence = {{ description = "Synthetic decision context" }} ''') config = output / "plugins.toml" config.write_text(f'''[[plugins.dynamic]] @@ -106,6 +168,13 @@ async def run(manifest, output): input_per_million=1000, output_per_million=2000, ), ), + *[model_pricing.ModelPricing( + provider="anthropic", model_id=f"test/decision-{name}", pricing_as_of="2026-11-04", + pricing_source="synthetic", rates=model_pricing.TokenPricingRates( + input_per_million=0 if name == "free" else 1000, + output_per_million=0 if name == "free" else 2000, + ), + ) for name in ["free", "paid", "missing"]], ])), ])) activation = await plugin.initialize(plugin.PluginConfig(components=[prices]), config) @@ -154,27 +223,132 @@ async def forbidden(_request): assert all(a["usage_status"] == "missing" for a in report["attempts"]) (output / f"{scenario}.json").write_text(json.dumps(report, indent=2)) print(f"{scenario}: subtotal={report['known_cost_subtotal_usd']}, total={report['total_cost_usd']}") + if include_decisions: + Provider.scenario = "decision_workflow" + for name, subtotal, total in [("free", 0.03, 0.03), ("paid", 0.04, 0.04), + ("unpriced", 0.03, None), ("missing", 0.03, None), ("reported_zero", 0.03, 0.03), ("error", 0.0, None)]: + await check_decisions(events, output, name, name, 1, subtotal, total, forbidden) + await check_decisions(events, output, "repeated", "paid", 2, 0.08, 0.08, forbidden) + await asyncio.gather( + check_decisions(events, output, "concurrent-a", "paid", 1, 0.04, 0.04, forbidden), + check_decisions(events, output, "concurrent-b", "paid", 1, 0.04, 0.04, forbidden), + ) + await check_cancelled_decision(events, output, forbidden) (output / "events.jsonl").write_text("".join(json.dumps(e) + "\n" for e in events)) finally: + (output / "events.jsonl").write_text("".join(json.dumps(e) + "\n" for e in events)) if activation is not None: await activation.close() subscribers.deregister("attempt-accounting-test") server.shutdown() server.server_close() thread.join() + if previous_key is None: + os.environ.pop("SWITCHYARD_REQ4_FIXTURE_KEY", None) + else: + os.environ["SWITCHYARD_REQ4_FIXTURE_KEY"] = previous_key + + +async def check_decisions(events, output, label, model, count, subtotal, total, forbidden): + with relay.use_scope_stack(relay.create_scope_stack()): + await check_decisions_in_scope(events, output, label, model, count, subtotal, total, forbidden) + + +async def check_decisions_in_scope(events, output, label, model, count, subtotal, total, forbidden): + headers = {} if label == "concurrent-a" else {"x-switchyard-task-id": f"task-{label}", "x-switchyard-turn-id": "turn-1", "x-switchyard-request-id": f"request-{label}"} + with relay.scope.scope(f"decision-{label}", relay.ScopeType.Agent): + for _ in range(count): + try: + await relay.llm.execute("openai.chat_completions", relay.LLMRequest(headers, { + "model": f"switchyard/decision_{model}", "messages": [{"role": "user", "content": "synthetic decision request"}], + }), forbidden, codec=codecs.OpenAIChatCodec(), response_codec=codecs.OpenAIChatCodec()) + except RuntimeError: + if model != "error": + raise + else: + assert model != "error", "expected a decision error" + await subscribers.flush_async() + workflow = next(e["uuid"] for e in events if e.get("name") == f"decision-{label}" and e.get("scope_category") == "start") + report = summarize(events, workflow) + assert len(report["decisions"]) == count, report + assert report["coverage_complete"], report + assert math.isclose(report["known_cost_subtotal_usd"], subtotal), report + if total is None: + assert report["total_cost_usd"] is None, report + else: + assert math.isclose(report["total_cost_usd"], total), report + ends = {e["uuid"]: e for e in events if e.get("kind") == "scope" and e.get("scope_category") == "end"} + for decision in report["decisions"]: + event = ends[decision["uuid"]] + assert event["name"] == "libsy.decision_call" + assert event["category"] == "llm" + assert decision["context_status"] == "bound" + parent = ends[event["parent_uuid"]] + assert parent["name"] == "openai.chat_completions" and parent["parent_uuid"] == workflow + if headers: + assert event["metadata"]["task_id"] == f"task-{label}" + else: + assert "task_id" not in event["metadata"] + assert event["data"]["model"] == f"test/decision-{model}" + dropped = [e for e in events if e["uuid"] != decision["uuid"]] + assert summarize(dropped, workflow)["total_cost_usd"] is None + assert summarize(events + events, workflow) == report + (output / f"decision-{label}.json").write_text(json.dumps(report, indent=2)) + print(f"decision-{label}: subtotal={report['known_cost_subtotal_usd']}, total={report['total_cost_usd']}") + + +async def check_cancelled_decision(events, output, forbidden): + Provider.scenario = "decision_cancel" + Provider.cancel_started.clear() + with relay.use_scope_stack(relay.create_scope_stack()): + with relay.scope.scope("decision-cancelled", relay.ScopeType.Agent): + await subscribers.flush_async() + workflow = next(e["uuid"] for e in events if e.get("name") == "decision-cancelled") + async def invoke(): + await relay.llm.execute("openai.chat_completions", relay.LLMRequest({}, { + "model": "switchyard/decision_paid", "messages": [{"role": "user", "content": "cancelled request"}], + }), forbidden, codec=codecs.OpenAIChatCodec(), response_codec=codecs.OpenAIChatCodec()) + + task = asyncio.create_task(invoke()) + try: + assert await asyncio.to_thread(Provider.cancel_started.wait, 1.0), "decision did not start" + finally: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + for _ in range(100): + await subscribers.flush_async() + if any(e.get("name") == "libsy.decision_call" and e.get("scope_category") == "end" and e.get("metadata", {}).get("outcome") == "cancelled" for e in events): + break + await asyncio.sleep(0.01) + await subscribers.flush_async() + report = summarize(events, workflow) + assert len(report["decisions"]) == 1, report + decision = report["decisions"][0] + assert decision["outcome"] == "cancelled" and decision["usage_status"] == "missing", report + assert decision["context_status"] == "bound", report + scopes = {e["uuid"]: e for e in events if e.get("kind") == "scope"} + parent = scopes[scopes[decision["uuid"]]["parent_uuid"]] + assert parent["name"] == "openai.chat_completions" and parent["parent_uuid"] == workflow + assert report["total_cost_usd"] is None + (output / "decision-cancelled.json").write_text(json.dumps(report, indent=2)) + print("decision-cancelled: usage=missing, total=None, native workflow preserved") def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--output", type=Path) + parser.add_argument("--include-decisions", action="store_true") args = parser.parse_args() if args.output: args.output.mkdir(parents=True, exist_ok=True) - asyncio.run(run(args.manifest.resolve(), args.output.resolve())) + asyncio.run(run(args.manifest.resolve(), args.output.resolve(), args.include_decisions)) else: with tempfile.TemporaryDirectory() as directory: - asyncio.run(run(args.manifest.resolve(), Path(directory))) + asyncio.run(run(args.manifest.resolve(), Path(directory), args.include_decisions)) if __name__ == "__main__": diff --git a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py index 2abccd73c0..38622da724 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py @@ -11,7 +11,7 @@ from attempt_costs import summarize # noqa: E402 -def report(*costs): +def event_fixture(*costs): events = [ {"kind": "scope", "uuid": "workflow", "scope_category": "end"}, { @@ -37,7 +37,11 @@ def report(*costs): "usage": {"input_tokens": 1, "output_tokens": 1, "cost": cost}, }, }) - return summarize(events, "workflow") + return events + + +def report(*costs): + return summarize(event_fixture(*costs), "workflow") class CurrencyTests(unittest.TestCase): @@ -69,5 +73,55 @@ def test_mixed_currencies_keep_only_the_usd_subtotal(self): self.assertEqual(result["cost_status"], "incomplete") +class DecisionCoverageTests(unittest.TestCase): + def events(self): + events = event_fixture({"total": 0.03, "currency": "USD"}) + events.extend([ + {"kind": "scope", "uuid": "decision", "parent_uuid": "workflow", "scope_category": "end", + "metadata": {"switchyard.accounting.unit": "decision_call", "switchyard.accounting.usage_status": "complete"}, + "data": {"type": "message", "role": "assistant", "model": "decision/model", "content": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "cost": {"total": 0.01, "currency": "USD"}}}}, + {"kind": "mark", "uuid": "decision-mark", "parent_uuid": "workflow", "name": "switchyard.routing.decision_call"}, + ]) + return events + + def test_completion_and_decision_costs_are_counted_once(self): + events = self.events() + result = summarize(events, "workflow") + self.assertEqual(result["total_cost_usd"], 0.04) + self.assertEqual(len(result["attempts"]), 1) + self.assertEqual(len(result["decisions"]), 1) + self.assertEqual(summarize(events + events, "workflow"), result) + + def test_missing_decision_or_coverage_mark_keeps_total_incomplete(self): + for dropped in ["decision", "decision-mark"]: + with self.subTest(dropped=dropped): + events = [e for e in self.events() if e["uuid"] != dropped] + result = summarize(events, "workflow") + self.assertFalse(result["coverage_complete"]) + self.assertIsNone(result["total_cost_usd"]) + + def test_wrong_parent_cannot_be_replaced_by_matching_session_metadata(self): + events = self.events() + decision = next(e for e in events if e["uuid"] == "decision") + decision["parent_uuid"] = "different-workflow" + decision["metadata"]["session_id"] = "workflow" + result = summarize(events, "workflow") + self.assertEqual(result["known_cost_subtotal_usd"], 0.03) + self.assertEqual(result["decisions"], []) + self.assertIsNone(result["total_cost_usd"]) + + def test_unattributed_decision_is_not_assigned_to_the_workflow(self): + events = self.events() + decision = next(e for e in events if e["uuid"] == "decision") + decision["parent_uuid"] = "unknown-root" + events = [e for e in events if e["uuid"] != "decision-mark"] + result = summarize(events, "workflow") + self.assertEqual(result["known_cost_subtotal_usd"], 0.03) + self.assertEqual(result["decisions"], []) + self.assertEqual(result["unattributed_decisions"], [{"uuid": "decision", "context_status": "unattributed"}]) + self.assertIsNone(result["total_cost_usd"]) + + if __name__ == "__main__": unittest.main() diff --git a/crates/switchyard-nemo-relay-plugin/src/decision_tests.rs b/crates/switchyard-nemo-relay-plugin/src/decision_tests.rs new file mode 100644 index 0000000000..e74fa2e559 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/decision_tests.rs @@ -0,0 +1,218 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::{config::SwitchyardConfig, runtime::SwitchyardRuntime}; +use std::sync::{Once, mpsc}; +use switchyard_protocol::{Metadata as RequestMetadata, Request, WireFormat, text_request}; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, +}; + +fn runtime(server: &MockServer, format: &str) -> SwitchyardRuntime { + static KEY: Once = Once::new(); + KEY.call_once(|| unsafe { + std::env::set_var("SWITCHYARD_REQ4_TEST_KEY", "synthetic-decision-key") + }); + let config = serde_json::from_value::(serde_json::json!({ + "switchyard_config": { + "schema_version": 1, + "llm_clients": {"local": {"format": "openai_chat", "base_url": format!("{}/v1",server.uri())}}, + "targets": {"answer": {"id": "answer/model", "llm_client": "local"}, "other": {"id":"other/model","llm_client":"local"}}, + "decision_clients": {"judge": {"format":format, "endpoint":format!("{}/decisions",server.uri()), "api_key_env":"SWITCHYARD_REQ4_TEST_KEY", "timeout_ms":1000}}, + "decision_targets": {"judge": {"id":"decision/model", "decision_client":"judge"}}, + "routes": {"default": { + "id": "switchyard/default", "type": "llm_classifier", "classifier_target":"judge", + "strong_target":"answer", "weak_target":"other", "fail_open":false, + "decision": {"cutoff":0.5, "candidates":{"capable_first":"answer","efficient_first":"other"}, "criteria":{"capable_first":"Capable","efficient_first":"Efficient"}, "instructions":{"task":"Choose"}, "evidence":{"description":"Synthetic context"}} + }} + } + })).unwrap(); + SwitchyardRuntime::new(config).unwrap() +} + +fn decision_response() -> Json { + serde_json::json!({"model":"decision/model", "answers":[{"type":"choice","name":"route","choice":"capable_first","confidence":1.0,"probabilities":[{"value":"capable_first","probability":1.0},{"value":"efficient_first","probability":0.0}]}], + "usage":{"input_tokens":42,"output_tokens":3,"total_tokens":45,"input_tokens_details":{"cached_tokens":10,"cache_write_tokens":2},"output_tokens_details":{"reasoning_tokens":1}}}) +} + +#[tokio::test] +async fn decision_usage_is_exported_once_without_payload_capture() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/decisions")) + .respond_with(ResponseTemplate::new(200).set_body_json(decision_response())) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")).and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"model":"answer/model","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}))) + .expect(1).mount(&server).await; + let runtime = runtime(&server, "openai_decisions"); + let (tx, rx) = mpsc::channel(); + let capture = SpanCapture::new(None, move |span| tx.send(span).unwrap()); + let result = capture + .capture(runtime.execute_buffered( + WireFormat::OpenAiChat, + Request { + llm_request: text_request(Some("switchyard/default".into()), "private context"), + metadata: Some(RequestMetadata { + session_id: Some("session-1".into()), + task_id: Some("task-1".into()), + turn_id: Some("turn-1".into()), + correlation_id: Some("request-1".into()), + ..Default::default() + }), + ..Default::default() + }, + )) + .await; + assert!(result.result.is_ok(), "{:?}", result.result); + let spans: Vec<_> = rx.try_iter().collect(); + let decisions: Vec<_> = spans + .iter() + .filter(|s| s.name == "libsy.decision_client_call") + .collect(); + assert_eq!(decisions.len(), 1, "{spans:?}"); + let (kind, metadata, input, output) = decisions[0].projection(); + assert!(matches!(kind, ScopeType::Llm)); + assert_eq!(metadata["switchyard.accounting.unit"], "decision_call"); + assert_eq!(metadata["switchyard.accounting.usage_status"], "complete"); + assert_eq!(metadata["session_id"], "session-1"); + assert_eq!(metadata["task_id"], "task-1"); + assert_eq!(metadata["turn_id"], "turn-1"); + assert_eq!(metadata["correlation_id"], "request-1"); + assert_eq!(input.unwrap()["model"], "decision/model"); + let output = output.unwrap(); + assert_eq!(output["usage"]["input_tokens"], 30); + assert_eq!(output["usage"]["cache_read_input_tokens"], 10); + assert_eq!(output["usage"]["cache_creation_input_tokens"], 2); + assert_eq!(output["usage"]["output_tokens"], 3); + assert_eq!(output["usage"]["reasoning_tokens"], 1); + assert!(!format!("{decisions:?}").contains("private context")); +} + +fn request() -> Request { + Request { + llm_request: text_request(Some("switchyard/default".into()), "private context"), + ..Default::default() + } +} + +async fn mount_answer(server: &MockServer) { + Mock::given(method("POST")).and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"model":"answer/model","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}))) + .mount(server).await; +} + +#[tokio::test] +async fn system_one_missing_partial_and_zero_usage_preserve_model_fallback() { + for (usage, status) in [ + (serde_json::json!({"provider_cost":0.0}), "missing"), + (serde_json::json!({"input_tokens":6}), "partial"), + ( + serde_json::json!({"input_tokens":0,"output_tokens":0}), + "complete", + ), + ( + serde_json::json!({"input_tokens":6,"output_tokens":2,"cached_input_tokens":3,"cache_creation_input_tokens":2,"cache_creation_1h_input_tokens":1,"reasoning_tokens":1}), + "complete", + ), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")).and(path("/decisions")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "id":"decision-id", "answers":{"route":{"type":"choice","choice":"capable_first","confidence":1.0,"probabilities":{"capable_first":1.0,"efficient_first":0.0}}},"usage":usage + }))).expect(1).mount(&server).await; + mount_answer(&server).await; + let runtime = runtime(&server, "system_one"); + let (tx, rx) = mpsc::channel(); + let capture = SpanCapture::new(None, move |span| tx.send(span).unwrap()); + assert!( + capture + .capture(runtime.execute_buffered(WireFormat::OpenAiChat, request())) + .await + .result + .is_ok() + ); + let spans: Vec<_> = rx.try_iter().filter(|s| s.name == DECISION_CALL).collect(); + assert_eq!(spans.len(), 1); + let (_, meta, _, output) = spans[0].projection(); + assert_eq!(meta["switchyard.accounting.usage_status"], status); + let output = output.unwrap(); + assert_eq!(output["model"], "decision/model"); + assert_eq!(output["id"], "decision-id"); + if status == "missing" { + assert_eq!(output["usage"]["cost"]["total"], 0.0); + assert!(output["usage"].get("input_tokens").is_none()); + } + if usage.get("cached_input_tokens").is_some() { + assert_eq!(output["usage"]["input_tokens"], 6); + assert_eq!( + output["usage"]["cache_creation"]["ephemeral_1h_input_tokens"], + 1 + ); + } + } +} + +#[tokio::test] +async fn decision_failures_keep_missing_usage_without_error_bodies() { + for (reply, error_type) in [ + (ResponseTemplate::new(503).set_body_string("private provider body"), "503"), + (ResponseTemplate::new(200).set_body_json(serde_json::json!({"model":"decision/model","answers":[],"usage":{"input_tokens":42}})), "invalid_response"), + ] { + let server=MockServer::start().await; + Mock::given(method("POST")).and(path("/decisions")).respond_with(reply).expect(1).mount(&server).await; + let runtime=runtime(&server,"openai_decisions"); + let (tx,rx)=mpsc::channel(); + let capture=SpanCapture::new(None,move |span|tx.send(span).unwrap()); + assert!(capture.capture(runtime.execute_buffered(WireFormat::OpenAiChat,request())).await.result.is_err()); + let spans:Vec<_>=rx.try_iter().collect(); + let [decision]=spans.as_slice() else {panic!("one decision span expected: {spans:?}")}; + let (_,meta,_,output)=decision.projection(); + assert_eq!(meta["outcome"],"error"); + assert_eq!(meta["error.type"],error_type); + assert_eq!(meta["switchyard.accounting.usage_status"],"missing"); + assert!(output.unwrap().get("usage").is_none()); + assert!(!format!("{spans:?}").contains("private provider body")); + } +} + +#[tokio::test] +async fn dropping_in_flight_host_work_exports_one_cancelled_decision() { + let server = MockServer::start().await; + let started = std::sync::Arc::new(tokio::sync::Notify::new()); + let seen = started.clone(); + Mock::given(method("POST")) + .and(path("/decisions")) + .respond_with(move |_: &wiremock::Request| { + seen.notify_one(); + ResponseTemplate::new(200) + .set_body_json(decision_response()) + .set_delay(std::time::Duration::from_secs(5)) + }) + .mount(&server) + .await; + let runtime = runtime(&server, "openai_decisions"); + let (tx, rx) = mpsc::channel(); + let capture = SpanCapture::new(None, move |span| tx.send(span).unwrap()); + let mut execution = + Box::pin(capture.capture(runtime.execute_buffered(WireFormat::OpenAiChat, request()))); + tokio::select! { + _=started.notified()=>{}, + _=&mut execution=>panic!("decision should remain in flight"), + _=tokio::time::sleep(std::time::Duration::from_secs(2))=>panic!("decision did not start"), + } + drop(execution); + let spans: Vec<_> = rx.try_iter().collect(); + let [decision] = spans.as_slice() else { + panic!("one cancelled span expected: {spans:?}") + }; + assert_eq!(decision.name, DECISION_CALL); + let (_, meta, _, _) = decision.projection(); + assert_eq!(meta["outcome"], "cancelled"); + assert_eq!(meta["switchyard.accounting.usage_status"], "missing"); + assert_eq!(meta["otel.status_code"], "ERROR"); +} diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs index 9b38c3634e..f189c77b0d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs +++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs @@ -22,6 +22,7 @@ use tracing_subscriber::registry::LookupSpan; const CLIENT_CALL: &str = "libsy.client_call"; const UPSTREAM_ATTEMPT: &str = "libsy.upstream_attempt"; +const DECISION_CALL: &str = "libsy.decision_client_call"; const PAYLOAD_TARGET: &str = "switchyard.payload"; const MAX_PAYLOAD_REDACTION_DEPTH: usize = 64; @@ -201,8 +202,13 @@ impl CapturedSpan { pub(crate) fn emit(self, runtime: &PluginRuntime) -> Result<(), String> { let (scope_type, metadata, input, output) = self.projection(); + let decision = self.name == DECISION_CALL; let mut scope = runtime.scope_at( - self.name, + if decision { + "libsy.decision_call" + } else { + self.name + }, scope_type, None, Some(&metadata), @@ -230,9 +236,10 @@ impl CapturedSpan { .and_then(Json::as_str); let summary = unit == Some("summary"); let attempt = unit == Some("completion_attempt"); + let decision = unit == Some("decision_call"); let scope_type = if summary { ScopeType::Custom - } else if attempt { + } else if attempt || decision { ScopeType::Llm } else { relay_scope_type(self.name) @@ -252,7 +259,7 @@ impl CapturedSpan { .count(), ); None - } else if attempt || self.name == CLIENT_CALL { + } else if attempt || decision || self.name == CLIENT_CALL { // Debug payloads supplement accounting, rather than replacing its usage. let mut output = response_data(&metadata); if let Some(payload) = &self.payloads.response { @@ -565,7 +572,11 @@ impl Visit for FieldVisitor<'_> { } fn captures(metadata: &Metadata<'_>) -> bool { - metadata.target() == "libsy" && matches!(metadata.name(), CLIENT_CALL | UPSTREAM_ATTEMPT) + metadata.target() == "libsy" + && matches!( + metadata.name(), + CLIENT_CALL | UPSTREAM_ATTEMPT | DECISION_CALL + ) } fn relay_scope_type(name: &str) -> ScopeType { @@ -591,6 +602,9 @@ fn is_safe_field(name: &str) -> bool { | "switchyard.candidate_count" | "switchyard.call_phase" | "selected_model" + | "task_id" + | "turn_id" + | "correlation_id" | "gen_ai.request.model" | "gen_ai.request.stream" | "gen_ai.request.temperature" @@ -1071,3 +1085,7 @@ mod tests { #[cfg(test)] #[path = "attempt_tests.rs"] mod attempt_tests; + +#[cfg(test)] +#[path = "decision_tests.rs"] +mod decision_tests; diff --git a/docs/integrations/nemo_relay.md b/docs/integrations/nemo_relay.md index 218a0c3ef1..631c970203 100644 --- a/docs/integrations/nemo_relay.md +++ b/docs/integrations/nemo_relay.md @@ -64,10 +64,15 @@ Each scope retains available usage and supported provider charges when the attempt fails or its stream is dropped. Local validation failures and deadlines that expire before dispatch create no attempt. -Use these scopes as the accounting source for Switchyard completion work: +Typed decision-model invocations also emit one `libsy.decision_call` scope, +with available normalized usage and outcome. This covers decision transports +such as OpenAI Decisions and SystemOne. LLM classifier calls that use the normal +completion client remain completion attempts. -1. Select end events with `switchyard.accounting.unit = "completion_attempt"` - in the workflow's scope tree. Count each event UUID once. +Use these scopes as the accounting source for Switchyard model work: + +1. Select end events with accounting unit `completion_attempt` or + `decision_call` in the workflow's native scope tree. Count each UUID once. 2. Decode each scope's content-free Messages-shaped output with Relay's `AnthropicMessagesCodec`. Relay uses reported charges first and configured model prices otherwise. Price each attempt with its own model. @@ -75,7 +80,9 @@ Use these scopes as the accounting source for Switchyard completion work: from this total. They describe work already represented by the attempts. 4. Check each summary's expected `switchyard.accounting.attempts` against `switchyard.accounting.attempt_records` and the received attempt scopes. - Missing records leave coverage incomplete. + Missing records leave coverage incomplete. Compare decision records with the + unique `switchyard.routing.decision_call` marks in the same workflow. These + marks check coverage and do not contribute another charge. The [ATOF cost-summary example](../../crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py) implements this selection and uses Relay's existing codec and pricing resolver. @@ -98,14 +105,24 @@ valid even when token usage is missing. Cumulative usage snapshots replace known fields; repeated snapshots are not added together. Cache and reasoning fields can overlap other token counts. Token coverage and charge coverage are separate: a known provider charge can -cover an attempt whose token usage is incomplete. +cover an attempt or decision whose token usage is incomplete. + +Configure known free decision models with explicit zero rates in Relay. A missing +price remains unknown. For example, a `$0.03` answer plus a zero-priced decision +has a `$0.03` total; with a decision priced at `$0.01`, the total is `$0.04`. +A decision with unknown usage or price leaves only the known subtotal. + +The consumer uses native scope parentage for workflow association. Session, +task, turn, and correlation IDs supplement it; they do not assign a record to a +workflow. It reports decisions without a known workflow ancestor in +`unattributed_decisions` and leaves the complete total absent. Supply the scope +tree and decision marks needed to verify coverage. Relay's [model-pricing guide](https://docs.nvidia.com/nemo/relay/configure-plugins/model-pricing) -explains how to supply model rates. The cost-summary example covers instrumented -completion attempts. Separate decision-model calls are not yet exported with -per-call usage. ATIF `final_metrics` still describes Relay's outer interaction; -it does not automatically include this retry total. Provider work after a -disconnect and missing telemetry can leave costs unknown. +explains how to supply model rates. ATIF `final_metrics` still describes Relay's +outer interaction; it does not automatically include the ATOF workflow total. +Failed decision bodies are not reparsed for usage. Invalid responses, provider +work after a disconnect, and missing telemetry can leave costs unknown. ## How Requests Flow @@ -325,11 +342,11 @@ can receive these records, but each output format presents them differently. | Output | What the current integration shows | | --- | --- | | ATIF | The request and response seen by the agent. Internal routing is not added as separate steps or included in `final_metrics`. Relay-managed ATIF files may retain the raw records under `extra.observed_events`. | -| OpenTelemetry traces, including OpenInference | Relay's outer LLM span, nested `libsy.client_call` summaries and `libsy.upstream_attempt` accounting scopes, plus eligible routing marks. | +| OpenTelemetry traces, including OpenInference | Relay's outer LLM span, nested `libsy.client_call` summaries, `libsy.upstream_attempt` accounting scopes, and `libsy.decision_call` scopes, plus eligible routing marks. | | OpenTelemetry metrics | Switchyard request, routing-call, latency, token, and failure measurements. Aggregate model-call measurements; use attempt scopes for retry cost accounting. | | OTLP logs | Non-metric Switchyard marks that meet the configured severity threshold. | -The plugin captures two existing Switchyard spans. Each routing or answer LLM +The plugin captures existing Switchyard client, attempt, and host decision spans. Each routing or answer LLM call becomes a `libsy.client_call` scope. Physical HTTP attempts and retries become `libsy.upstream_attempt` child scopes. Buffered calls close with the response; streaming calls close when the stream finishes, fails, or is dropped. The @@ -337,6 +354,13 @@ projection contains allowlisted routing metadata by default. Optional [routing payload capture](#inspect-classifier-and-judge-payloads) adds the final classifier/judge request and successful buffered response. +The existing host `libsy.decision_client_call` span is projected as +`libsy.decision_call` with accounting unit `decision_call`. Each invocation is +one record, even when it asks several questions. The core decision span, marks, +and token metrics retain their diagnostic roles. Decision accounting works with +payload capture disabled and excludes questions, context, answers, and raw +provider error bodies. + `switchyard.call_phase` records when Switchyard issued a client call: `routing` while the algorithm was still deciding, or `completion` after a target was selected. A routing-phase response can later become the final @@ -348,6 +372,7 @@ classification as `routing` or `answer`. | --- | --- | --- | --- | | Routing or answer model call | `libsy.client_call` scope | Relay's managed request scope | Starts with the Switchyard call and closes with the buffered response or terminal stream event. | | Physical provider request or retry | `libsy.upstream_attempt` scope | Its `libsy.client_call` scope | Starts before the HTTP attempt and closes on response, error, or cancellation. | +| Typed decision-model invocation | `libsy.decision_call` scope | Relay's managed request scope | Closes on response, error, or cancellation. | | Routing request, decision, overhead, or failure | Existing `switchyard.routing.*` mark | Active Relay scope at emission | Point-in-time mark using the existing outcome fields. | ### Inspect Classifier and Judge Payloads diff --git a/docs/internal/req4_decision_accounting_implementation.md b/docs/internal/req4_decision_accounting_implementation.md new file mode 100644 index 0000000000..d17ad08e6d --- /dev/null +++ b/docs/internal/req4_decision_accounting_implementation.md @@ -0,0 +1,122 @@ +# Decision usage and workflow attribution implementation + +Req 4 is implemented on `bbednarski/req4-decision-accounting`, based on Req 3 +commit `8e00fd053`. Its ordinary upstream PR targets `main` and depends on #1221. +After #1221 merges, manually rebase or cherry-pick only the Req 4 commits onto +updated `main` so the PR comparison contains only Req 4. GitHub's managed stack +feature is not used. + +The existing host `libsy.decision_client_call` span is the accounting source. +It records available `DecisionResponse.usage`, selected/response model, response +ID, safe outcome/error category, and selected request identity fields. The +native plugin projects it as one `libsy.decision_call` LLM scope with accounting +unit `decision_call`. Multiple questions in one invocation remain one record. + +The core decision span and existing marks/metrics retain their diagnostic roles. +They are not additional charges. Decision transports, protocol types, completion +retry policy, and translation codecs have no Req 4 changes. + +## Components + +Green components are reused. Yellow components are augmented. Red components +are new code or test cases within existing modules. + +```mermaid +flowchart TB + Core["Existing libsy decision driver"] --> Host + Providers["Existing OpenAI Decisions and SystemOne clients"] <--> Host + Types["Existing DecisionResponse / Usage"] -.-> Host + Context["Existing Relay request and workflow scopes"] -.-> Capture + subgraph Client["switchyard-llm-client"] + Host["Augment run.rs host decision span
Usage, status, request identity"] + Shared["Shared accounting recorder in observability.rs
Tokens, charge, usage status"] + Attempts["Existing attempt guard
Reuse shared recorder"] + Host --> Shared + Attempts --> Shared + end + Capture["Augment plugin span_capture.rs
Capture host span, project one decision scope"] + Relay["Existing Relay SDK, native parentage, and pricing"] + Consumer["Augment ATOF consumer
Include decisions, check coverage and attribution"] + Tests["NEW decision cases in shared fixture
Zero/paid rates, missing data, concurrency, cancellation"] + Shared --> Capture --> Relay --> Consumer + Consumer -.-> Tests + classDef reused fill:#dcfce7,stroke:#15803d,color:#14532d; + classDef augmented fill:#fef3c7,stroke:#b45309,color:#78350f; + classDef added fill:#fee2e2,stroke:#b91c1c,color:#7f1d1d; + class Core,Providers,Types,Context,Relay reused; + class Host,Shared,Attempts,Capture,Consumer augmented; + class Tests added; +``` + +Only two production crates change. `attempt.rs` delegates its existing usage +recording to the shared crate-private helper; its lifecycle and retry behavior +are unchanged. A small request-local context copies only session, task, turn, +and correlation IDs to the host decision span, excluding HTTP credentials and +arbitrary request metadata. + +## Accounting and context boundary + +The ATOF consumer prices unique completion-attempt and decision-call end events +with Relay's existing codec. It retains separate `attempts` and `decisions` +records, a known USD subtotal, and a nullable complete USD total. Missing usage, +missing price, explicit zero, and non-USD charges preserve their distinctions. + +Completion coverage uses the Req 3 parent summaries. Decision coverage compares +accounting records with the existing unique `switchyard.routing.decision_call` +marks in the workflow. Marks establish expected record coverage; they are never +priced. Missing either a decision record or its coverage mark leaves the total +incomplete. No second decision export path is built from observations. + +Native scope ancestry establishes workflow association. Header IDs supplement +that relationship and cannot substitute for it. Decisions without a known +workflow ancestor are reported as `unattributed_decisions`, are not assigned to +a guessed workflow, and prevent a complete total for the supplied event set. +Wholly missing telemetry cannot be detected or reconciled. + +Relay's `scope_stack_active()` reports explicit stack ownership, not whether a +Python-synchronized native parent exists. The implementation does not use that +flag as evidence of attribution. Native acceptance asserts actual request and +workflow parent UUIDs. Concurrent logical workflows use separate Relay scope +stacks, including one workflow with no identity headers. + +## Validation + +A baseline test reproduced the gap: successful decision execution emitted a +completion span but no decision accounting span. It now passes with cache, +reasoning, total-token, model, and request-identity assertions. + +Rust regressions cover both decision provider formats; missing/partial/zero +usage; reported zero charges; response-model fallback; HTTP/invalid-response +errors; host cancellation; multiple invocations and questions; and unchanged +observations and metric counts. Existing completion regressions remain in the +full workspace run. + +The packaged native bundle was tested with Relay/SDK 0.10.0, a loopback provider, +and synthetic prices. The shared fixture retains all five Req 3 scenarios and +adds these decision checks: + +| Scenario | Known subtotal | Complete total | +| --- | --- | --- | +| Answer $0.03 plus explicitly zero-priced decision | $0.03 | $0.03 | +| Answer $0.03 plus decision priced at $0.01 | $0.04 | $0.04 | +| Two paid decision/answer invocations in one workflow | $0.08 | $0.08 | +| Answer known, decision price or usage missing | $0.03 | Absent | +| Decision reports $0, tokens missing; answer $0.03 | $0.03 | $0.03 | +| Decision provider error | $0 | Absent | +| Two concurrent paid workflows | $0.04 each | $0.04 each | +| Cancellation during a native decision call | $0 | Absent | + +The cancellation record retains the native request/workflow parent and reports +missing usage. Duplicate event delivery and missing decision records are also +checked. Native acceptance writes raw events, scenario reports, and environment +and bundle-digest metadata. Run it with `--include-decisions`; see the plugin +README for the command. + +## Limits + +ATIF `final_metrics` is unchanged. The verified combined workflow total is the +opt-in ATOF consumer output. Pricing remains Relay-owned and free decision models +require explicit zero rates. Errors without a usable `DecisionResponse` retain +missing usage; this change does not extract hidden charges from failed or invalid +decision provider bodies. Process termination and provider work after disconnect +can leave consumption unobserved. From a2c7c71e9a068ef6d1bb5ae96fa201aee078c723 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 5 Nov 2026 14:57:47 -0800 Subject: [PATCH 4/4] fix(observability): preserve accounting projection compatibility Signed-off-by: Bryan Bednarski --- crates/libsy-llm-client/src/attempt.rs | 10 +++ crates/switchyard-nemo-relay-plugin/README.md | 3 + .../examples/attempt_costs.py | 4 +- .../scripts/test_attempt_accounting.py | 9 +++ .../scripts/test_attempt_costs.py | 49 ++++++++++++++- .../src/attempt_tests.rs | 62 ++++++++++++++++++- .../src/span_capture.rs | 42 ++++++++----- docs/integrations/nemo_relay.md | 34 ++++++++++ 8 files changed, 195 insertions(+), 18 deletions(-) diff --git a/crates/libsy-llm-client/src/attempt.rs b/crates/libsy-llm-client/src/attempt.rs index ea3fa78dc2..55fde1fd3b 100644 --- a/crates/libsy-llm-client/src/attempt.rs +++ b/crates/libsy-llm-client/src/attempt.rs @@ -51,6 +51,8 @@ impl AttemptGuard { gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty, gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty, gen_ai.usage.reasoning.output_tokens = tracing::field::Empty, + switchyard.usage.input_tokens = tracing::field::Empty, + switchyard.usage.input_tokens_reported = tracing::field::Empty, switchyard.usage.total_tokens = tracing::field::Empty, switchyard.usage.provider_cost_usd = tracing::field::Empty, switchyard.accounting.charge_source = tracing::field::Empty, @@ -199,6 +201,14 @@ impl AttemptGuard { return; } record_gen_ai_usage(&self.span, &self.usage); + // Aggregate semantic counts cannot distinguish missing uncached input from zero. + self.span.record( + "switchyard.usage.input_tokens_reported", + self.usage.input_tokens.is_some(), + ); + if let Some(input) = self.usage.input_tokens { + self.span.record("switchyard.usage.input_tokens", input); + } if let Some(total) = self.usage.total_tokens { self.span.record("switchyard.usage.total_tokens", total); } diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 13fa0298de..fde3866ed2 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -340,3 +340,6 @@ The [cost-summary example](examples/attempt_costs.py) selects unique attempt records and uses Relay's codec to obtain costs. These summaries cover Switchyard completion work. They do not change ATIF `final_metrics` or include separate decision-model usage. + +See the [versioned accounting scope profile](../../docs/integrations/nemo_relay.md#accounting-scope-profile-version-1) +for stable fields and the limits of `coverage_complete` and `total_cost_usd`. diff --git a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py index 1f4167208b..408a04937c 100644 --- a/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py +++ b/crates/switchyard-nemo-relay-plugin/examples/attempt_costs.py @@ -16,7 +16,9 @@ def summarize(events, workflow_uuid): Configure Relay's pricing component before calling this function to enable estimates. Provider-reported charges work without a pricing catalog. - This covers instrumented completion attempts, including routing calls. + Coverage applies only to observed instrumented completion calls, including + routing calls. It excludes separate decisions and other workflow work and + cannot detect calls whose entire telemetry is absent. """ scopes = {e["uuid"]: e for e in events if e.get("kind") == "scope"} diff --git a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py index 0c5ae3983f..12dc6e8e37 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_accounting.py @@ -136,6 +136,15 @@ async def forbidden(_request): assert Provider.calls == 2 assert len(report["attempts"]) == 2 assert report["attempts"][0]["outcome"] == "error" + ends = {e["uuid"]: e for e in events if e.get("scope_category") == "end"} + for record in report["attempts"]: + attempt = ends[record["uuid"]] + assert attempt["name"] == "libsy.upstream_attempt" + assert attempt["metadata"]["switchyard.accounting.schema_version"] == 1 + parent = ends[attempt["parent_uuid"]] + assert parent["name"] == "libsy.client_call" + assert parent["metadata"]["switchyard.accounting.unit"] == "summary" + assert parent["metadata"]["switchyard.accounting.schema_version"] == 1 assert report["coverage_complete"] assert math.isclose(report["known_cost_subtotal_usd"], expected_subtotal) if expected_total is None: diff --git a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py index 2abccd73c0..bea86810ff 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/test_attempt_costs.py @@ -11,7 +11,7 @@ from attempt_costs import summarize # noqa: E402 -def report(*costs): +def cost_events(*costs): events = [ {"kind": "scope", "uuid": "workflow", "scope_category": "end"}, { @@ -37,7 +37,11 @@ def report(*costs): "usage": {"input_tokens": 1, "output_tokens": 1, "cost": cost}, }, }) - return summarize(events, "workflow") + return events + + +def report(*costs): + return summarize(cost_events(*costs), "workflow") class CurrencyTests(unittest.TestCase): @@ -69,5 +73,46 @@ def test_mixed_currencies_keep_only_the_usd_subtotal(self): self.assertEqual(result["cost_status"], "incomplete") +class CompatibilityTests(unittest.TestCase): + def test_versioned_and_legacy_attempts_keep_completion_only_results(self): + events = cost_events({"total": 0.02}, {"total": 0.03}) + expected = summarize(events, "workflow") + for event in [events[1], events[3]]: + event["metadata"].update({ + "switchyard.accounting.schema_version": 1, "future_field": "ignored", + }) + events.append(events[3].copy()) + for name, unit in [("libsy.decision_call", "decision_call"), ("outer", "summary")]: + events.append({ + **events[2], "uuid": name, "name": name, "parent_uuid": "workflow", + "metadata": {"switchyard.accounting.unit": unit}, + }) + result = summarize(events, "workflow") + self.assertEqual(result, expected) + self.assertEqual(set(result), { + "workflow_uuid", "attempts", "coverage_complete", "cost_status", + "known_cost_subtotal_usd", "total_cost_usd", + }) + self.assertEqual(result["total_cost_usd"], 0.05) + self.assertEqual([r["uuid"] for r in result["attempts"]], ["attempt-1", "attempt-2"]) + + def test_missing_consumption_and_reported_zero_remain_distinct(self): + events = cost_events({}) + attempt = events[-1] + attempt["metadata"]["switchyard.accounting.usage_status"] = "missing" + attempt["data"].pop("usage") + missing = summarize(events, "workflow") + self.assertTrue(missing["coverage_complete"]) + self.assertEqual(missing["known_cost_subtotal_usd"], 0.0) + self.assertIsNone(missing["total_cost_usd"]) + attempt["data"]["usage"] = {"cost": { + "total": 0.0, "currency": "USD", "source": "provider_reported", + }} + zero = summarize(events, "workflow") + self.assertEqual(zero["total_cost_usd"], 0.0) + self.assertEqual(zero["attempts"][0]["usage_status"], "missing") + self.assertEqual(zero["cost_status"], "complete") + + if __name__ == "__main__": unittest.main() diff --git a/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs b/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs index 5d1d3859ce..b3f6e2be68 100644 --- a/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs +++ b/crates/switchyard-nemo-relay-plugin/src/attempt_tests.rs @@ -70,11 +70,14 @@ async fn retry_usage_and_reported_charges_survive_failure_without_parent_charge( .await; assert!(result.result.is_ok()); let call = rx.try_recv().unwrap(); + assert_eq!(call.name, CLIENT_CALL); assert_eq!(call.children.len(), 2); assert_eq!(call.fields["switchyard.accounting.attempts"], 2); let (kind, metadata, _, output) = call.projection(); assert!(matches!(kind, ScopeType::Custom)); assert!(output.is_none()); + assert_eq!(metadata["switchyard.accounting.schema_version"], 1); + assert_eq!(metadata["switchyard.accounting.unit"], "summary"); assert_eq!(metadata["switchyard.accounting.attempt_records"], 2); for (attempt, outcome, input, cost) in [ (&call.children[0], "error", 11, 0.02), @@ -82,6 +85,9 @@ async fn retry_usage_and_reported_charges_survive_failure_without_parent_charge( ] { let (kind, metadata, input_data, output) = attempt.projection(); assert!(matches!(kind, ScopeType::Llm)); + assert_eq!(attempt.name, UPSTREAM_ATTEMPT); + assert_eq!(metadata["switchyard.accounting.schema_version"], 1); + assert_eq!(metadata["switchyard.accounting.unit"], "completion_attempt"); assert_eq!(metadata["outcome"], outcome); assert_eq!(metadata["switchyard.accounting.usage_status"], "complete"); assert_eq!(input_data.unwrap()["model"], "test/model"); @@ -89,6 +95,8 @@ async fn retry_usage_and_reported_charges_survive_failure_without_parent_charge( assert_eq!(output["model"], "test/model"); assert_eq!(output["usage"]["input_tokens"], input); assert_eq!(output["usage"]["cost"]["total"], cost); + assert_eq!(output["usage"]["cost"]["currency"], "USD"); + assert_eq!(output["usage"]["cost"]["source"], "provider_reported"); } assert_eq!( call.children[1].fields["switchyard.accounting.charge_source"], @@ -97,6 +105,57 @@ async fn retry_usage_and_reported_charges_survive_failure_without_parent_charge( assert_eq!(calls.load(Ordering::SeqCst), 2); } +#[tokio::test] +async fn cache_usage_preserves_missing_and_zero_uncached_input() { + for input in [None, Some(0), Some(7)] { + let server = MockServer::start().await; + let mut usage = serde_json::json!({"cache_read_input_tokens":10,"output_tokens":2}); + if let Some(input) = input { + usage["input_tokens"] = Json::from(input); + } + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "id":"response", "type":"message", "role":"assistant", "model":"test/model", + "content":[{"type":"text","text":"ok"}], "stop_reason":"end_turn", "usage":usage + }))) + .expect(1) + .mount(&server) + .await; + let runtime = runtime(&server, "anthropic_messages", 0); + let (capture, rx) = recorder(false); + assert!( + capture + .capture(runtime.execute_buffered(WireFormat::AnthropicMessages, request(false))) + .await + .result + .is_ok() + ); + let call = rx.try_recv().unwrap(); + let (_, metadata, _, output) = call.children[0].projection(); + assert_eq!( + metadata["switchyard.usage.input_tokens_reported"], + input.is_some() + ); + assert_eq!( + metadata + .get("switchyard.usage.input_tokens") + .and_then(Json::as_u64), + input + ); + let usage = &output.as_ref().unwrap()["usage"]; + assert_eq!(usage.get("input_tokens").and_then(Json::as_u64), input); + assert_eq!(usage["cache_read_input_tokens"], 10); + assert_eq!( + metadata["switchyard.accounting.usage_status"], + if input.is_some() { + "complete" + } else { + "partial" + } + ); + } +} + #[tokio::test] async fn missing_retry_usage_is_not_zero() { let server = MockServer::start().await; @@ -234,7 +293,7 @@ async fn abandoning_stream_preserves_partial_usage() { #[test] fn debug_payloads_cannot_replace_usage_and_summary_is_not_priceable() { - let fields = serde_json::json!({"gen_ai.request.model":"model", "gen_ai.usage.input_tokens":12, "gen_ai.usage.output_tokens":0, "switchyard.accounting.unit":"completion_attempt"}).as_object().unwrap().clone(); + let fields = serde_json::json!({"gen_ai.request.model":"model", "gen_ai.usage.input_tokens":12, "gen_ai.usage.cache_read.input_tokens":5, "gen_ai.usage.output_tokens":0, "switchyard.accounting.unit":"completion_attempt"}).as_object().unwrap().clone(); let mut attempt = CapturedSpan { name: UPSTREAM_ATTEMPT, fields, @@ -249,6 +308,7 @@ fn debug_payloads_cannot_replace_usage_and_summary_is_not_priceable() { let (_, _, input, output) = attempt.projection(); assert_eq!(input.unwrap()["model"], "model"); let output = output.unwrap(); + assert_eq!(output["usage"]["input_tokens"], 7); assert_eq!(output["usage"]["output_tokens"], 0); assert_eq!(output["truncated"], true); attempt.fields.insert( diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs index 9b38c3634e..767c99b0b3 100644 --- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs +++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs @@ -230,6 +230,9 @@ impl CapturedSpan { .and_then(Json::as_str); let summary = unit == Some("summary"); let attempt = unit == Some("completion_attempt"); + if summary || attempt { + metadata["switchyard.accounting.schema_version"] = Json::from(1); + } let scope_type = if summary { ScopeType::Custom } else if attempt { @@ -372,22 +375,31 @@ fn response_data(fields: &Json) -> Option { serde_json::json!({"ephemeral_1h_input_tokens": value}), ); } - if let Some(input) = fields - .get("gen_ai.usage.input_tokens") - .and_then(Json::as_u64) + let input = match fields + .get("switchyard.usage.input_tokens_reported") + .and_then(Json::as_bool) { - let cached = usage - .get("cache_read_input_tokens") - .and_then(Json::as_u64) - .unwrap_or(0); - let created = usage - .get("cache_creation_input_tokens") + Some(true) => fields + .get("switchyard.usage.input_tokens") + .and_then(Json::as_u64), + Some(false) => None, + None => fields + .get("gen_ai.usage.input_tokens") .and_then(Json::as_u64) - .unwrap_or(0); - usage.insert( - "input_tokens".into(), - input.saturating_sub(cached).saturating_sub(created).into(), - ); + .map(|input| { + let cached = usage + .get("cache_read_input_tokens") + .and_then(Json::as_u64) + .unwrap_or(0); + let created = usage + .get("cache_creation_input_tokens") + .and_then(Json::as_u64) + .unwrap_or(0); + input.saturating_sub(cached).saturating_sub(created) + }), + }; + if let Some(input) = input { + usage.insert("input_tokens".into(), input.into()); } if !usage.is_empty() { output.insert("usage".into(), Json::Object(usage)); @@ -584,6 +596,8 @@ fn is_safe_field(name: &str) -> bool { | "switchyard.accounting.attempts" | "switchyard.accounting.usage_status" | "switchyard.accounting.charge_source" + | "switchyard.usage.input_tokens" + | "switchyard.usage.input_tokens_reported" | "switchyard.usage.total_tokens" | "switchyard.usage.provider_cost_usd" | "algorithm" diff --git a/docs/integrations/nemo_relay.md b/docs/integrations/nemo_relay.md index 218a0c3ef1..c0dcda67e4 100644 --- a/docs/integrations/nemo_relay.md +++ b/docs/integrations/nemo_relay.md @@ -100,6 +100,40 @@ added together. Cache and reasoning fields can overlap other token counts. Token coverage and charge coverage are separate: a known provider charge can cover an attempt whose token usage is incomplete. +#### Accounting Scope Profile, Version 1 + +Completion-attempt and summary scopes set the integer metadata field +`switchyard.accounting.schema_version` to `1`. This versions the scope +representation independently of Relay's worker protocol and mark `DataSchema`. +Consumers should tolerate additional fields and unknown values. Removing or +renaming fields, changing their types, or changing their meaning requires a +new profile version. + +| Record | Scope type and accounting unit | Identity and output | +| --- | --- | --- | +| `libsy.client_call` | Custom; `summary` | Parent of its attempts; no priceable response output | +| `libsy.upstream_attempt` | LLM; `completion_attempt` | Its scope UUID identifies one attempt; `parent_uuid` refers to its client call; content-free Messages-shaped output carries known usage and USD charges | + +The summary declares expected attempts in `switchyard.accounting.attempts` +and captured attempts in `switchyard.accounting.attempt_records`. Each child +has a one-based `attempt` ordinal. `outcome` (`ok`, `error`, or `cancelled`) +is separate from `switchyard.accounting.usage_status`. Missing token counts +remain absent; reported zero remains zero. Charges use `usage.cost` with +`currency = "USD"` and `source = "provider_reported"`. +`switchyard.accounting.charge_source` distinguishes body charges (`provider`) +from the supported header fallback (`header`); body charges take precedence. + +`switchyard.usage.input_tokens_reported` preserves whether the original +non-cached input count was known. When true, `switchyard.usage.input_tokens` +contains that count, including zero. When false, the projected input count +stays absent even if cache usage is known. Older captures without this flag +retain the aggregate-minus-cache fallback. + +The example's `coverage_complete` checks the received, instrumented completion +records against their summary counts. It cannot detect a wholly absent call +or certify full workflow coverage. Its `total_cost_usd` has that same scope; +decision calls and other producers require a separate accounting view. + Relay's [model-pricing guide](https://docs.nvidia.com/nemo/relay/configure-plugins/model-pricing) explains how to supply model rates. The cost-summary example covers instrumented completion attempts. Separate decision-model calls are not yet exported with