diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 1c68428645..fd3bbb03e3 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -84,6 +84,14 @@ observer does not buffer headers or alter transport behavior. A receipt confirms only observation of a response head, not completion of its body. Process crashes and requests that remain hung cannot be finalized by the guard. +An upstream HTTP child span measures the actual request and response, while a +response-delivery child records framing, middleware selection and completion. +A failure after final headers produces one additional terminal HTTP Activity +record, since the primary record cannot be revised. Routine successful delivery +does not add another OCSF record. A locally selected error response records its +effective status separately from the upstream status; telemetry never substitutes +an intended status for bytes that were not accepted by the client connection. + When selected, request and response middleware have separate child spans. Routine invocations stay in sampled traces; security findings and notable failures remain in OCSF output, linked by the same request ID. Each phase closes when its work diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index c3efdb327b..f402c04d4a 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -28,6 +28,7 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; pub struct RequestCorrelation { request_id: String, span: tracing::Span, + event_context: HttpEventContext, // Keep the audit payload off the async state machine. state: Box>, } @@ -40,6 +41,9 @@ struct RequestState { outcome: Option<&'static str>, policy_span: Option, middleware: [Option; 2], + response: ResponseDeliveryState, + response_span: Option, + upstream_span: Option, jsonrpc: JsonRpcCorrelationState, mcp_span: Option, resources: ResourceCorrelationState, @@ -47,6 +51,92 @@ struct RequestState { credential_span: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResponseSelectionCause { + UpstreamResponse, + UpstreamIoError, + UpstreamTruncated, + UpstreamIdleTimeout, + UpstreamProtocolError, + MiddlewareDenied, + MiddlewareFailed, + PolicyReloaded, + Cancelled, + PlatformError, + Unknown, +} + +impl ResponseSelectionCause { + fn as_str(self) -> &'static str { + match self { + Self::UpstreamResponse => "upstream_response", + Self::UpstreamIoError => "upstream_io_error", + Self::UpstreamTruncated => "upstream_truncated", + Self::UpstreamIdleTimeout => "upstream_idle_timeout", + Self::UpstreamProtocolError => "upstream_protocol_error", + Self::MiddlewareDenied => "middleware_denied", + Self::MiddlewareFailed => "middleware_failed", + Self::PolicyReloaded => "policy_reloaded", + Self::Cancelled => "cancelled", + Self::PlatformError => "platform_error", + Self::Unknown => "unknown", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResponseDeliveryOutcome { + Complete, + DownstreamIoError, + CancelledOrIncomplete, +} + +impl ResponseDeliveryOutcome { + fn as_str(self) -> &'static str { + match self { + Self::Complete => "complete", + Self::DownstreamIoError => "downstream_io_error", + Self::CancelledOrIncomplete => "cancelled_or_incomplete", + } + } +} + +#[derive(Debug, Default)] +struct ResponseDeliveryState { + lifecycle_started: bool, + upstream_status: Option, + effective_status: Option, + selection_cause: Option, + http_outcome: Option<&'static str>, + error_stage: Option<&'static str>, + pending_delivery: Option, + interim_bytes_forwarded: u64, + delivery_started: bool, + partial_delivery: bool, + terminal: Option, +} + +#[derive(Debug, Clone, Copy)] +struct ResponseDeliverySnapshot { + upstream_status: Option, + effective_status: Option, + selection_cause: ResponseSelectionCause, + interim_bytes_forwarded: u64, + delivery_started: bool, + partial_delivery: bool, + delivery_outcome: ResponseDeliveryOutcome, + http_outcome: &'static str, + error_stage: Option<&'static str>, +} + +#[derive(Debug)] +struct HttpEventContext { + method: &'static str, + host: String, + port: u16, + policy_name: String, +} + #[derive(Debug, Default)] struct JsonRpcCorrelationState { effective: Option, @@ -402,6 +492,12 @@ impl RequestCorrelation { Self { request_id, span, + event_context: HttpEventContext { + method, + host: ctx.host.clone(), + port: ctx.port, + policy_name: ctx.policy_name.clone(), + }, state: Box::new(Mutex::new(RequestState { primary: Some(fallback), status: None, @@ -409,6 +505,9 @@ impl RequestCorrelation { outcome: None, policy_span: None, middleware: [None, None], + response: ResponseDeliveryState::default(), + response_span: None, + upstream_span: None, jsonrpc: JsonRpcCorrelationState::default(), mcp_span: None, resources: ResourceCorrelationState::default(), @@ -424,11 +523,7 @@ impl RequestCorrelation { } pub(crate) fn observe_response<'a, C>(&'a self, client: &'a mut C) -> AuditedClient<'a, C> { - AuditedClient { - inner: client, - correlation: self, - head: ResponseHead::default(), - } + observe_response(client, Some(self)) } pub(crate) async fn within_span(&self, future: F) -> F::Output @@ -456,6 +551,7 @@ impl RequestCorrelation { .take(); if let Some(span) = policy_span { span.record("openshell.policy.decision", decision); + close_span(span); } } @@ -491,6 +587,11 @@ impl RequestCorrelation { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .status = Some(status); + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .response + .effective_status = Some(status); } pub(crate) fn record_upstream_status(&self, status: u16) { @@ -500,6 +601,51 @@ impl RequestCorrelation { .upstream_status = Some(status); } + pub(crate) fn record_upstream_response_status(&self, status: u16) { + self.record_upstream_status(status); + let span = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.response.upstream_status = Some(status); + state.upstream_span.clone() + }; + if let Some(span) = span { + span.record("http.response.status_code", status); + } + } + + fn delivery_span(&self) -> tracing::Span { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state + .response_span + .get_or_insert_with(|| { + self.span + .with_subscriber(|(_, dispatch)| { + tracing::dispatcher::with_default(dispatch, || { + tracing::info_span!(parent: &self.span, "openshell.response.deliver", + otel.status_code = tracing::field::Empty, + "openshell.response.selection_cause" = tracing::field::Empty, + "openshell.response.delivery_outcome" = tracing::field::Empty, + "openshell.response.delivery_started" = false, + "openshell.response.partial_delivery" = false, + "openshell.response.interim_delivery_started" = false, + "openshell.response.interim_bytes_forwarded" = 0_i64, + "openshell.upstream.http.response.status_code" = tracing::field::Empty, + "http.response.status_code" = tracing::field::Empty, + "openshell.http.outcome" = tracing::field::Empty, + "openshell.error.stage" = tracing::field::Empty) + }) + }) + .unwrap_or_else(tracing::Span::none) + }) + .clone() + } + pub(crate) fn record_policy_runtime_generation(&self, generation: u64) { { let mut resources = self @@ -631,20 +777,23 @@ impl RequestCorrelation { fn record_jsonrpc_attributes(&self) { let span = self.take_mcp_span(); - let (summary, selected_revision, tools_call_observed) = { + let summary = { let state = self .state .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let state = &state.jsonrpc; - let Some(summary) = state.effective else { - return; - }; - ( - summary, - state.selected_mcp_revision, - state.mcp_tools_call_observed, - ) + state.effective.map(|summary| { + ( + summary, + state.selected_mcp_revision, + state.mcp_tools_call_observed, + ) + }) + }; + let Some((summary, selected_revision, tools_call_observed)) = summary else { + close_span(span); + return; }; span.record("openshell.l7.protocol", summary.protocol); @@ -691,6 +840,7 @@ impl RequestCorrelation { } span.record("openshell.mcp.tools_call_observed", tools_call_observed); } + close_span(span); } fn new_mcp_span(&self) -> tracing::Span { @@ -746,6 +896,7 @@ impl RequestCorrelation { .mcp_span .take(); } + close_span(span); result } } @@ -757,6 +908,24 @@ impl RequestCorrelation { pub(crate) fn record_request_delivery_started(&self) { self.span.record("openshell.request.delivery_started", true); self.finish_credential_span(); + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.upstream_span.get_or_insert_with(|| { + self.span + .with_subscriber(|(_, dispatch)| { + tracing::dispatcher::with_default(dispatch, || { + tracing::info_span!(parent: &self.span, "openshell.upstream.http", + otel.kind = "client", otel.status_code = tracing::field::Empty, + "http.request.method" = self.event_context.method, + "server.address" = %self.event_context.host, + "server.port" = self.event_context.port, + "http.response.status_code" = tracing::field::Empty) + }) + }) + .unwrap_or_else(tracing::Span::none) + }); } fn credential_span(&self, state: &mut RequestState) { @@ -864,6 +1033,7 @@ impl RequestCorrelation { openshell_otel::mark_error(&span); } } + close_span(span); } pub(crate) fn record_error_stage(&self, _stage: &str) { @@ -1027,25 +1197,210 @@ impl RequestCorrelation { } } - pub(crate) fn record_response_delivery_started(&self) { - self.observe_middleware( - MiddlewarePhase::Response, - MiddlewareObservation { - delivery_started: true, - ..Default::default() - }, + pub(crate) fn begin_response_lifecycle(&self) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .response + .lifecycle_started = true; + } + + pub(crate) fn record_response_selection( + &self, + cause: ResponseSelectionCause, + http_outcome: &'static str, + error_stage: Option<&'static str>, + ) { + let mut response = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let response = &mut response.response; + response.lifecycle_started = true; + response.selection_cause = Some(cause); + response.http_outcome = Some(http_outcome); + response.error_stage = error_stage; + } + + pub(crate) fn record_pending_response_delivery(&self, outcome: ResponseDeliveryOutcome) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .response + .pending_delivery = Some(outcome); + } + + pub(crate) fn record_interim_response_delivery(&self, accepted: usize) { + if accepted == 0 { + return; + } + let bytes = { + let mut response = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let response = &mut response.response; + response.lifecycle_started = true; + response.interim_bytes_forwarded = response + .interim_bytes_forwarded + .saturating_add(u64::try_from(accepted).unwrap_or(u64::MAX)); + response.interim_bytes_forwarded + }; + self.delivery_span() + .record("openshell.response.interim_delivery_started", true); + self.delivery_span().record( + "openshell.response.interim_bytes_forwarded", + i64::try_from(bytes).unwrap_or(i64::MAX), ); } - pub(crate) fn record_response_partial_delivery(&self) { - self.observe_middleware( - MiddlewarePhase::Response, - MiddlewareObservation { - delivery_started: true, - partial_delivery: true, - ..Default::default() - }, + pub(crate) fn record_response_delivery_started(&self) { + { + let mut response = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let response = &mut response.response; + response.lifecycle_started = true; + response.delivery_started = true; + } + self.delivery_span() + .record("openshell.response.delivery_started", true); + } + + pub(crate) fn record_response_partial_delivery(&self) -> bool { + { + let mut response = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let response = &mut response.response; + if !response.delivery_started { + return false; + } + response.partial_delivery = true; + } + self.delivery_span() + .record("openshell.response.partial_delivery", true); + true + } + + pub(crate) fn complete_pending_response_delivery(&self) { + let delivery = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .response + .pending_delivery + .unwrap_or(ResponseDeliveryOutcome::Complete); + self.complete_response_delivery(delivery, None, None, None); + } + + pub(crate) fn complete_response_failure( + &self, + delivery: ResponseDeliveryOutcome, + cause: Option, + http_outcome: &'static str, + error_stage: &'static str, + ) { + self.complete_response_delivery(delivery, cause, Some(http_outcome), Some(error_stage)); + } + + fn complete_response_delivery( + &self, + delivery: ResponseDeliveryOutcome, + cause: Option, + http_outcome: Option<&'static str>, + error_stage: Option<&'static str>, + ) { + let snapshot = { + let mut response = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let response = &mut response.response; + if response.terminal.is_some() { + return; + } + response.lifecycle_started = true; + if let Some(cause) = cause { + response.selection_cause = Some(cause); + } + if let Some(http_outcome) = http_outcome { + response.http_outcome = Some(http_outcome); + } + if let Some(error_stage) = error_stage { + response.error_stage = Some(error_stage); + } + if delivery != ResponseDeliveryOutcome::Complete && response.delivery_started { + response.partial_delivery = true; + } + response.terminal = Some(delivery); + ResponseDeliverySnapshot { + upstream_status: response.upstream_status, + effective_status: response.effective_status, + selection_cause: response + .selection_cause + .unwrap_or(ResponseSelectionCause::Unknown), + interim_bytes_forwarded: response.interim_bytes_forwarded, + delivery_started: response.delivery_started, + partial_delivery: response.partial_delivery, + delivery_outcome: delivery, + http_outcome: response.http_outcome.unwrap_or("incomplete"), + error_stage: response.error_stage, + } + }; + + if let Some(status) = snapshot.upstream_status { + self.delivery_span() + .record("openshell.upstream.http.response.status_code", status); + } + if let Some(status) = snapshot.effective_status { + self.delivery_span() + .record("http.response.status_code", status); + } + self.delivery_span().record( + "openshell.response.selection_cause", + snapshot.selection_cause.as_str(), + ); + self.delivery_span() + .record("openshell.response.delivery_outcome", delivery.as_str()); + self.delivery_span().record( + "openshell.response.delivery_started", + snapshot.delivery_started, ); + self.delivery_span().record( + "openshell.response.partial_delivery", + snapshot.partial_delivery, + ); + self.delivery_span() + .record("openshell.http.outcome", snapshot.http_outcome); + if let Some(stage) = snapshot.error_stage { + self.delivery_span().record("openshell.error.stage", stage); + } + + self.record_final_http_outcome(snapshot.http_outcome); + if snapshot.delivery_outcome != ResponseDeliveryOutcome::Complete + || snapshot.http_outcome != "allowed" + { + openshell_otel::mark_error(&self.span); + openshell_otel::mark_error(&self.delivery_span()); + } + let committed = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .primary + .is_none(); + if committed + && (snapshot.delivery_outcome != ResponseDeliveryOutcome::Complete + || snapshot.partial_delivery) + { + let mut event = response_terminal_event(&self.event_context, snapshot); + self.enrich_event(&mut event); + self.delivery_span() + .in_scope(|| crate::telemetry::emit_correlated(event)); + } } pub(crate) fn emit(&self, event: OcsfEvent) { @@ -1078,6 +1433,26 @@ impl RequestCorrelation { return; }; let mut primary = primary; + if let Some(outcome) = state.response.terminal { + primary.add_unmapped("response_delivery_outcome", outcome.as_str()); + primary.add_unmapped("delivery_started", state.response.delivery_started); + primary.add_unmapped("partial_delivery", state.response.partial_delivery); + primary.add_unmapped( + "interim_delivery_started", + state.response.interim_bytes_forwarded > 0, + ); + primary.add_unmapped( + "interim_bytes_forwarded", + state.response.interim_bytes_forwarded, + ); + } + if let Some(cause) = state + .response + .selection_cause + .filter(|cause| *cause != ResponseSelectionCause::UpstreamResponse) + { + primary.add_unmapped("response_selection_cause", cause.as_str()); + } if let Some(snapshot) = state.credential.snapshot() && snapshot.outcome != CredentialOutcome::Prepared && let OcsfEvent::HttpActivity(http) = &mut primary @@ -1200,18 +1575,8 @@ pub fn emit_phase_events( }; let span = correlation.middleware_span(phase); for mut event in events { - if let OcsfEvent::HttpActivity(http) = &event { - // Routine invocation records belong to the phase span. A late - // failure is separate from the already-written primary record. - let emitted = correlation - .state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .primary - .is_none(); - if !emitted || http.base.status != Some(StatusId::Failure) { - continue; - } + if matches!(event, OcsfEvent::HttpActivity(_)) { + continue; } correlation.enrich_event(&mut event); span.in_scope(|| crate::telemetry::emit_correlated(event)); @@ -1221,10 +1586,21 @@ pub fn emit_phase_events( /// Observe accepted response bytes without buffering or changing delivery. pub struct AuditedClient<'a, C> { inner: &'a mut C, - correlation: &'a RequestCorrelation, + correlation: Option<&'a RequestCorrelation>, head: ResponseHead, } +pub fn observe_response<'a, C>( + client: &'a mut C, + correlation: Option<&'a RequestCorrelation>, +) -> AuditedClient<'a, C> { + AuditedClient { + inner: client, + correlation, + head: ResponseHead::default(), + } +} + #[derive(Default)] struct ResponseHead { prefix: [u8; 12], @@ -1247,8 +1623,9 @@ impl ResponseHead { if self.delimiter != u32::from_be_bytes(*b"\r\n\r\n") { continue; } - let status = std::str::from_utf8(&self.prefix[9..12]) - .ok() + let status = matches!(&self.prefix[..9], b"HTTP/1.0 " | b"HTTP/1.1 ") + .then(|| std::str::from_utf8(&self.prefix[9..12]).ok()) + .flatten() .and_then(|code| code.parse::().ok()); if status.is_some_and(|code| (100..200).contains(&code) && code != 101) { *self = Self::default(); @@ -1280,8 +1657,10 @@ impl AsyncWrite for AuditedClient<'_, C> { ) -> Poll> { let this = self.get_mut(); let result = Pin::new(&mut *this.inner).poll_write(cx, bytes); - if let Poll::Ready(Ok(accepted)) = result { - this.head.accept(&bytes[..accepted], this.correlation); + if let Poll::Ready(Ok(accepted)) = result + && let Some(correlation) = this.correlation + { + this.head.accept(&bytes[..accepted], correlation); } result } @@ -1405,6 +1784,64 @@ fn mcp_method_classification(info: &jsonrpc::JsonRpcRequestInfo) -> &'static str } } +fn response_terminal_event( + context: &HttpEventContext, + snapshot: ResponseDeliverySnapshot, +) -> OcsfEvent { + let (severity, status, message) = match snapshot.delivery_outcome { + ResponseDeliveryOutcome::Complete => ( + SeverityId::Informational, + StatusId::Success, + "HTTP response delivery completed", + ), + ResponseDeliveryOutcome::CancelledOrIncomplete => ( + SeverityId::Low, + StatusId::Unknown, + "HTTP response delivery did not complete", + ), + ResponseDeliveryOutcome::DownstreamIoError => ( + SeverityId::Medium, + StatusId::Failure, + "HTTP response delivery failed downstream", + ), + }; + let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::for_http_method(context.method)) + .action(ActionId::Observed) + .disposition(openshell_ocsf::DispositionId::Logged) + .severity(severity) + .status(status) + .http_request(HttpRequest { + http_method: context.method.parse().expect("bounded HTTP method"), + url: None, + }) + .dst_endpoint(Endpoint::from_domain(&context.host, context.port)) + .firewall_rule(&context.policy_name, "response-correlation") + .unmapped("response_event_kind", "delivery_terminal") + .unmapped( + "response_selection_cause", + snapshot.selection_cause.as_str(), + ) + .unmapped( + "response_delivery_outcome", + snapshot.delivery_outcome.as_str(), + ) + .unmapped( + "interim_delivery_started", + snapshot.interim_bytes_forwarded > 0, + ) + .unmapped("interim_bytes_forwarded", snapshot.interim_bytes_forwarded) + .unmapped("delivery_started", snapshot.delivery_started) + .unmapped("partial_delivery", snapshot.partial_delivery); + if let Some(status) = snapshot.upstream_status { + event = event.unmapped("upstream_response_code", status); + } + if let Some(status) = snapshot.effective_status { + event = event.http_response(HttpResponse { code: status }); + } + event.message(message).build() +} + fn bounded_http_method(method: &str) -> &'static str { [ "CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE", @@ -1416,6 +1853,21 @@ fn bounded_http_method(method: &str) -> &'static str { impl Drop for RequestCorrelation { fn drop(&mut self) { + let unfinished_response = { + let state = self + .state + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.response.lifecycle_started && state.response.terminal.is_none() + }; + if unfinished_response { + self.complete_response_failure( + ResponseDeliveryOutcome::CancelledOrIncomplete, + Some(ResponseSelectionCause::Cancelled), + "incomplete", + "response_cancelled_or_incomplete", + ); + } if self .state .get_mut() @@ -1427,6 +1879,24 @@ impl Drop for RequestCorrelation { openshell_otel::mark_error(&self.span); } self.emit_primary(true); + let root = std::mem::replace(&mut self.span, tracing::Span::none()); + if let Some(dispatch) = root.with_subscriber(|(_, dispatch)| dispatch.clone()) { + tracing::dispatcher::with_default(&dispatch, || { + let state = self + .state + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + drop(std::mem::take(&mut state.middleware)); + drop([ + state.policy_span.take(), + state.credential_span.take(), + state.mcp_span.take(), + state.response_span.take(), + state.upstream_span.take(), + ]); + drop(root); + }); + } } } @@ -1685,6 +2155,58 @@ mod tests { } } + #[test] + fn request_children_close_under_their_own_subscriber() { + use tracing_subscriber::layer::SubscriberExt as _; + + let left_exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let right_exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let left_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(left_exporter.clone()) + .build(); + let right_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(right_exporter.clone()) + .build(); + let left = + tracing_subscriber::registry().with(openshell_otel::layer(&left_provider, "left")); + let right = + tracing_subscriber::registry().with(openshell_otel::layer(&right_provider, "right")); + let correlation = tracing::subscriber::with_default(left, || { + let correlation = RequestCorrelation::new(&request(), &L7EvalContext::default()); + correlation + .observe_middleware(MiddlewarePhase::Request, MiddlewareObservation::default()); + correlation + .observe_credential(CredentialMechanism::TokenGrant, CredentialSurface::Header); + correlation.begin_response_lifecycle(); + correlation.record_response_delivery_started(); + correlation + }); + tracing::subscriber::with_default(right, || { + let other = tracing::info_span!("unrelated"); + drop(correlation); + other.in_scope(|| tracing::info!("still usable")); + }); + + let left_spans = left_exporter.get_finished_spans().unwrap(); + let root = left_spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + assert_eq!(left_spans.len(), 4); + assert!( + left_spans + .iter() + .filter(|span| span.name != root.name) + .all(|span| { + span.parent_span_id == root.span_context.span_id() + && span.span_context.trace_id() == root.span_context.trace_id() + }) + ); + let right_spans = right_exporter.get_finished_spans().unwrap(); + assert_eq!(right_spans.len(), 1); + assert_eq!(right_spans[0].name, "unrelated"); + } + #[test] fn extracts_only_traceparent_and_preserves_duplicates() { let headers = trace_headers( diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 10d47e642b..f3edede1e4 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -43,6 +43,8 @@ use openshell_ocsf::ctx::ctx as ocsf_ctx; use sha1::{Digest, Sha1}; use std::collections::{HashMap, HashSet}; use std::fmt::{self, Write as _}; +use std::future::Future; +use thiserror::Error; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tracing::debug; @@ -93,11 +95,131 @@ pub(crate) const UNSUPPORTED_JSONRPC_UPGRADE_DETAIL: &str = pub(crate) const UNSUPPORTED_GRAPHQL_UPGRADE_DETAIL: &str = "HTTP upgrade is not supported for GraphQL endpoints; serve GraphQL over WebSocket from a separate protocol: websocket endpoint on another path or port"; const MIN_HTTP2_PREFACE_DETECTION_BYTES: usize = 8; -/// Idle timeout for `relay_until_eof`. If no data arrives within this window -/// the body is considered complete. Prevents blocking on servers that keep -/// the TCP connection alive after the response body (common with CDN keep-alive). +/// Idle timeout for an unframed, close-delimited response body. const RELAY_EOF_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +async fn guarded_relay_read( + guard: Option<&PolicyGenerationGuard>, + read: impl Future>, +) -> Result { + let Some(guard) = guard else { + return read.await; + }; + let mut read = std::pin::pin!(read); + tokio::select! { + result = &mut read => result, + () = guard.wait_until_stale() => { + if guard.is_stale() { + Err(RelayError::PolicyReloaded(format!( + "policy generation changed [captured_generation:{} current_generation:{}]", + guard.captured_generation(), + guard.current_generation(), + )).into()) + } else { + read.await + } + }, + } +} + +fn ensure_relay_generation_current(guard: &PolicyGenerationGuard) -> Result<()> { + guard + .ensure_current() + .map_err(|error| RelayError::PolicyReloaded(error.to_string()).into()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RelayWriteStage { + Write, + Flush, + Shutdown, + InterimWrite, + InterimFlush, +} + +#[derive(Debug, Error)] +enum RelayError { + #[error("HTTP relay read failed: {0}")] + Read(#[source] std::io::Error), + #[error("HTTP relay write failed: {source}")] + Write { + #[source] + source: std::io::Error, + stage: RelayWriteStage, + }, + #[error("{detail}")] + Protocol { code: &'static str, detail: String }, + #[error("{0}")] + Truncated(&'static str), + #[error("HTTP response body idle timeout")] + IdleTimeout, + #[error("{0}")] + PolicyReloaded(String), +} + +impl RelayError { + fn read(source: std::io::Error) -> Self { + Self::Read(source) + } + + fn write(source: std::io::Error) -> Self { + Self::Write { + source, + stage: RelayWriteStage::Write, + } + } + + fn flush(source: std::io::Error) -> Self { + Self::Write { + source, + stage: RelayWriteStage::Flush, + } + } + + fn shutdown(source: std::io::Error) -> Self { + Self::Write { + source, + stage: RelayWriteStage::Shutdown, + } + } + + fn interim_write(source: std::io::Error) -> Self { + Self::Write { + source, + stage: RelayWriteStage::InterimWrite, + } + } + + fn interim_flush(source: std::io::Error) -> Self { + Self::Write { + source, + stage: RelayWriteStage::InterimFlush, + } + } + + fn protocol(code: &'static str, detail: impl Into) -> Self { + Self::Protocol { + code, + detail: detail.into(), + } + } +} + +impl miette::Diagnostic for RelayError { + fn code<'a>(&'a self) -> Option> { + match self { + Self::Protocol { code, .. } => Some(Box::new(code)), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RelayUntilEofOutcome { + Complete, + IdleTimeout, +} + /// HTTP/1.1 REST protocol provider. /// /// Carries the path-canonicalization options derived from the endpoint @@ -3729,16 +3851,24 @@ where let to_read = usize::try_from(remaining) .unwrap_or(buf.len()) .min(buf.len()); - let n = reader.read(&mut buf[..to_read]).await.into_diagnostic()?; - if n == 0 { - return Err(miette!( - "Connection closed with {remaining} bytes remaining" - )); - } + let n = guarded_relay_read(generation_guard, async { + reader + .read(&mut buf[..to_read]) + .await + .map_err(RelayError::read) + .map_err(miette::Report::new) + }) + .await?; if let Some(guard) = generation_guard { - guard.ensure_current()?; + ensure_relay_generation_current(guard)?; + } + if n == 0 { + return Err(RelayError::Truncated("HTTP body ended before Content-Length").into()); } - writer.write_all(&buf[..n]).await.into_diagnostic()?; + writer + .write_all(&buf[..n]) + .await + .map_err(RelayError::write)?; remaining -= n as u64; } Ok(()) @@ -3772,6 +3902,7 @@ where let size_line = input .read_line( reader, + generation_guard, "Chunked body ended before chunk-size line", "Chunk-size line exceeds limit", ) @@ -3786,6 +3917,7 @@ where let trailer_line = input .read_line( reader, + generation_guard, "Chunked body ended before trailer terminator", "Chunk trailer line exceeds limit", ) @@ -3837,10 +3969,18 @@ where .await?; let terminator = [ input - .read_byte(reader, "Chunked body ended before chunk terminator") + .read_byte( + reader, + generation_guard, + "Chunked body ended before chunk terminator", + ) .await?, input - .read_byte(reader, "Chunked body ended before chunk terminator") + .read_byte( + reader, + generation_guard, + "Chunked body ended before chunk terminator", + ) .await?, ]; if terminator != *b"\r\n" { @@ -3874,6 +4014,7 @@ impl<'a> ChunkedRelayInput<'a> { async fn read_line( &mut self, reader: &mut R, + generation_guard: Option<&PolicyGenerationGuard>, eof_message: &'static str, limit_message: &'static str, ) -> Result> @@ -3885,7 +4026,10 @@ impl<'a> ChunkedRelayInput<'a> { if line.len() == MAX_CHUNK_LINE_BYTES { return Err(miette!(limit_message)); } - line.push(self.read_byte(reader, eof_message).await?); + line.push( + self.read_byte(reader, generation_guard, eof_message) + .await?, + ); if line.ends_with(b"\r\n") { line.truncate(line.len() - 2); return Ok(line); @@ -3922,24 +4066,33 @@ impl<'a> ChunkedRelayInput<'a> { } let start = self.pending.len(); self.pending.resize(start + to_read, 0); - let read = reader - .read(&mut self.pending[start..]) - .await - .into_diagnostic()?; + let read = guarded_relay_read(generation_guard, async { + reader + .read(&mut self.pending[start..]) + .await + .map_err(RelayError::read) + .map_err(miette::Report::new) + }) + .await?; if read == 0 { self.pending.truncate(start); - return Err(miette!(eof_message)); + return Err(RelayError::Truncated(eof_message).into()); } self.pending.truncate(start + read); if let Some(guard) = generation_guard { - guard.ensure_current()?; + ensure_relay_generation_current(guard)?; } remaining -= read; } Ok(()) } - async fn read_byte(&mut self, reader: &mut R, eof_message: &'static str) -> Result + async fn read_byte( + &mut self, + reader: &mut R, + generation_guard: Option<&PolicyGenerationGuard>, + eof_message: &'static str, + ) -> Result where R: AsyncRead + Unpin, { @@ -3949,13 +4102,19 @@ impl<'a> ChunkedRelayInput<'a> { self.pending.push(byte); return Ok(byte); } - let byte = match reader.read_u8().await { - Ok(byte) => byte, - Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { - return Err(miette!(eof_message)); + let byte = guarded_relay_read(generation_guard, async { + match reader.read_u8().await { + Ok(byte) => Ok(byte), + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { + Err(RelayError::Truncated(eof_message).into()) + } + Err(error) => Err(RelayError::read(error).into()), } - Err(error) => return Err(error).into_diagnostic(), - }; + }) + .await?; + if let Some(guard) = generation_guard { + ensure_relay_generation_current(guard)?; + } self.pending.push(byte); Ok(byte) } @@ -3972,9 +4131,12 @@ impl<'a> ChunkedRelayInput<'a> { return Ok(()); } if let Some(guard) = generation_guard { - guard.ensure_current()?; + ensure_relay_generation_current(guard)?; } - writer.write_all(&self.pending).await.into_diagnostic()?; + writer + .write_all(&self.pending) + .await + .map_err(RelayError::write)?; self.pending.clear(); Ok(()) } @@ -4178,26 +4340,48 @@ fn is_bodiless_response(request_method: &str, status_code: u16) -> bool { /// no Transfer-Encoding) where the body is delimited by connection close. /// An idle timeout prevents blocking when servers keep the TCP connection /// alive longer than expected (e.g. CDN keep-alive timers). -async fn relay_until_eof(reader: &mut R, writer: &mut W) -> Result<()> +async fn relay_until_eof( + reader: &mut R, + writer: &mut W, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result where R: AsyncRead + Unpin, W: AsyncWrite + Unpin, { let mut buf = [0u8; RELAY_BUF_SIZE]; loop { - match tokio::time::timeout(RELAY_EOF_IDLE_TIMEOUT, reader.read(&mut buf)).await { - Ok(Ok(0)) => return Ok(()), + match tokio::time::timeout( + RELAY_EOF_IDLE_TIMEOUT, + guarded_relay_read(generation_guard, async { + reader + .read(&mut buf) + .await + .map_err(|error| RelayError::read(error).into()) + }), + ) + .await + { Ok(Ok(n)) => { - writer.write_all(&buf[..n]).await.into_diagnostic()?; - writer.flush().await.into_diagnostic()?; + if let Some(guard) = generation_guard { + ensure_relay_generation_current(guard)?; + } + if n == 0 { + return Ok(RelayUntilEofOutcome::Complete); + } + writer + .write_all(&buf[..n]) + .await + .map_err(RelayError::write)?; + writer.flush().await.map_err(RelayError::flush)?; } - Ok(Err(e)) => return Err(miette::miette!("{e}")), + Ok(Err(error)) => return Err(error), Err(_) => { debug!( "relay_until_eof idle timeout after {:?}", RELAY_EOF_IDLE_TIMEOUT ); - return Ok(()); + return Ok(RelayUntilEofOutcome::IdleTimeout); } } } @@ -4207,19 +4391,35 @@ where /// /// Used for server-sent events, where long idle gaps are part of the protocol /// and do not mean the response body is complete. -async fn relay_until_eof_without_idle_timeout(reader: &mut R, writer: &mut W) -> Result<()> +async fn relay_until_eof_without_idle_timeout( + reader: &mut R, + writer: &mut W, + generation_guard: Option<&PolicyGenerationGuard>, +) -> Result<()> where R: AsyncRead + Unpin, W: AsyncWrite + Unpin, { let mut buf = [0u8; RELAY_BUF_SIZE]; loop { - let n = reader.read(&mut buf).await.into_diagnostic()?; + let n = guarded_relay_read(generation_guard, async { + reader + .read(&mut buf) + .await + .map_err(|error| RelayError::read(error).into()) + }) + .await?; + if let Some(guard) = generation_guard { + ensure_relay_generation_current(guard)?; + } if n == 0 { return Ok(()); } - writer.write_all(&buf[..n]).await.into_diagnostic()?; - writer.flush().await.into_diagnostic()?; + writer + .write_all(&buf[..n]) + .await + .map_err(RelayError::write)?; + writer.flush().await.map_err(RelayError::flush)?; } } @@ -4300,6 +4500,40 @@ mod tests { const VALID_WS_ACCEPT: &str = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="; const TEXT_OPCODE: u8 = 0x1; + #[tokio::test] + async fn guarded_relay_read_ignores_closed_generation_channel_while_current() { + use tokio::io::AsyncWriteExt as _; + + let engine = + OpaEngine::from_strings(TEST_POLICY, "network_policies: {}\n").expect("test policy"); + let guard = engine + .generation_guard(engine.current_generation()) + .expect("generation guard"); + drop(engine); + assert!(!guard.is_stale()); + + let (mut reader, mut writer) = tokio::io::duplex(1); + let send = tokio::spawn(async move { + tokio::task::yield_now().await; + writer.write_all(b"x").await.expect("seed guarded read"); + }); + let mut byte = [0u8; 1]; + + let read = tokio::time::timeout( + std::time::Duration::from_millis(100), + guarded_relay_read(Some(&guard), async { + reader.read(&mut byte).await.into_diagnostic() + }), + ) + .await + .expect("closed generation sender must not cancel a current read") + .expect("guarded read"); + send.await.expect("seed task"); + + assert_eq!(read, 1); + assert_eq!(byte, *b"x"); + } + #[derive(Debug, Clone, Copy)] enum ResponseRelayScript { HeadersOnly, @@ -9485,7 +9719,7 @@ mod tests { async fn relay_rejects_unsolicited_101_without_client_upgrade_header() { // Client sends a normal GET without Upgrade headers. // Upstream responds with 101 (non-compliant). The relay should - // reject the upgrade and return Consumed instead. + // reject the upgrade as a protocol failure. let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); @@ -9533,10 +9767,12 @@ mod tests { .await .expect("relay must not deadlock"); - let outcome = result.expect("relay should succeed"); + let error = result.expect_err("unsolicited 101 should be rejected"); assert!( - matches!(outcome, RelayOutcome::Consumed), - "unsolicited 101 should be rejected as Consumed, got {outcome:?}" + error + .to_string() + .contains("without a client upgrade request"), + "{error}" ); upstream_task.await.expect("upstream task should complete"); diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 1b152b6a99..fa086ed3ea 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -5,14 +5,20 @@ use super::*; use crate::l7::correlation::{ - MiddlewareObservation, MiddlewareOutcome, MiddlewarePhase, RequestCorrelation, emit_ocsf, - emit_phase_events, + MiddlewareObservation, MiddlewareOutcome, MiddlewarePhase, RequestCorrelation, + ResponseDeliveryOutcome, ResponseSelectionCause, emit_ocsf, emit_phase_events, }; /// Default wall-clock bound shared by whole-body stages in one response. pub const DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT: std::time::Duration = std::time::Duration::from_mins(2); +fn ensure_response_policy_current(guard: &PolicyGenerationGuard) -> Result<()> { + guard + .ensure_current() + .map_err(|error| miette::Report::new(RelayError::PolicyReloaded(error.to_string()))) +} + /// Context retained from request evaluation for the matching response hook. pub struct HttpResponseMiddlewareRelay<'a> { pub(crate) chain: &'a [openshell_supervisor_middleware::ChainEntry], @@ -52,11 +58,49 @@ pub(super) async fn relay_response( options: RelayResponseOptions<'_>, response_middleware: Option>, ) -> Result +where + U: AsyncRead + Unpin, + C: AsyncWrite + Unpin, +{ + let correlation = options.correlation; + if let Some(correlation) = correlation { + correlation.begin_response_lifecycle(); + } + let mut audited_client = crate::l7::correlation::observe_response(client, correlation); + let result = relay_response_inner( + request_method, + upstream, + &mut audited_client, + options, + response_middleware, + ) + .await; + match &result { + Ok(_) => { + if let Some(correlation) = correlation { + correlation.complete_pending_response_delivery(); + } + } + Err(error) => complete_response_error(correlation, error), + } + result +} + +async fn relay_response_inner( + request_method: &str, + upstream: &mut U, + client: &mut C, + options: RelayResponseOptions<'_>, + response_middleware: Option>, +) -> Result where U: AsyncRead + Unpin, C: AsyncWrite + Unpin, { let started_at = std::time::Instant::now(); + let response_generation_guard = response_middleware + .as_ref() + .and_then(|middleware| middleware.generation_guard); let mut buf = Vec::with_capacity(4096); let mut tmp = [0u8; 1024]; @@ -75,7 +119,11 @@ where if let Some(observer) = options.observer.as_ref() { observer.observe(EndpointResult::TransportFailed); } - return Err(miette!("HTTP response headers exceed limit")); + return Err(RelayError::protocol( + "response_headers_too_large", + "HTTP response headers exceed limit", + ) + .into()); } if let Some(header_end) = header_end { @@ -83,8 +131,9 @@ where if let Some(observer) = options.observer.as_ref() { observer.observe(EndpointResult::TransportFailed); } - record_upstream_response_error(options.correlation, "upstream_response_headers"); - return Err(error); + return Err( + RelayError::protocol("invalid_response_headers", error.to_string()).into(), + ); } let header_str = String::from_utf8_lossy(&buf[..header_end]); if matches!( @@ -93,21 +142,35 @@ where ) { informational_header_bytes += header_end; let sanitized = strip_platform_request_id_from_head(&buf[..header_end]); - client.write_all(&sanitized).await.into_diagnostic()?; - client.flush().await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + write_correlated_interim_bytes(client, &sanitized, options.correlation).await?; + client.flush().await.map_err(RelayError::interim_flush)?; buf.drain(..header_end); continue; } break header_end; } - let n = upstream.read(&mut tmp).await.into_diagnostic()?; + let n = guarded_relay_read(response_generation_guard, async { + upstream + .read(&mut tmp) + .await + .map_err(RelayError::read) + .map_err(miette::Report::new) + }) + .await?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } if n == 0 { if let Some(observer) = options.observer.as_ref() { observer.observe(EndpointResult::TransportFailed); } if let Some(correlation) = options.correlation { send_incomplete_response(client, correlation).await?; + finish_response(client, true).await?; } else if !buf.is_empty() { client.write_all(&buf).await.into_diagnostic()?; } @@ -132,7 +195,7 @@ where let header_str = String::from_utf8_lossy(&buf[..header_end]); let observed_status = parse_observed_http_status_code(&header_str); if let (Some(correlation), Some(status_code)) = (options.correlation, observed_status) { - correlation.record_upstream_status(status_code); + correlation.record_upstream_response_status(status_code); } if let Some(observer) = options.observer.as_ref() { match observed_status { @@ -148,7 +211,17 @@ where let http_10_closes_by_default = response_is_http_10(&header_str) && !parse_connection_keep_alive(&header_str); let event_stream = response_is_event_stream(&header_str); - let body_length = parse_body_length(&header_str)?; + let body_length = parse_body_length(&header_str) + .map_err(|error| RelayError::protocol("invalid_response_framing", error.to_string()))?; + if let BodyLength::ContentLength(length) = body_length + && u64::try_from(buf.len() - header_end).unwrap_or(u64::MAX) > length + { + return Err(RelayError::protocol( + "content_length_overflow", + "HTTP response body exceeds Content-Length", + ) + .into()); + } debug!( status_code, @@ -166,23 +239,31 @@ where // upgraded protocol and must be forwarded before switching. if status_code == 101 { if !options.client_requested_upgrade { - return Ok(RelayOutcome::Consumed); + return Err(RelayError::protocol( + "unsolicited_switching_protocols", + "upstream returned 101 Switching Protocols without a client upgrade request", + ) + .into()); } let (websocket_permessage_deflate, websocket_subprotocol) = validate_websocket_response( &header_str, options.websocket_extensions, options.websocket.as_ref(), - )?; + ) + .map_err(|error| RelayError::protocol("invalid_websocket_handshake", error.to_string()))?; let final_head = final_response_head(&buf[..header_end], options.correlation); - client.write_all(&final_head).await.into_diagnostic()?; - client.flush().await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + select_allowed_response(options.correlation, status_code); + write_correlated_response_bytes(client, &final_head, options.correlation, false).await?; + client.flush().await.map_err(RelayError::flush)?; let overflow = buf[header_end..].to_vec(); debug!( request_method, overflow_bytes = overflow.len(), "101 Switching Protocols — signaling protocol upgrade" ); - record_allowed_response(options.correlation); return Ok(RelayOutcome::Upgraded { overflow, websocket_permessage_deflate, @@ -217,10 +298,13 @@ where // Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body if is_bodiless_response(request_method, status_code) { - client.write_all(&final_head).await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + select_allowed_response(options.correlation, status_code); + write_correlated_response_bytes(client, &final_head, options.correlation, false).await?; let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; - record_allowed_response(options.correlation); return Ok(outcome); } @@ -231,42 +315,73 @@ where // Server indicated it will close, or this is a streaming response // such as SSE where the body is intentionally delimited by EOF. let before_end = &final_head[..final_head.len() - 2]; - client.write_all(before_end).await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + select_allowed_response(options.correlation, status_code); + write_correlated_response_bytes(client, before_end, options.correlation, false).await?; if server_wants_close || http_10_closes_by_default { client .write_all(b"Connection: close\r\n\r\n") .await - .into_diagnostic()?; + .map_err(RelayError::write)?; } else { - client.write_all(b"\r\n").await.into_diagnostic()?; + client.write_all(b"\r\n").await.map_err(RelayError::write)?; } let overflow = &buf[header_end..]; if !overflow.is_empty() { - client.write_all(overflow).await.into_diagnostic()?; - client.flush().await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + client + .write_all(overflow) + .await + .map_err(RelayError::write)?; + client.flush().await.map_err(RelayError::flush)?; } if event_stream { - relay_until_eof_without_idle_timeout(upstream, client).await?; + relay_until_eof_without_idle_timeout(upstream, client, response_generation_guard) + .await?; } else { - relay_until_eof(upstream, client).await?; + match relay_until_eof(upstream, client, response_generation_guard).await? { + RelayUntilEofOutcome::Complete => {} + RelayUntilEofOutcome::IdleTimeout => { + record_committed_response_error( + options.correlation, + "upstream_error", + "upstream_response_idle_timeout", + ); + if let Some(correlation) = options.correlation { + correlation.record_pending_response_delivery( + ResponseDeliveryOutcome::CancelledOrIncomplete, + ); + } + } + } } - client.flush().await.into_diagnostic()?; - client.shutdown().await.into_diagnostic()?; - record_allowed_response(options.correlation); + client.flush().await.map_err(RelayError::flush)?; + client.shutdown().await.map_err(RelayError::shutdown)?; return Ok(RelayOutcome::Consumed); } // No Connection: close — an HTTP/1.1 keep-alive server that omits // framing headers has an empty body. Forward headers and continue // the relay loop instead of blocking on relay_until_eof. debug!("BodyLength::None without Connection: close — treating body as empty"); - client.write_all(&final_head).await.into_diagnostic()?; - client.flush().await.into_diagnostic()?; - record_allowed_response(options.correlation); + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + select_allowed_response(options.correlation, status_code); + write_correlated_response_bytes(client, &final_head, options.correlation, false).await?; + client.flush().await.map_err(RelayError::flush)?; return Ok(RelayOutcome::Reusable); } // Hold buffered chunk framing until relay_chunked validates it. - client.write_all(&final_head).await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + select_allowed_response(options.correlation, status_code); + write_correlated_response_bytes(client, &final_head, options.correlation, false).await?; let overflow = &buf[header_end..]; let overflow_len = (buf.len() - header_end) as u64; @@ -274,19 +389,25 @@ where match body_length { BodyLength::ContentLength(len) => { if !overflow.is_empty() { - client.write_all(overflow).await.into_diagnostic()?; + if let Some(guard) = response_generation_guard { + ensure_response_policy_current(guard)?; + } + client + .write_all(overflow) + .await + .map_err(RelayError::write)?; } - let remaining = len.saturating_sub(overflow_len); + let remaining = len - overflow_len; if remaining > 0 { - relay_fixed(upstream, client, remaining, None).await?; + relay_fixed(upstream, client, remaining, response_generation_guard).await?; } } - BodyLength::Chunked => { - relay_chunked(upstream, client, overflow, None).await?; - } + BodyLength::Chunked => relay_chunked(upstream, client, overflow, response_generation_guard) + .await + .map_err(type_response_protocol_error)?, BodyLength::None => unreachable!(), } - client.flush().await.into_diagnostic()?; + client.flush().await.map_err(RelayError::flush)?; debug!( request_method, elapsed_ms = started_at.elapsed().as_millis(), @@ -294,7 +415,6 @@ where ); let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; - record_allowed_response(options.correlation); Ok(outcome) } @@ -315,51 +435,186 @@ async fn send_incomplete_response( body.len(), body, ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic() + correlation.record_response_status(502); + write_correlated_response_bytes(client, response.as_bytes(), Some(correlation), false).await?; + client.flush().await.map_err(RelayError::flush)?; + Ok(()) } -fn record_allowed_response(correlation: Option<&RequestCorrelation>) { +fn select_allowed_response(correlation: Option<&RequestCorrelation>, status_code: u16) { if let Some(correlation) = correlation { - correlation.record_final_http_outcome("allowed"); + correlation.record_response_status(status_code); + correlation.record_response_selection( + ResponseSelectionCause::UpstreamResponse, + "allowed", + None, + ); } } -fn record_denied_response(correlation: Option<&RequestCorrelation>, stage: &str) { +fn record_denied_response(correlation: Option<&RequestCorrelation>, stage: &'static str) { if let Some(correlation) = correlation { correlation.record_response_status(403); - correlation.record_final_http_outcome("denied"); - correlation.record_error_stage(stage); + correlation.record_response_selection( + ResponseSelectionCause::MiddlewareDenied, + "denied", + Some(stage), + ); } } -fn record_platform_response_error(correlation: Option<&RequestCorrelation>, stage: &str) { +fn select_failure_response( + correlation: Option<&RequestCorrelation>, + cause: ResponseSelectionCause, + http_outcome: &'static str, + stage: &'static str, +) { if let Some(correlation) = correlation { correlation.record_response_status(502); - correlation.record_final_http_outcome("platform_error"); - correlation.record_error_stage(stage); + correlation.record_response_selection(cause, http_outcome, Some(stage)); } } -fn record_upstream_response_error(correlation: Option<&RequestCorrelation>, stage: &str) { +fn record_upstream_response_error(correlation: Option<&RequestCorrelation>, stage: &'static str) { if let Some(correlation) = correlation { - correlation.record_response_status(502); - correlation.record_final_http_outcome("upstream_error"); - correlation.record_error_stage(stage); + correlation.record_response_selection( + ResponseSelectionCause::UpstreamTruncated, + "upstream_error", + Some(stage), + ); + } +} + +#[derive(Debug, Clone, Copy)] +struct ResponseFailureClassification { + cause: Option, + delivery: ResponseDeliveryOutcome, + http_outcome: &'static str, + error_stage: &'static str, +} + +fn type_response_protocol_error(error: miette::Report) -> miette::Report { + if error.downcast_ref::().is_some() { + error + } else { + RelayError::protocol("invalid_response_body", error.to_string()).into() + } +} + +fn complete_response_error(correlation: Option<&RequestCorrelation>, error: &miette::Report) { + let Some(correlation) = correlation else { + return; + }; + let failure = classify_response_failure(error); + let _ = correlation.record_response_partial_delivery(); + correlation.complete_response_failure( + failure.delivery, + failure.cause, + failure.http_outcome, + failure.error_stage, + ); +} + +fn classify_response_failure(error: &miette::Report) -> ResponseFailureClassification { + if let Some(relay_error) = error.downcast_ref::() { + let (cause, delivery, http_outcome, error_stage) = match relay_error { + RelayError::Write { stage, .. } => { + let (cause, error_stage) = match stage { + RelayWriteStage::Write => (None, "downstream_response_write"), + RelayWriteStage::Flush => (None, "downstream_response_flush"), + RelayWriteStage::Shutdown => (None, "downstream_response_shutdown"), + RelayWriteStage::InterimWrite => ( + Some(ResponseSelectionCause::PlatformError), + "downstream_interim_write", + ), + RelayWriteStage::InterimFlush => ( + Some(ResponseSelectionCause::PlatformError), + "downstream_interim_flush", + ), + }; + ( + cause, + ResponseDeliveryOutcome::DownstreamIoError, + "platform_error", + error_stage, + ) + } + RelayError::IdleTimeout => ( + Some(ResponseSelectionCause::UpstreamIdleTimeout), + ResponseDeliveryOutcome::CancelledOrIncomplete, + "upstream_error", + "upstream_response_idle_timeout", + ), + RelayError::Truncated(_) => ( + Some(ResponseSelectionCause::UpstreamTruncated), + ResponseDeliveryOutcome::CancelledOrIncomplete, + "upstream_error", + "upstream_response_truncated", + ), + RelayError::Read(_) => ( + Some(ResponseSelectionCause::UpstreamIoError), + ResponseDeliveryOutcome::CancelledOrIncomplete, + "upstream_error", + "upstream_response_read", + ), + RelayError::Protocol { .. } => ( + Some(ResponseSelectionCause::UpstreamProtocolError), + ResponseDeliveryOutcome::CancelledOrIncomplete, + "upstream_error", + "upstream_response_protocol", + ), + RelayError::PolicyReloaded(_) => ( + Some(ResponseSelectionCause::PolicyReloaded), + ResponseDeliveryOutcome::CancelledOrIncomplete, + "platform_error", + "response_policy_reload", + ), + }; + return ResponseFailureClassification { + cause, + delivery, + http_outcome, + error_stage, + }; + } + if let Some(stop) = error.downcast_ref::() { + let denied = stop.failure.denial.is_some() + || response_invocations_blocked(&stop.failure.diagnostics.invocations); + return ResponseFailureClassification { + cause: Some(if denied { + ResponseSelectionCause::MiddlewareDenied + } else { + ResponseSelectionCause::MiddlewareFailed + }), + delivery: ResponseDeliveryOutcome::CancelledOrIncomplete, + http_outcome: if denied { "denied" } else { "platform_error" }, + error_stage: if denied { + "response_middleware_denied" + } else { + "response_middleware_failed" + }, + }; + } + ResponseFailureClassification { + cause: Some(ResponseSelectionCause::Unknown), + delivery: ResponseDeliveryOutcome::CancelledOrIncomplete, + http_outcome: "platform_error", + error_stage: "response_unknown", } } fn record_committed_response_error( correlation: Option<&RequestCorrelation>, outcome: &'static str, - stage: &str, + stage: &'static str, ) { if let Some(correlation) = correlation { - correlation.record_final_http_outcome(outcome); - correlation.record_error_stage(stage); + let cause = match stage { + "upstream_response_idle_timeout" => ResponseSelectionCause::UpstreamIdleTimeout, + "response_policy_reload" => ResponseSelectionCause::PolicyReloaded, + _ => ResponseSelectionCause::PlatformError, + }; + correlation.record_response_selection(cause, outcome, Some(stage)); } } @@ -544,9 +799,9 @@ async fn finish_response(client: &mut C, close: bool) -> Result where C: AsyncWrite + Unpin, { - client.flush().await.into_diagnostic()?; + client.flush().await.map_err(RelayError::flush)?; if close { - client.shutdown().await.into_diagnostic()?; + client.shutdown().await.map_err(RelayError::shutdown)?; Ok(RelayOutcome::Consumed) } else { Ok(RelayOutcome::Reusable) @@ -572,7 +827,7 @@ where C: AsyncWrite + Unpin, { if let Some(guard) = middleware.generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } let header_bytes = &buffered[..header_end]; // Ordinary responses retain the HTTP parser's byte-preserving behavior. @@ -580,33 +835,21 @@ where if middleware.chain.is_empty() { return Ok(None); } - let parsed = match middleware + let described_result = middleware .runner .describe_http_response_chain(middleware.chain) - .await - { - Ok(described) if described.is_empty() => return Ok(None), - Ok(described) => { - if let Some(correlation) = response_correlation { - correlation.observe_middleware( - MiddlewarePhase::Response, - MiddlewareObservation::default(), - ); - } - parse_response_head_for_middleware(header_bytes).map(|parsed| (described, parsed)) - } - Err(error) => Err(error), - }; + .await; if let Some(guard) = middleware.generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } - let (described, parsed) = match parsed { - Ok(parsed) => parsed, + let described = match described_result { + Ok(described) if described.is_empty() => return Ok(None), + Ok(described) => described, Err(error) => { let _phase = response_correlation .map(|correlation| correlation.begin_middleware(MiddlewarePhase::Response)); complete_response_middleware(response_correlation, MiddlewareOutcome::PlatformError); - debug!(error = %error, "HTTP response head normalization failed"); + debug!(error = %error, "HTTP response middleware chain description failed"); emit_http_response_middleware_failure( middleware.policy_name, &middleware.target, @@ -619,7 +862,42 @@ where request_method, middleware.policy_name, &middleware.target, - "response_middleware_headers", + ( + ResponseSelectionCause::MiddlewareFailed, + "platform_error", + "response_middleware_describe", + ), + true, + response_correlation, + ) + .await?; + return Ok(Some(RelayOutcome::Consumed)); + } + }; + let parsed = parse_response_head_for_middleware(header_bytes).map_err(|error| { + miette::Report::new(RelayError::protocol( + "invalid_response_headers", + error.to_string(), + )) + }); + if let Some(guard) = middleware.generation_guard { + ensure_response_policy_current(guard)?; + } + let parsed = match parsed { + Ok(parsed) => parsed, + Err(error) => { + debug!(error = %error, "HTTP response head normalization failed"); + send_response_delivery_failure( + client, + request_method, + middleware.policy_name, + &middleware.target, + ( + ResponseSelectionCause::UpstreamProtocolError, + "upstream_error", + "response_middleware_headers", + ), + false, response_correlation, ) .await?; @@ -628,6 +906,9 @@ where }; let _phase = response_correlation .map(|correlation| correlation.begin_middleware(MiddlewarePhase::Response)); + if let Some(correlation) = response_correlation { + correlation.observe_middleware(MiddlewarePhase::Response, MiddlewareObservation::default()); + } let original_headers = parsed.headers.clone(); let preserved_credential_headers = parsed.preserved_credential_headers; let upstream_declared_trailers = parsed.declared_trailers.clone(); @@ -659,7 +940,19 @@ where .runner .http_response_input_unrepresentable(&described)) }; - let mut preflight = match preflight_result { + if let Some(guard) = middleware.generation_guard + && let Err(error) = ensure_response_policy_current(guard) + { + if let Ok(mut preflight) = preflight_result + && let Some(session) = preflight.session.take() + { + session + .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) + .await; + } + return Err(error); + } + let preflight = match preflight_result { Ok(preflight) => preflight, Err(error) => { observe_response_admission_exhaustion( @@ -682,7 +975,12 @@ where request_method, middleware.policy_name, &middleware.target, - "response_middleware_preflight", + ( + ResponseSelectionCause::MiddlewareFailed, + "platform_error", + "response_middleware_preflight", + ), + true, response_correlation, ) .await?; @@ -696,16 +994,6 @@ where preflight.session_capacity_exhausted, false, ); - if let Some(guard) = middleware.generation_guard - && let Err(error) = guard.ensure_current() - { - if let Some(session) = preflight.session.take() { - session - .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) - .await; - } - return Err(error); - } debug!( configured_stage_count = middleware.chain.len(), active_session = preflight.session.is_some(), @@ -764,7 +1052,12 @@ where request_method, middleware.policy_name, &middleware.target, - "response_middleware_preflight", + ( + ResponseSelectionCause::MiddlewareFailed, + "platform_error", + "response_middleware_preflight", + ), + true, response_correlation, ) .await?; @@ -799,10 +1092,10 @@ where body_length, server_wants_close, event_stream, + middleware.generation_guard, response_correlation, ) .await?; - record_allowed_response(response_correlation); return Ok(Some(outcome)); }; @@ -810,7 +1103,11 @@ where let supports_chunked_response = !status_line.starts_with("HTTP/1.0 "); let bodiless = is_bodiless_response(request_method, status_code); if bodiless { - let finish = match session.finish(Vec::new()).await { + let finish_result = session.finish(Vec::new()).await; + if let Some(guard) = middleware.generation_guard { + ensure_response_policy_current(guard)?; + } + let finish = match finish_result { Ok(finish) => finish, Err(error) => { observe_response_diagnostics(response_correlation, &error.diagnostics); @@ -818,9 +1115,14 @@ where response_correlation, error.admission_exhausted(), ); + let denial = response_middleware_failure_denial(&error); complete_response_middleware( response_correlation, - middleware_failure_outcome(error.denial.is_some(), &error.diagnostics), + if denial.is_some() { + MiddlewareOutcome::Denied + } else { + MiddlewareOutcome::PlatformError + }, ); debug!(error = %error, "HTTP response middleware finalization failed"); emit_http_response_diagnostics( @@ -832,15 +1134,40 @@ where Some(false), response_correlation, ); - send_response_delivery_failure( - client, - request_method, - middleware.policy_name, - &middleware.target, - "response_middleware_finish", - response_correlation, - ) - .await?; + if let Some(denial) = denial.as_ref() { + send_response_middleware_denial( + client, + request_method, + middleware.policy_name, + &middleware.target, + denial, + "response_middleware_finish", + response_correlation, + ) + .await?; + } else { + emit_http_response_middleware_failure( + middleware.policy_name, + &middleware.target, + status_code, + false, + response_correlation, + ); + send_response_delivery_failure( + client, + request_method, + middleware.policy_name, + &middleware.target, + ( + ResponseSelectionCause::MiddlewareFailed, + "platform_error", + "response_middleware_finish", + ), + true, + response_correlation, + ) + .await?; + } return Ok(Some(RelayOutcome::Consumed)); } }; @@ -851,9 +1178,6 @@ where false, finish.diagnostics_truncated, ); - if let Some(guard) = middleware.generation_guard { - guard.ensure_current()?; - } let mut headers = preflight.headers; emit_http_response_middleware_invocations( middleware.policy_name, @@ -884,9 +1208,9 @@ where ); let head = final_response_head(&head, response_correlation); complete_response_middleware(response_correlation, MiddlewareOutcome::Allowed); - client.write_all(&head).await.into_diagnostic()?; - client.flush().await.into_diagnostic()?; - record_allowed_response(response_correlation); + select_allowed_response(response_correlation, status_code); + write_correlated_response_bytes(client, &head, response_correlation, true).await?; + client.flush().await.map_err(RelayError::flush)?; return Ok(Some(if server_wants_close { RelayOutcome::Consumed } else { @@ -920,20 +1244,24 @@ where downstream_trailers, ); let streaming_head = final_response_head(&streaming_head, response_correlation); - let mut committed = !whole_body; - if committed { - if let Err(error) = client.write_all(&streaming_head).await { + let mut committed = false; + if !whole_body { + select_allowed_response(response_correlation, status_code); + if let Err(error) = + write_correlated_response_bytes(client, &streaming_head, response_correlation, true) + .await + { session .end(openshell_core::proto::MiddlewareSessionEndReason::DownstreamDisconnect) .await; - return Err(error).into_diagnostic(); + return Err(error); } - record_response_delivery_started(response_correlation); + committed = true; if let Err(error) = client.flush().await { session .end(openshell_core::proto::MiddlewareSessionEndReason::DownstreamDisconnect) .await; - return Err(error).into_diagnostic(); + return Err(RelayError::flush(error).into()); } } @@ -966,7 +1294,10 @@ where .is_some_and(PolicyGenerationGuard::is_stale) { openshell_core::proto::MiddlewareSessionEndReason::PolicyReload - } else if error.downcast_ref::().is_some() { + } else if error + .downcast_ref::() + .is_some_and(|error| matches!(error, RelayError::Write { .. })) + { openshell_core::proto::MiddlewareSessionEndReason::DownstreamDisconnect } else if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { openshell_core::proto::MiddlewareSessionEndReason::MiddlewareDenial @@ -992,7 +1323,7 @@ where response_correlation, ); if committed { - record_response_partial_delivery(response_correlation); + record_response_middleware_partial_delivery(response_correlation); } if let Some(stop) = middleware_stop { observe_response_diagnostics(response_correlation, &stop.failure.diagnostics); @@ -1012,19 +1343,6 @@ where ); } if committed { - if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { - record_committed_response_error( - response_correlation, - "denied", - "response_middleware_body", - ); - } else { - record_committed_response_error( - response_correlation, - "platform_error", - "response_middleware_body", - ); - } return Err(error); } debug!(error = %error, "HTTP response processing failed before commitment"); @@ -1040,12 +1358,18 @@ where ) .await?; } else { + let failure = classify_response_failure(&error); send_response_delivery_failure( client, request_method, middleware.policy_name, &middleware.target, - "response_middleware_body", + ( + failure.cause.unwrap_or(ResponseSelectionCause::Unknown), + failure.http_outcome, + failure.error_stage, + ), + true, response_correlation, ) .await?; @@ -1055,10 +1379,10 @@ where }; if let Some(guard) = middleware.generation_guard - && let Err(error) = guard.ensure_current() + && let Err(error) = ensure_response_policy_current(guard) { if committed { - record_response_partial_delivery(response_correlation); + record_response_middleware_partial_delivery(response_correlation); } session .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) @@ -1067,7 +1391,16 @@ where } let input_trailers = trailers.clone(); - let finish = match session.finish(trailers).await { + let finish_result = session.finish(trailers).await; + if let Some(guard) = middleware.generation_guard + && let Err(error) = ensure_response_policy_current(guard) + { + if committed { + record_response_middleware_partial_delivery(response_correlation); + } + return Err(error); + } + let finish = match finish_result { Ok(finish) => finish, Err(error) => { observe_response_diagnostics(response_correlation, &error.diagnostics); @@ -1076,7 +1409,7 @@ where error.admission_exhausted(), ); if committed { - record_response_partial_delivery(response_correlation); + record_response_middleware_partial_delivery(response_correlation); } complete_response_middleware( response_correlation, @@ -1092,22 +1425,11 @@ where response_correlation, ); if committed { - if error.denial.is_some() { - record_committed_response_error( - response_correlation, - "denied", - "response_middleware_finish", - ); - } else { - record_committed_response_error( - response_correlation, - "platform_error", - "response_middleware_finish", - ); - } - return Err(miette!( - "HTTP response middleware failed after commitment: {error}" - )); + return Err(ResponseMiddlewareStop::new( + error, + openshell_supervisor_middleware::HttpResponseDiagnostics::default(), + ) + .into()); } debug!(error = %error, "HTTP response middleware failed before commitment"); if let Some(denial) = error.denial.as_ref() { @@ -1127,7 +1449,12 @@ where request_method, middleware.policy_name, &middleware.target, - "response_middleware_finish", + ( + ResponseSelectionCause::MiddlewareFailed, + "platform_error", + "response_middleware_finish", + ), + true, response_correlation, ) .await?; @@ -1142,14 +1469,6 @@ where false, finish.diagnostics_truncated, ); - if let Some(guard) = middleware.generation_guard - && let Err(error) = guard.ensure_current() - { - if committed { - record_response_partial_delivery(response_correlation); - } - return Err(error); - } emit_http_response_middleware_invocations( middleware.policy_name, &middleware.target, @@ -1204,7 +1523,8 @@ where ); let head = final_response_head(&head, response_correlation); complete_response_middleware(response_correlation, MiddlewareOutcome::Allowed); - client.write_all(&head).await.into_diagnostic()?; + select_allowed_response(response_correlation, status_code); + write_correlated_response_bytes(client, &head, response_correlation, true).await?; if matches!(framing, ResponseFraming::Chunked) { for unit in &finish.body_units { write_downstream_response_chunk(client, unit).await?; @@ -1212,7 +1532,7 @@ where write_response_trailers(client, &finish.trailers).await?; } else { for unit in &finish.body_units { - client.write_all(unit).await.into_diagnostic()?; + client.write_all(unit).await.map_err(RelayError::write)?; } } } else { @@ -1221,15 +1541,14 @@ where if chunked_output { write_downstream_response_chunk(client, unit).await?; } else { - client.write_all(unit).await.into_diagnostic()?; + client.write_all(unit).await.map_err(RelayError::write)?; } } if chunked_output { write_response_trailers(client, &finish.trailers).await?; } } - client.flush().await.into_diagnostic()?; - record_allowed_response(response_correlation); + client.flush().await.map_err(RelayError::flush)?; Ok(Some( if server_wants_close || (committed && close_delimited_output) @@ -1256,6 +1575,7 @@ async fn relay_headers_only_response( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + generation_guard: Option<&PolicyGenerationGuard>, response_correlation: Option<&RequestCorrelation>, ) -> Result where @@ -1271,52 +1591,92 @@ where declared_trailers, ); let head = final_response_head(&head, response_correlation); - client.write_all(&head).await.into_diagnostic()?; + if let Some(guard) = generation_guard { + ensure_response_policy_current(guard)?; + } + select_allowed_response(response_correlation, status_code); + write_correlated_response_bytes(client, &head, response_correlation, true).await?; if is_bodiless_response(request_method, status_code) { - client.flush().await.into_diagnostic()?; - return Ok(if server_wants_close { + client.flush().await.map_err(RelayError::flush)?; + let outcome = if server_wants_close { RelayOutcome::Consumed } else { RelayOutcome::Reusable - }); + }; + return Ok(outcome); } match body_length { BodyLength::ContentLength(length) => { if !overflow.is_empty() { - client.write_all(overflow).await.into_diagnostic()?; + if let Some(guard) = generation_guard { + ensure_response_policy_current(guard)?; + } + client + .write_all(overflow) + .await + .map_err(RelayError::write)?; } - let remaining = length.saturating_sub(overflow.len() as u64); + let remaining = length - overflow.len() as u64; if remaining > 0 { - relay_fixed(upstream, client, remaining, None).await?; + relay_fixed(upstream, client, remaining, generation_guard).await?; } } - BodyLength::Chunked => relay_chunked(upstream, client, overflow, None).await?, + BodyLength::Chunked => relay_chunked(upstream, client, overflow, generation_guard) + .await + .map_err(type_response_protocol_error)?, BodyLength::None if server_wants_close || event_stream => { if !overflow.is_empty() { - client.write_all(overflow).await.into_diagnostic()?; + if let Some(guard) = generation_guard { + ensure_response_policy_current(guard)?; + } + client + .write_all(overflow) + .await + .map_err(RelayError::write)?; } - if event_stream { - relay_until_eof_without_idle_timeout(upstream, client).await?; + let idle_timed_out = if event_stream { + relay_until_eof_without_idle_timeout(upstream, client, generation_guard).await?; + false } else { - relay_until_eof(upstream, client).await?; + relay_until_eof(upstream, client, generation_guard).await? + == RelayUntilEofOutcome::IdleTimeout + }; + client.flush().await.map_err(RelayError::flush)?; + if idle_timed_out { + record_committed_response_error( + response_correlation, + "upstream_error", + "upstream_response_idle_timeout", + ); + if let Some(correlation) = response_correlation { + correlation.record_pending_response_delivery( + ResponseDeliveryOutcome::CancelledOrIncomplete, + ); + } } - client.flush().await.into_diagnostic()?; return Ok(RelayOutcome::Consumed); } BodyLength::None => { if !overflow.is_empty() { - client.write_all(overflow).await.into_diagnostic()?; + if let Some(guard) = generation_guard { + ensure_response_policy_current(guard)?; + } + client + .write_all(overflow) + .await + .map_err(RelayError::write)?; } } } - client.flush().await.into_diagnostic()?; - Ok(if server_wants_close { + client.flush().await.map_err(RelayError::flush)?; + let outcome = if server_wants_close { RelayOutcome::Consumed } else { RelayOutcome::Reusable - }) + }; + Ok(outcome) } fn http_response_decision_events( @@ -1423,10 +1783,117 @@ fn record_response_delivery_started(correlation: Option<&RequestCorrelation>) { } } -fn record_response_partial_delivery(correlation: Option<&RequestCorrelation>) { +fn record_response_middleware_delivery_started(correlation: Option<&RequestCorrelation>) { if let Some(correlation) = correlation { - correlation.record_response_partial_delivery(); + correlation.record_response_delivery_started(); + correlation.observe_middleware( + MiddlewarePhase::Response, + MiddlewareObservation { + delivery_started: true, + ..Default::default() + }, + ); + } +} + +fn record_response_middleware_partial_delivery(correlation: Option<&RequestCorrelation>) { + if let Some(correlation) = correlation + && correlation.record_response_partial_delivery() + { + correlation.observe_middleware( + MiddlewarePhase::Response, + MiddlewareObservation { + delivery_started: true, + partial_delivery: true, + ..Default::default() + }, + ); + } +} + +async fn write_correlated_interim_bytes( + client: &mut C, + bytes: &[u8], + correlation: Option<&RequestCorrelation>, +) -> Result<()> { + let mut written = 0usize; + while written < bytes.len() { + let count = client + .write(&bytes[written..]) + .await + .map_err(RelayError::interim_write)?; + if count == 0 { + return Err(RelayError::interim_write(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "failed to write interim HTTP response bytes", + )) + .into()); + } + if let Some(correlation) = correlation { + correlation.record_interim_response_delivery(count); + } + written += count; + } + Ok(()) +} + +async fn write_correlated_response_bytes( + client: &mut C, + bytes: &[u8], + correlation: Option<&RequestCorrelation>, + middleware_selected: bool, +) -> Result<()> { + let mut written = 0usize; + while written < bytes.len() { + let count = client + .write(&bytes[written..]) + .await + .map_err(RelayError::write)?; + if count == 0 { + return Err(RelayError::write(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "failed to write HTTP response bytes", + )) + .into()); + } + if written == 0 { + if middleware_selected { + record_response_middleware_delivery_started(correlation); + } else { + record_response_delivery_started(correlation); + } + } + written += count; + } + Ok(()) +} + +async fn write_correlated_response_commit( + client: &mut C, + bytes: &[u8], + correlation: Option<&RequestCorrelation>, + committed: &mut bool, +) -> Result<()> { + let mut written = 0usize; + while written < bytes.len() { + let count = client + .write(&bytes[written..]) + .await + .map_err(RelayError::write)?; + if count == 0 { + return Err(RelayError::write(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "failed to write HTTP response bytes", + )) + .into()); + } + if !*committed { + *committed = true; + record_response_middleware_delivery_started(correlation); + } + written += count; } + Ok(()) } fn complete_response_middleware( @@ -1458,6 +1925,27 @@ fn middleware_failure_outcome( } } +fn response_middleware_failure_denial( + failure: &openshell_supervisor_middleware::HttpResponseMiddlewareFailure, +) -> Option { + failure.denial.clone().or_else(|| { + failure + .diagnostics + .invocations + .iter() + .find(|invocation| { + invocation.outcome + == openshell_supervisor_middleware::HttpResponseInvocationOutcome::BlockDelivery + }) + .map( + |invocation| openshell_supervisor_middleware::MiddlewareDenial { + config_name: invocation.config_name.clone(), + reason_code: invocation.reason_code.clone(), + }, + ) + }) +} + fn emit_http_response_middleware_invocations( policy_name: &str, target: &HttpRequestTarget, @@ -1911,10 +2399,21 @@ fn response_status_line(header_bytes: &[u8]) -> Result { let line_end = header_bytes .windows(2) .position(|window| window == b"\r\n") - .ok_or_else(|| miette!("HTTP response status line is incomplete"))?; + .ok_or_else(|| { + RelayError::protocol( + "invalid_response_status_line", + "HTTP response status line is incomplete", + ) + })?; std::str::from_utf8(&header_bytes[..line_end]) .map(str::to_string) - .map_err(|_| miette!("HTTP response status line contains invalid UTF-8")) + .map_err(|_| { + RelayError::protocol( + "invalid_response_status_line", + "HTTP response status line contains invalid UTF-8", + ) + .into() + }) } #[derive(Clone, Copy)] @@ -2006,7 +2505,11 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { return Ok(Some(data)); } let mut data = vec![0u8; limit.max(1)]; - let count = self.upstream.read(&mut data).await.into_diagnostic()?; + let count = self + .upstream + .read(&mut data) + .await + .map_err(RelayError::read)?; if count == 0 { return Ok(None); } @@ -2028,7 +2531,7 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { while self.exact_buffer.len() < length { let remaining = length - self.exact_buffer.len(); let Some(data) = self.read_some(remaining).await? else { - return Err(miette!("HTTP response body ended unexpectedly")); + return Err(RelayError::Truncated("HTTP response body ended unexpectedly").into()); }; self.exact_buffer.extend_from_slice(&data); } @@ -2039,11 +2542,17 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { async fn read_line(&mut self) -> Result> { loop { let Some(byte) = self.read_some(1).await? else { - return Err(miette!("HTTP response ended before line terminator")); + return Err( + RelayError::Truncated("HTTP response ended before line terminator").into(), + ); }; self.line_buffer.push(byte[0]); if self.line_buffer.len() > MAX_HEADER_BYTES { - return Err(miette!("HTTP response line exceeds limit")); + return Err(RelayError::protocol( + "response_framing_line_too_large", + "HTTP response framing line exceeds limit", + ) + .into()); } if self.line_buffer.ends_with(b"\r\n") { self.line_buffer.truncate(self.line_buffer.len() - 2); @@ -2103,7 +2612,7 @@ where ) .await?; if let Some(guard) = generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } let partial = unit.len() < length; remaining -= unit.len() as u64; @@ -2133,7 +2642,9 @@ where let mut size_line = read_response_line_with_deadline(reader, session, client, &mut framing).await?; loop { - let chunk_size = parse_chunk_size_line(&size_line)?; + let chunk_size = parse_chunk_size_line(&size_line).map_err(|error| { + RelayError::protocol("invalid_chunk_size", error.to_string()) + })?; if chunk_size == 0 { flush_normalized_response_bytes(session, client, pending, &mut framing).await?; return read_response_trailers( @@ -2158,7 +2669,7 @@ where ) .await?; if let Some(guard) = generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } let partial = unit.len() < length; remaining -= unit.len(); @@ -2181,9 +2692,11 @@ where .await?; } } - let terminator = if let Ok(result) = - tokio::time::timeout(RESPONSE_UNIT_COALESCE_TIMEOUT, reader.read_exact_vec(2)) - .await + let terminator = if let Ok(result) = tokio::time::timeout( + RESPONSE_UNIT_COALESCE_TIMEOUT, + guarded_relay_read(generation_guard, reader.read_exact_vec(2)), + ) + .await { result? } else { @@ -2198,10 +2711,17 @@ where .await? }; if terminator != b"\r\n" { - return Err(miette!("HTTP response chunk is missing its terminator")); + return Err(RelayError::protocol( + "invalid_chunk_terminator", + "HTTP response chunk is missing its terminator", + ) + .into()); } - size_line = if let Ok(line) = - tokio::time::timeout(RESPONSE_UNIT_COALESCE_TIMEOUT, reader.read_line()).await + size_line = if let Ok(line) = tokio::time::timeout( + RESPONSE_UNIT_COALESCE_TIMEOUT, + guarded_relay_read(generation_guard, reader.read_line()), + ) + .await { line? } else { @@ -2234,15 +2754,18 @@ where (a, b) => a.or(b), }; let next = if let Some(deadline) = deadline { - if let Ok(result) = - tokio::time::timeout_at(deadline, reader.read_some(unit_limit)).await + if let Ok(result) = tokio::time::timeout_at( + deadline, + guarded_relay_read(generation_guard, reader.read_some(unit_limit)), + ) + .await { result? } else { if whole_deadline.is_some_and(|d| d <= tokio::time::Instant::now()) { expire_whole_body_deadline(session, client, &mut framing).await?; } else if pending.is_empty() { - return Ok(Vec::new()); + return Err(RelayError::IdleTimeout.into()); } else { flush_normalized_response_bytes( session, @@ -2255,14 +2778,14 @@ where continue; } } else { - reader.read_some(unit_limit).await? + guarded_relay_read(generation_guard, reader.read_some(unit_limit)).await? }; let Some(unit) = next else { flush_normalized_response_bytes(session, client, pending, &mut framing).await?; return Ok(Vec::new()); }; if let Some(guard) = generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } buffer_normalized_response_bytes( session, @@ -2324,23 +2847,6 @@ impl std::error::Error for ResponseMiddlewareStop {} impl miette::Diagnostic for ResponseMiddlewareStop {} -#[derive(Debug)] -struct ResponseDownstreamWrite(std::io::Error); - -impl fmt::Display for ResponseDownstreamWrite { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "HTTP response client write failed: {}", self.0) - } -} - -impl std::error::Error for ResponseDownstreamWrite { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.0) - } -} - -impl miette::Diagnostic for ResponseDownstreamWrite {} - async fn expire_whole_body_deadline( session: &mut openshell_supervisor_middleware::HttpResponseSession, client: &mut C, @@ -2356,7 +2862,7 @@ async fn expire_whole_body_deadline( ); } if let Some(guard) = framing.generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } if output.is_ok() { emit_http_response_diagnostics( @@ -2396,7 +2902,10 @@ where (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), }; - let read = reader.read_some(limit - payload.len()); + let read = guarded_relay_read( + framing.generation_guard, + reader.read_some(limit - payload.len()), + ); let result = if let Some(deadline) = deadline { if let Ok(result) = tokio::time::timeout_at(deadline, read).await { result @@ -2412,7 +2921,9 @@ where } else { read.await }; - let data = result?.ok_or_else(|| miette!("HTTP response body ended unexpectedly"))?; + let data = result?.ok_or(RelayError::Truncated( + "HTTP response body ended unexpectedly", + ))?; payload.extend_from_slice(&data); if payload.len() == limit { return Ok(payload); @@ -2435,9 +2946,15 @@ where { loop { let Some(deadline) = session.whole_body_deadline() else { - return reader.read_exact_vec(length).await; + return guarded_relay_read(framing.generation_guard, reader.read_exact_vec(length)) + .await; }; - match tokio::time::timeout_at(deadline, reader.read_exact_vec(length)).await { + match tokio::time::timeout_at( + deadline, + guarded_relay_read(framing.generation_guard, reader.read_exact_vec(length)), + ) + .await + { Ok(result) => return result, Err(_) => expire_whole_body_deadline(session, client, framing).await?, } @@ -2456,9 +2973,14 @@ where { loop { let Some(deadline) = session.whole_body_deadline() else { - return reader.read_line().await; + return guarded_relay_read(framing.generation_guard, reader.read_line()).await; }; - match tokio::time::timeout_at(deadline, reader.read_line()).await { + match tokio::time::timeout_at( + deadline, + guarded_relay_read(framing.generation_guard, reader.read_line()), + ) + .await + { Ok(result) => return result, Err(_) => expire_whole_body_deadline(session, client, framing).await?, } @@ -2510,7 +3032,7 @@ async fn process_response_unit( ); } if let Some(guard) = framing.generation_guard { - guard.ensure_current()?; + ensure_response_policy_current(guard)?; } if output.is_ok() { emit_http_response_diagnostics( @@ -2539,29 +3061,25 @@ async fn deliver_response_units( "whole-body response middleware released output before finalization" )); } - // `write_all` may return an error after a partial write. Treat the - // response as committed before the attempt so callers never append a - // canonical error response behind a partially delivered upstream head. - *framing.committed = true; - client - .write_all(framing.commit_head) - .await - .map_err(ResponseDownstreamWrite)?; - record_response_delivery_started(framing.response_correlation); - client.flush().await.map_err(ResponseDownstreamWrite)?; + select_allowed_response(framing.response_correlation, framing.status_code); + write_correlated_response_commit( + client, + framing.commit_head, + framing.response_correlation, + framing.committed, + ) + .await?; + client.flush().await.map_err(RelayError::flush)?; } if *framing.committed { for unit in output { if framing.chunked { write_downstream_response_chunk(client, &unit).await?; } else { - client - .write_all(&unit) - .await - .map_err(ResponseDownstreamWrite)?; + client.write_all(&unit).await.map_err(RelayError::write)?; } } - client.flush().await.map_err(ResponseDownstreamWrite)?; + client.flush().await.map_err(RelayError::flush)?; } Ok(()) } @@ -2576,15 +3094,9 @@ async fn write_downstream_response_chunk( client .write_all(format!("{:X}\r\n", payload.len()).as_bytes()) .await - .map_err(ResponseDownstreamWrite)?; - client - .write_all(payload) - .await - .map_err(ResponseDownstreamWrite)?; - client - .write_all(b"\r\n") - .await - .map_err(ResponseDownstreamWrite)?; + .map_err(RelayError::write)?; + client.write_all(payload).await.map_err(RelayError::write)?; + client.write_all(b"\r\n").await.map_err(RelayError::write)?; Ok(()) } @@ -2602,27 +3114,45 @@ where let mut trailers = Vec::new(); loop { let line = read_response_line_with_deadline(reader, session, client, framing).await?; - validate_chunk_trailer_line(&line)?; + validate_chunk_trailer_line(&line) + .map_err(|error| RelayError::protocol("invalid_chunk_trailer", error.to_string()))?; if line.is_empty() { return Ok(trailers); } - let line = std::str::from_utf8(&line) - .map_err(|_| miette!("HTTP response trailer contains invalid UTF-8"))?; - let (name, value) = line - .split_once(':') - .ok_or_else(|| miette!("Malformed HTTP response trailer"))?; - validate_http_field_name(name)?; - validate_http_field_value(value.trim())?; + let line = std::str::from_utf8(&line).map_err(|_| { + RelayError::protocol( + "invalid_chunk_trailer", + "HTTP response trailer contains invalid UTF-8", + ) + })?; + let (name, value) = line.split_once(':').ok_or_else(|| { + RelayError::protocol( + "invalid_chunk_trailer", + "HTTP response trailer is malformed", + ) + })?; + validate_http_field_name(name) + .map_err(|error| RelayError::protocol("invalid_chunk_trailer", error.to_string()))?; + validate_http_field_value(value.trim()) + .map_err(|error| RelayError::protocol("invalid_chunk_trailer", error.to_string()))?; let name = name.to_ascii_lowercase(); if is_protected_response_field(&name) || connection_nominated_headers.contains(&name) { - return Err(miette!("HTTP response trailer uses a protected field name")); + return Err(RelayError::protocol( + "invalid_chunk_trailer", + "HTTP response trailer uses a protected field name", + ) + .into()); } trailers.push(HttpHeader { name, value: value.trim().to_string(), }); if trailers.len() > openshell_supervisor_middleware::MAX_MIDDLEWARE_HEADERS { - return Err(miette!("HTTP response trailer count exceeds limit")); + return Err(RelayError::protocol( + "invalid_chunk_trailer", + "HTTP response trailer count exceeds limit", + ) + .into()); } } } @@ -2631,14 +3161,17 @@ async fn write_response_trailers( client: &mut C, trailers: &[HttpHeader], ) -> Result<()> { - client.write_all(b"0\r\n").await.into_diagnostic()?; + client + .write_all(b"0\r\n") + .await + .map_err(RelayError::write)?; for trailer in trailers { client .write_all(format!("{}: {}\r\n", trailer.name, trailer.value).as_bytes()) .await - .into_diagnostic()?; + .map_err(RelayError::write)?; } - client.write_all(b"\r\n").await.into_diagnostic()?; + client.write_all(b"\r\n").await.map_err(RelayError::write)?; Ok(()) } @@ -2651,7 +3184,6 @@ async fn send_response_middleware_denial( stage: &'static str, correlation: Option<&RequestCorrelation>, ) -> Result<()> { - record_denied_response(correlation, stage); let mut body = serde_json::Map::new(); body.insert("error".into(), serde_json::json!("middleware_denied")); body.insert( @@ -2676,12 +3208,13 @@ async fn send_response_middleware_denial( "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); + record_denied_response(correlation, stage); let head = final_response_head(head.as_bytes(), correlation); - client.write_all(&head).await.into_diagnostic()?; + write_correlated_response_bytes(client, &head, correlation, true).await?; if !request_method.eq_ignore_ascii_case("HEAD") { - client.write_all(&body).await.into_diagnostic()?; + client.write_all(&body).await.map_err(RelayError::write)?; } - client.flush().await.into_diagnostic()?; + client.flush().await.map_err(RelayError::flush)?; Ok(()) } @@ -2690,10 +3223,11 @@ async fn send_response_delivery_failure( request_method: &str, policy_name: &str, target: &HttpRequestTarget, - stage: &'static str, + failure: (ResponseSelectionCause, &'static str, &'static str), + middleware_selected: bool, correlation: Option<&RequestCorrelation>, ) -> Result<()> { - record_platform_response_error(correlation, stage); + let (cause, http_outcome, stage) = failure; let body = serde_json::to_vec(&serde_json::json!({ "error": "response_delivery_failed", "detail": "The upstream request may have completed, but OpenShell could not deliver its response. Retrying may repeat upstream side effects.", @@ -2709,12 +3243,13 @@ async fn send_response_delivery_failure( "HTTP/1.1 502 Bad Gateway\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); + select_failure_response(correlation, cause, http_outcome, stage); let head = final_response_head(head.as_bytes(), correlation); - client.write_all(&head).await.into_diagnostic()?; + write_correlated_response_bytes(client, &head, correlation, middleware_selected).await?; if !request_method.eq_ignore_ascii_case("HEAD") { - client.write_all(&body).await.into_diagnostic()?; + client.write_all(&body).await.map_err(RelayError::write)?; } - client.flush().await.into_diagnostic()?; + client.flush().await.map_err(RelayError::flush)?; Ok(()) } @@ -2788,1583 +3323,4 @@ pub(super) fn response_is_event_stream(headers: &str) -> bool { } #[cfg(test)] -mod tests { - use openshell_supervisor_middleware::{ - HttpResponseInvocation, HttpResponseInvocationOutcome as Outcome, NamespacedFinding, - }; - - fn invocation(outcome: Outcome, failed: bool) -> HttpResponseInvocation { - HttpResponseInvocation { - config_name: "guard".to_string(), - implementation: "test/guard".to_string(), - outcome, - sequence: None, - input_size: 0, - output_size: None, - failed, - transformed: false, - stage_disabled: false, - reason_code: None, - failure_category: None, - } - } - - fn finding() -> NamespacedFinding { - NamespacedFinding { - middleware: "guard".to_string(), - finding: openshell_core::proto::Finding { - r#type: "test.finding".to_string(), - label: "Test finding".to_string(), - count: 1, - severity: "high".to_string(), - ..Default::default() - }, - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn response_terminal_failure_without_diagnostics_has_no_deny_trace() { - use crate::telemetry::tests::{assert_no_deny, exported}; - - // Platform refusals can lack invocation diagnostics, but must retain - // their terminal failure record. Without an HTTP record, no deny span. - for committed in [false, true] { - let (events, received) = exported("warn", || { - super::emit_http_response_middleware_failure( - "response-policy", - &HttpRequestTarget::default(), - 200, - committed, - None, - ); - }) - .await; - - assert_eq!(events.len(), 1, "only the terminal failure is emitted"); - assert_eq!( - events[0]["finding_info"]["uid"], - "openshell.middleware.http_response_failure" - ); - assert_no_deny(&events, &received); - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn response_diagnostic_findings_share_a_deny_span_only_when_delivery_is_denied() { - use crate::telemetry::tests::{assert_deny_resolves, assert_no_deny, exported}; - - for denied in [false, true] { - let (events, received) = exported("warn", || { - let outcome = if denied { - Outcome::BlockDelivery - } else { - Outcome::PassThrough - }; - let diagnostics = openshell_supervisor_middleware::HttpResponseDiagnostics { - invocations: vec![invocation(outcome, false)], - findings: vec![finding()], - ..Default::default() - }; - super::emit_http_response_diagnostics( - "response-policy", - &HttpRequestTarget::default(), - 200, - &diagnostics, - denied, - None, - None, - ); - }) - .await; - // A blocking invocation adds its own block finding. - assert_eq!(events.len(), if denied { 3 } else { 2 }); - if denied { - let blocked = events - .iter() - .find(|event| { - event["finding_info"]["uid"] == "openshell.middleware.http_response_blocked" - }) - .expect("blocked delivery emits its refusal finding"); - assert_eq!(blocked["action_id"], 2); - assert_eq!(blocked["action"], "Denied"); - assert_eq!(received.spans.len(), 1); - assert_deny_resolves(&events, &received, true); - } else { - assert_no_deny(&events, &received); - } - } - } - - #[test] - fn fail_closed_invocations_and_enforced_failures_record_the_denial() { - let target = HttpRequestTarget::default(); - for (outcome, failed, action_id) in [ - (Outcome::BlockDelivery, false, 2), - (Outcome::FailClosed, true, 2), - (Outcome::FailOpen, true, 99), - (Outcome::PassThrough, false, 1), - ] { - let events = super::http_response_middleware_invocation_events( - "p", - &target, - 200, - &[invocation(outcome, failed)], - ); - assert_eq!( - events[0].to_json().unwrap()["action_id"], - action_id, - "{outcome:?}" - ); - } - for denied in [false, true] { - let failure = - super::http_response_middleware_failure_event("p", &target, 200, false, denied) - .to_json() - .unwrap(); - assert_eq!(failure["action_id"] == 2, denied); - } - } - - use super::{ - DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, HttpResponseMiddlewareRelay, - RelayResponseOptions, ResponseFraming, http_response_diagnostic_truncation_event, - parse_connection_keep_alive, parse_response_head_for_middleware, relay_response, - response_is_http_10, response_middleware_observation, serialize_response_head, - validate_http_response_header_block, - }; - use crate::l7::correlation::MiddlewareObservation; - use crate::l7::provider::{BodyLength, L7Request, RelayOutcome}; - use crate::l7::relay::L7EvalContext; - use openshell_core::proto::{ - Decision, ExistingHeaderAction, HeaderMutation, HttpHeader, HttpRequestResult, - HttpRequestTarget, HttpResponseBlockDelivery, HttpResponseBodyMode, HttpResponseBodyResult, - HttpResponseBodySkipRemaining, HttpResponseBodyTransform, HttpResponseEvent, - HttpResponseEventResult, HttpResponsePreflightInspect, HttpResponsePreflightResult, - HttpResponseTrailersResult, MiddlewareBinding, MiddlewareManifest, RequestContext, - SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, WriteHeader, header_mutation, - http_response_body_result, http_response_body_skip_remaining, http_response_body_transform, - http_response_body_unit, http_response_event, http_response_event_result, - http_response_preflight_result, - }; - use std::collections::HashMap; - use std::sync::Arc; - use tokio::sync::mpsc; - use tokio_stream::wrappers::ReceiverStream; - - struct StreamingResponseMiddleware; - - struct BlockingResponseMiddleware; - - struct RequestOnlyMiddleware; - - #[tonic::async_trait] - impl openshell_supervisor_middleware::InProcessMiddleware for RequestOnlyMiddleware { - async fn describe(&self) -> MiddlewareManifest { - let mut manifest = openshell_supervisor_middleware::InProcessMiddleware::describe( - &BlockingResponseMiddleware, - ) - .await; - manifest.name = "test/request-only".into(); - manifest.bindings[0].operation = SupervisorMiddlewareOperation::HttpRequest as i32; - manifest.bindings[0].phase = SupervisorMiddlewarePhase::PreCredentials as i32; - manifest - } - - async fn validate_config(&self, _: &str, _: &prost_types::Struct) -> miette::Result<()> { - Ok(()) - } - - async fn evaluate_http_request( - &self, - _: openshell_supervisor_middleware::HttpRequestView<'_>, - ) -> miette::Result { - panic!("request hook must not run on a response") - } - } - - #[tokio::test] - async fn an_absent_response_binding_does_not_create_a_cancelled_phase() { - use tracing_subscriber::layer::SubscriberExt as _; - let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() - .with_simple_exporter(exporter.clone()) - .build(); - let subscriber = tracing_subscriber::registry() - .with(openshell_otel::layer(&provider, "no-response-hook")); - let _subscriber = tracing::subscriber::set_default(subscriber); - let correlation = test_correlation("request-only"); - let runner = - openshell_supervisor_middleware::ChainRunner::new(Arc::new(RequestOnlyMiddleware)); - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "request".into(), - implementation: "test/request-only".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; - let mut output = Vec::new(); - relay_response( - "GET", - &mut upstream, - &mut output, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - Some(test_response_middleware(&runner, &chain, "request-only")), - ) - .await - .unwrap(); - drop(correlation); - assert!(output.ends_with(b"\r\n\r\nbody")); - assert!( - !exporter - .get_finished_spans() - .unwrap() - .iter() - .any(|span| span.name == "openshell.middleware.response") - ); - } - - fn test_response_middleware<'a>( - runner: &'a openshell_supervisor_middleware::ChainRunner, - chain: &'a [openshell_supervisor_middleware::ChainEntry], - request_id: &str, - ) -> HttpResponseMiddlewareRelay<'a> { - HttpResponseMiddlewareRelay { - chain, - runner, - request_context: RequestContext { - request_id: request_id.into(), - sandbox_id: "sandbox-1".into(), - ..Default::default() - }, - target: HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "GET".into(), - path: "/data".into(), - query: String::new(), - }, - policy_name: "test-policy", - generation_guard: None, - whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, - } - } - - enum BodyTestAction { - Block, - ReloadAndSkipTransform(Arc), - } - - struct BodyResponseMiddleware { - action: BodyTestAction, - } - - #[tonic::async_trait] - impl openshell_supervisor_middleware::InProcessMiddleware for BlockingResponseMiddleware { - async fn describe(&self) -> MiddlewareManifest { - MiddlewareManifest { - name: "test/blocking-response".into(), - service_version: "test".into(), - bindings: vec![MiddlewareBinding { - operation: SupervisorMiddlewareOperation::HttpResponse as i32, - phase: SupervisorMiddlewarePhase::PreReturn as i32, - max_payload_bytes: 4096, - request_timeout: None, - }], - expected_audience: String::new(), - extension: Some(openshell_core::extension_protocol::extension_metadata( - openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, - "openshell/test-blocking-response", - "test", - [], - )), - } - } - - async fn validate_config( - &self, - _middleware_name: &str, - _config: &prost_types::Struct, - ) -> miette::Result<()> { - Ok(()) - } - - async fn evaluate_http_request( - &self, - _request: openshell_supervisor_middleware::HttpRequestView<'_>, - ) -> miette::Result { - Ok(HttpRequestResult { - decision: Decision::Allow as i32, - ..Default::default() - }) - } - - async fn open_http_response_pre_return( - &self, - mut requests: mpsc::Receiver, - ) -> Result - { - let first = requests - .recv() - .await - .ok_or_else(|| tonic::Status::invalid_argument("missing response preflight"))?; - if !matches!(first.event, Some(http_response_event::Event::Preflight(_))) { - return Err(tonic::Status::invalid_argument( - "expected response preflight", - )); - } - let result = HttpResponseEventResult { - result: Some(http_response_event_result::Result::PreflightResult( - HttpResponsePreflightResult { - action: Some(http_response_preflight_result::Action::BlockDelivery( - HttpResponseBlockDelivery {}, - )), - reason_code: "test_block".into(), - ..Default::default() - }, - )), - }; - Ok(Box::pin(futures::stream::iter([Ok(result)]))) - } - } - - #[tonic::async_trait] - impl openshell_supervisor_middleware::InProcessMiddleware for StreamingResponseMiddleware { - async fn describe(&self) -> MiddlewareManifest { - MiddlewareManifest { - name: "test/streaming-response".into(), - service_version: "test".into(), - bindings: vec![MiddlewareBinding { - operation: SupervisorMiddlewareOperation::HttpResponse as i32, - phase: SupervisorMiddlewarePhase::PreReturn as i32, - max_payload_bytes: 4096, - request_timeout: None, - }], - expected_audience: String::new(), - extension: Some(openshell_core::extension_protocol::extension_metadata( - openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, - "openshell/test-streaming-response", - "test", - [], - )), - } - } - - async fn validate_config( - &self, - _middleware_name: &str, - _config: &prost_types::Struct, - ) -> miette::Result<()> { - Ok(()) - } - - async fn evaluate_http_request( - &self, - _request: openshell_supervisor_middleware::HttpRequestView<'_>, - ) -> miette::Result { - Ok(HttpRequestResult { - decision: Decision::Allow as i32, - ..Default::default() - }) - } - - async fn open_http_response_pre_return( - &self, - mut requests: mpsc::Receiver, - ) -> Result - { - let (sender, receiver) = mpsc::channel(4); - tokio::spawn(async move { - while let Some(event) = requests.recv().await { - let Some(event) = event.event else { - break; - }; - let result = match event { - http_response_event::Event::Preflight(_) => HttpResponseEventResult { - result: Some(http_response_event_result::Result::PreflightResult( - HttpResponsePreflightResult { - action: Some(http_response_preflight_result::Action::Inspect( - HttpResponsePreflightInspect { - body_mode: HttpResponseBodyMode::StreamBytes as i32, - header_mutations: vec![HeaderMutation { - operation: Some(header_mutation::Operation::Write( - WriteHeader { - name: "x-inspected".into(), - value: "true".into(), - on_existing: ExistingHeaderAction::Overwrite - as i32, - }, - )), - }], - }, - )), - ..Default::default() - }, - )), - }, - http_response_event::Event::Body(body) => { - let Some(http_response_body_unit::Payload::Data(data)) = body.payload - else { - break; - }; - HttpResponseEventResult { - result: Some(http_response_event_result::Result::BodyResult( - HttpResponseBodyResult { - sequence: body.sequence, - action: Some(http_response_body_result::Action::Transform( - HttpResponseBodyTransform { - replacement: Some( - http_response_body_transform::Replacement::Data( - data, - ), - ), - }, - )), - ..Default::default() - }, - )), - } - } - http_response_event::Event::Trailers(_) => HttpResponseEventResult { - result: Some(http_response_event_result::Result::TrailersResult( - HttpResponseTrailersResult::default(), - )), - }, - http_response_event::Event::SessionEnd(_) => break, - }; - if sender.send(Ok(result)).await.is_err() { - break; - } - } - }); - Ok(Box::pin(ReceiverStream::new(receiver))) - } - } - - #[tonic::async_trait] - impl openshell_supervisor_middleware::InProcessMiddleware for BodyResponseMiddleware { - async fn describe(&self) -> MiddlewareManifest { - MiddlewareManifest { - name: "test/body-response".into(), - service_version: "test".into(), - bindings: vec![MiddlewareBinding { - operation: SupervisorMiddlewareOperation::HttpResponse as i32, - phase: SupervisorMiddlewarePhase::PreReturn as i32, - max_payload_bytes: 4096, - request_timeout: None, - }], - expected_audience: String::new(), - extension: Some(openshell_core::extension_protocol::extension_metadata( - openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, - "openshell/test-body-response", - "test", - [], - )), - } - } - - async fn validate_config( - &self, - _middleware_name: &str, - _config: &prost_types::Struct, - ) -> miette::Result<()> { - Ok(()) - } - - async fn evaluate_http_request( - &self, - _request: openshell_supervisor_middleware::HttpRequestView<'_>, - ) -> miette::Result { - Ok(HttpRequestResult { - decision: Decision::Allow as i32, - ..Default::default() - }) - } - - async fn open_http_response_pre_return( - &self, - mut requests: mpsc::Receiver, - ) -> Result - { - let (sender, receiver) = mpsc::channel(4); - let action = match &self.action { - BodyTestAction::Block => BodyTestAction::Block, - BodyTestAction::ReloadAndSkipTransform(engine) => { - BodyTestAction::ReloadAndSkipTransform(Arc::clone(engine)) - } - }; - tokio::spawn(async move { - while let Some(event) = requests.recv().await { - let Some(event) = event.event else { - break; - }; - let result = match event { - http_response_event::Event::Preflight(_) => HttpResponseEventResult { - result: Some(http_response_event_result::Result::PreflightResult( - HttpResponsePreflightResult { - action: Some(http_response_preflight_result::Action::Inspect( - HttpResponsePreflightInspect { - body_mode: HttpResponseBodyMode::StreamBytes as i32, - header_mutations: Vec::new(), - }, - )), - ..Default::default() - }, - )), - }, - http_response_event::Event::Body(body) => { - let Some(http_response_body_unit::Payload::Data(data)) = body.payload - else { - break; - }; - let action = match &action { - BodyTestAction::Block => { - http_response_body_result::Action::BlockDelivery( - HttpResponseBlockDelivery {}, - ) - } - BodyTestAction::ReloadAndSkipTransform(engine) => { - engine - .reload( - include_str!("../../../data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .expect("reload test policy"); - http_response_body_result::Action::SkipRemaining( - HttpResponseBodySkipRemaining { - current: Some( - http_response_body_skip_remaining::Current::Transform( - HttpResponseBodyTransform { - replacement: Some( - http_response_body_transform::Replacement::Data(data), - ), - }, - ), - ), - }, - ) - } - }; - HttpResponseEventResult { - result: Some(http_response_event_result::Result::BodyResult( - HttpResponseBodyResult { - sequence: body.sequence, - action: Some(action), - reason_code: "test_body_action".into(), - ..Default::default() - }, - )), - } - } - http_response_event::Event::Trailers(_) => HttpResponseEventResult { - result: Some(http_response_event_result::Result::TrailersResult( - HttpResponseTrailersResult::default(), - )), - }, - http_response_event::Event::SessionEnd(_) => break, - }; - if sender.send(Ok(result)).await.is_err() { - break; - } - } - }); - Ok(Box::pin(ReceiverStream::new(receiver))) - } - } - - fn test_correlation_for_method( - request_id: &str, - method: &str, - ) -> crate::l7::correlation::RequestCorrelation { - let request = L7Request { - action: method.into(), - target: "/data".into(), - query_params: HashMap::new(), - raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), - body_length: BodyLength::None, - }; - crate::l7::correlation::RequestCorrelation::with_request_id( - &request, - &L7EvalContext { - host: "example.test".into(), - port: 443, - ..Default::default() - }, - request_id.into(), - ) - } - - fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { - test_correlation_for_method(request_id, "GET") - } - - fn request_id_header_count(response: &[u8]) -> usize { - String::from_utf8_lossy(response) - .lines() - .filter(|line| { - line.split_once(':').is_some_and(|(name, _)| { - name.eq_ignore_ascii_case( - openshell_core::token_grant_failure::REQUEST_ID_HEADER, - ) - }) - }) - .count() - } - - #[tokio::test] - async fn allowed_final_response_carries_platform_receipt() { - let correlation = test_correlation("receipt-1"); - let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; - let mut client = Vec::new(); - - let outcome = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect("relay response"); - - assert!(matches!(outcome, RelayOutcome::Reusable)); - assert_eq!(request_id_header_count(&client), 1); - assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); - } - - #[tokio::test] - async fn branch08_response_middleware_denial_records_that_upstream_was_reached() { - use tracing_subscriber::layer::SubscriberExt as _; - - let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() - .with_simple_exporter(exporter.clone()) - .build(); - let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); - let subscriber = tracing_subscriber::registry() - .with(openshell_otel::layer( - &provider, - "response-correlation-test", - )) - .with(ocsf.clone()); - let _subscriber = tracing::subscriber::set_default(subscriber); - tracing::callsite::rebuild_interest_cache(); - let correlation = test_correlation_for_method("response-denied", "POST"); - let runner = - openshell_supervisor_middleware::ChainRunner::new(Arc::new(BlockingResponseMiddleware)); - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "response".into(), - implementation: "test/blocking-response".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let middleware = HttpResponseMiddlewareRelay { - chain: &chain, - runner: &runner, - request_context: RequestContext { - request_id: "response-denied".into(), - sandbox_id: "sandbox-1".into(), - ..Default::default() - }, - target: HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "POST".into(), - path: "/mutate".into(), - query: String::new(), - }, - policy_name: "test-policy", - generation_guard: None, - whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, - }; - let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; - let mut client = Vec::new(); - - let outcome = relay_response( - "POST", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - Some(middleware), - ) - .await - .expect("middleware denial response"); - - assert!(matches!(outcome, RelayOutcome::Consumed)); - assert!(client.starts_with(b"HTTP/1.1 403 Forbidden\r\n")); - assert!( - String::from_utf8_lossy(&client) - .contains("x-openshell-request-id: response-denied\r\n") - ); - drop(correlation); - - let span = exporter - .get_finished_spans() - .expect("finished spans") - .into_iter() - .find(|span| span.name == "openshell.http.request") - .expect("HTTP enforcement span"); - assert!(matches!( - span.status, - opentelemetry::trace::Status::Error { .. } - )); - let attributes = span - .attributes - .into_iter() - .map(|attribute| { - ( - attribute.key.as_str().to_string(), - attribute.value.to_string(), - ) - }) - .collect::>(); - assert_eq!(attributes["openshell.target.reached"], "true"); - assert_eq!(attributes["openshell.http.outcome"], "denied"); - assert_eq!(attributes["http.response.status_code"], "403"); - assert!(!attributes.contains_key("openshell.error.stage")); - } - - #[tokio::test] - async fn streaming_response_middleware_aggregates_header_and_body_mutations() { - use tracing_subscriber::layer::SubscriberExt as _; - - let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() - .with_simple_exporter(exporter.clone()) - .build(); - let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( - &provider, - "response-correlation-test", - )); - let correlation = tracing::subscriber::with_default(subscriber, || { - test_correlation_for_method("response-streaming", "GET") - }); - let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( - StreamingResponseMiddleware, - )); - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "response".into(), - implementation: "test/streaming-response".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let middleware = HttpResponseMiddlewareRelay { - chain: &chain, - runner: &runner, - request_context: RequestContext { - request_id: "response-streaming".into(), - sandbox_id: "sandbox-1".into(), - ..Default::default() - }, - target: HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "GET".into(), - path: "/data".into(), - query: String::new(), - }, - policy_name: "test-policy", - generation_guard: None, - whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, - }; - let body = vec![b'a'; 9_000]; - let mut response = - format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()).into_bytes(); - response.extend_from_slice(&body); - let mut upstream = response.as_slice(); - let mut client = Vec::new(); - - let outcome = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - Some(middleware), - ) - .await - .expect("streaming middleware response"); - - assert!(matches!(outcome, RelayOutcome::Reusable)); - assert!(String::from_utf8_lossy(&client).contains("x-inspected: true\r\n")); - assert!( - String::from_utf8_lossy(&client) - .contains("x-openshell-request-id: response-streaming\r\n") - ); - drop(correlation); - - let span = exporter - .get_finished_spans() - .expect("finished spans") - .into_iter() - .find(|span| span.name == "openshell.middleware.response") - .expect("HTTP enforcement span"); - let attributes = span - .attributes - .into_iter() - .map(|attribute| { - ( - attribute.key.as_str().to_string(), - attribute.value.to_string(), - ) - }) - .collect::>(); - assert_eq!(attributes["openshell.middleware.outcome"], "allowed"); - assert_eq!(attributes["openshell.middleware.mutated"], "true"); - assert_eq!(attributes["openshell.middleware.fail_open"], "false"); - assert!(!attributes.contains_key("openshell.request.id")); - } - - #[tokio::test] - async fn streaming_body_denial_records_partial_delivery() { - use tracing_subscriber::layer::SubscriberExt as _; - - let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() - .with_simple_exporter(exporter.clone()) - .build(); - let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); - let subscriber = tracing_subscriber::registry() - .with(openshell_otel::layer( - &provider, - "response-correlation-test", - )) - .with(ocsf.clone()); - let _subscriber = tracing::subscriber::set_default(subscriber); - tracing::callsite::rebuild_interest_cache(); - let correlation = test_correlation_for_method("response-partial-denial", "GET"); - let runner = - openshell_supervisor_middleware::ChainRunner::new(Arc::new(BodyResponseMiddleware { - action: BodyTestAction::Block, - })); - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "response".into(), - implementation: "test/body-response".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let middleware = HttpResponseMiddlewareRelay { - chain: &chain, - runner: &runner, - request_context: RequestContext { - request_id: "response-partial-denial".into(), - sandbox_id: "sandbox-1".into(), - ..Default::default() - }, - target: HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "GET".into(), - path: "/data".into(), - query: String::new(), - }, - policy_name: "test-policy", - generation_guard: None, - whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, - }; - let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; - let mut client = Vec::new(); - - let error = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - Some(middleware), - ) - .await - .expect_err("streaming body denial follows response commitment"); - assert!(error.to_string().contains("stopped delivery"), "{error}"); - assert!(client.starts_with(b"HTTP/1.1 200 OK\r\n")); - drop(correlation); - provider.force_flush().expect("flush spans"); - - let span = exporter - .get_finished_spans() - .expect("finished spans") - .into_iter() - .find(|span| span.name == "openshell.middleware.response") - .expect("HTTP enforcement span"); - let attributes = span - .attributes - .into_iter() - .map(|attribute| { - ( - attribute.key.as_str().to_string(), - attribute.value.to_string(), - ) - }) - .collect::>(); - assert_eq!(attributes["openshell.middleware.outcome"], "denied"); - assert_eq!(attributes["openshell.response.delivery_started"], "true"); - assert_eq!(attributes["openshell.response.partial_delivery"], "true"); - assert!( - !ocsf - .json_events() - .iter() - .any(|event| event["firewall_rule"]["type"] == "middleware-correlation"), - "routine middleware aggregates are not separate audit records" - ); - } - - #[tokio::test] - async fn stale_generation_preserves_skip_transform_observation() { - use tracing_subscriber::layer::SubscriberExt as _; - - let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() - .with_simple_exporter(exporter.clone()) - .build(); - let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( - &provider, - "response-correlation-test", - )); - let correlation = tracing::subscriber::with_default(subscriber, || { - test_correlation_for_method("response-stale-transform", "GET") - }); - let engine = Arc::new( - crate::opa::OpaEngine::from_strings( - include_str!("../../../data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .expect("test policy"), - ); - let generation_guard = engine - .generation_guard(engine.current_generation()) - .expect("generation guard"); - let runner = - openshell_supervisor_middleware::ChainRunner::new(Arc::new(BodyResponseMiddleware { - action: BodyTestAction::ReloadAndSkipTransform(Arc::clone(&engine)), - })); - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "response".into(), - implementation: "test/body-response".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let middleware = HttpResponseMiddlewareRelay { - chain: &chain, - runner: &runner, - request_context: RequestContext { - request_id: "response-stale-transform".into(), - sandbox_id: "sandbox-1".into(), - ..Default::default() - }, - target: HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "GET".into(), - path: "/data".into(), - query: String::new(), - }, - policy_name: "test-policy", - generation_guard: Some(&generation_guard), - whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, - }; - let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; - let mut client = Vec::new(); - - let error = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - Some(middleware), - ) - .await - .expect_err("policy reload must stop the stale response"); - assert!( - error.to_string().contains("policy generation is stale"), - "{error}" - ); - drop(correlation); - provider.force_flush().expect("flush spans"); - - let span = exporter - .get_finished_spans() - .expect("finished spans") - .into_iter() - .find(|span| span.name == "openshell.middleware.response") - .expect("HTTP enforcement span"); - let attributes = span - .attributes - .into_iter() - .map(|attribute| { - ( - attribute.key.as_str().to_string(), - attribute.value.to_string(), - ) - }) - .collect::>(); - assert_eq!(attributes["openshell.middleware.outcome"], "incomplete"); - assert_eq!(attributes["openshell.middleware.mutated"], "true"); - assert_eq!(attributes["openshell.response.delivery_started"], "true"); - assert_eq!(attributes["openshell.response.partial_delivery"], "true"); - } - - #[test] - fn response_middleware_observation_maps_bounded_outcomes() { - let invocation = |outcome| HttpResponseInvocation { - config_name: "not-exported".into(), - implementation: "not-exported".into(), - outcome, - sequence: None, - input_size: 17, - output_size: None, - transformed: outcome - == openshell_supervisor_middleware::HttpResponseInvocationOutcome::Transform, - failed: false, - stage_disabled: false, - reason_code: None, - failure_category: None, - }; - let mut invocations = vec![ - invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::Transform), - invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::FailOpen), - invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::FailClosed), - ]; - let mut skip_transform = invocation( - openshell_supervisor_middleware::HttpResponseInvocationOutcome::SkipRemaining, - ); - skip_transform.transformed = true; - invocations.push(skip_transform); - let observation = response_middleware_observation(&invocations, false, true); - - assert_eq!( - observation, - MiddlewareObservation { - mutated: true, - fail_open: true, - fail_closed: true, - admission_exhausted: true, - ..Default::default() - } - ); - } - - #[test] - fn response_diagnostic_truncation_event_is_bounded() { - let event = http_response_diagnostic_truncation_event( - "test-policy", - &HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "GET".into(), - path: "/private".into(), - query: "secret=value".into(), - }, - ) - .to_json() - .expect("serialize truncation event"); - - assert_eq!(event["severity"], "Low"); - assert_eq!( - event["finding_info"]["uid"], - "openshell.middleware.http_response_diagnostics_truncated" - ); - assert_eq!(event["unmapped"]["phase"], "pre_return"); - let encoded = serde_json::to_string(&event).expect("encode truncation event"); - assert!(!encoded.contains("/private")); - assert!(!encoded.contains("secret=value")); - } - - #[tokio::test] - async fn upstream_receipt_is_replaced_by_platform_receipt() { - let correlation = test_correlation("trusted"); - let mut upstream = - &b"HTTP/1.1 200 OK\r\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; - let mut client = Vec::new(); - - relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect("relay response"); - - assert!(!String::from_utf8_lossy(&client).contains("forged")); - assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: trusted\r\n")); - assert_eq!(request_id_header_count(&client), 1); - } - - #[tokio::test] - async fn ordinary_response_removes_receipt_connection_nomination() { - let correlation = test_correlation("trusted"); - let mut upstream = &b"HTTP/1.1 200 OK\r\n\ - Connection: X-OpenShell-Request-ID\r\n\ - cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id\r\n\ - X-OpenShell-Request-ID: forged-head\r\n\ - Trailer: x-openshell-request-id, X-Checksum\r\n\ - Transfer-Encoding: chunked\r\n\r\n\ - 4\r\nbody\r\n\ - 0\r\n\ - x-OPENshell-request-ID: forged-trailer\r\n\ - X-Checksum: ok\r\n\r\n"[..]; - let mut client = Vec::new(); - - let outcome = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect("relay response"); - - assert!(matches!(outcome, RelayOutcome::Reusable)); - let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); - let head = delivered - .split_once("\r\n\r\n") - .expect("response head terminator") - .0; - let connection_headers: Vec<_> = head - .lines() - .filter(|line| { - line.split_once(':') - .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) - }) - .collect(); - assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive"]); - assert!(!delivered.contains("forged"), "{delivered}"); - assert!(head.contains("Trailer: X-Checksum"), "{head}"); - assert!(delivered.contains("0\r\nX-Checksum: ok\r\n\r\n")); - assert!(head.contains("x-openshell-request-id: trusted"), "{head}"); - assert_eq!(request_id_header_count(delivered.as_bytes()), 1); - } - - #[tokio::test] - async fn ordinary_response_rejects_folded_receipt_connection_nomination() { - let correlation = test_correlation("trusted"); - let mut upstream = &b"HTTP/1.1 200 OK\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nContent-Length: 0\r\n\r\n"[..]; - let mut client = Vec::new(); - - let error = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect_err("folded response fields must fail closed"); - - assert!( - error - .to_string() - .contains("response header continuation lines are not supported"), - "{error}" - ); - assert!(client.is_empty()); - } - - #[tokio::test] - async fn ordinary_response_rejects_bare_lf_receipt_smuggling() { - let correlation = test_correlation("trusted"); - let mut upstream = &b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; - let mut client = Vec::new(); - - let error = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect_err("bare LF response fields must fail closed"); - - assert!(error.to_string().contains("bare LF"), "{error}"); - assert!(client.is_empty()); - } - - #[test] - fn response_head_validation_rejects_malformed_syntax() { - let cases: &[(&str, &[u8])] = &[ - ("incomplete block", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n"), - ("bare LF", b"HTTP/1.1 200 OK\r\nX-Test: yes\n\r\n"), - ("bare CR", b"HTTP/1.1 200 OK\r\nX-Test: yes\rX: no\r\n\r\n"), - ( - "obsolete fold", - b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n continued\r\n\r\n", - ), - ("unsupported protocol", b"HTTP/2 200 OK\r\n\r\n"), - ("invalid status range", b"HTTP/1.1 099 Nope\r\n\r\n"), - ("non-numeric status", b"HTTP/1.1 2O0 Nope\r\n\r\n"), - ("four-digit status", b"HTTP/1.1 2000 Nope\r\n\r\n"), - ("missing reason separator", b"HTTP/1.1 200\r\n\r\n"), - ("malformed field", b"HTTP/1.1 200 OK\r\nX-Test\r\n\r\n"), - ("empty field name", b"HTTP/1.1 200 OK\r\n: value\r\n\r\n"), - ( - "non-token field name", - b"HTTP/1.1 200 OK\r\nBad Name: value\r\n\r\n", - ), - ( - "whitespace before colon", - b"HTTP/1.1 200 OK\r\nX-Test : value\r\n\r\n", - ), - ( - "field control byte", - b"HTTP/1.1 200 OK\r\nX-Test: value\x0bmore\r\n\r\n", - ), - ( - "field delete byte", - b"HTTP/1.1 200 OK\r\nX-Test: value\x7f\r\n\r\n", - ), - ("reason control byte", b"HTTP/1.1 200 O\x00K\r\n\r\n"), - ( - "early empty line", - b"HTTP/1.1 200 OK\r\n\r\nX-Smuggled: yes\r\n\r\n", - ), - ]; - - for (name, input) in cases { - assert!( - validate_http_response_header_block(input).is_err(), - "{name} unexpectedly passed validation" - ); - } - } - - #[test] - fn response_head_validation_accepts_supported_informational_and_upgrade_responses() { - for response in [ - &b"HTTP/1.0 200 OK\r\nX-Test: yes\r\n\r\n"[..], - &b"HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n"[..], - &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"[..], - ] { - validate_http_response_header_block(response).expect("valid HTTP/1.x response head"); - } - } - - #[tokio::test] - async fn response_middleware_rejects_forged_request_id_trailer() { - let correlation = test_correlation("trusted"); - let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( - StreamingResponseMiddleware, - )); - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "response".into(), - implementation: "test/streaming-response".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let middleware = HttpResponseMiddlewareRelay { - chain: &chain, - runner: &runner, - request_context: RequestContext { - request_id: "middleware-request".into(), - sandbox_id: "sandbox-1".into(), - ..Default::default() - }, - target: HttpRequestTarget { - scheme: "https".into(), - host: "example.test".into(), - port: 443, - method: "GET".into(), - path: "/data".into(), - query: String::new(), - }, - policy_name: "test-policy", - generation_guard: None, - whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, - }; - let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; - let mut client = Vec::new(); - - let error = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - Some(middleware), - ) - .await - .expect_err("platform correlation trailers must fail closed"); - - assert!( - error.to_string().contains("protected field name"), - "{error}" - ); - let delivered = String::from_utf8_lossy(&client); - assert!(!delivered.contains("forged"), "{delivered}"); - assert!( - delivered.contains("x-openshell-request-id: trusted\r\n"), - "{delivered}" - ); - assert_eq!(request_id_header_count(&client), 1); - } - - #[test] - fn response_middleware_rejects_declared_request_id_trailer() { - let error = parse_response_head_for_middleware( - b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: X-OpenShell-Request-ID\r\n\r\n", - ) - .expect_err("platform correlation trailer declarations must fail closed"); - - assert!( - error.to_string().contains("protected trailer field"), - "{error}" - ); - } - - #[tokio::test] - async fn informational_responses_are_not_decorated_but_upgrade_receives_a_receipt() { - let correlation = test_correlation("receipt-1"); - let mut upstream = - &b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; - let mut client = Vec::new(); - - relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect("relay response"); - assert!(client.starts_with(b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n")); - assert_eq!(request_id_header_count(&client), 1); - - let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; - let mut client = Vec::new(); - let outcome = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect("relay response"); - assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); - assert_eq!(request_id_header_count(&client), 1); - assert!(!String::from_utf8_lossy(&client).contains("forged")); - assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); - } - - #[tokio::test] - async fn switching_protocols_removes_receipt_connection_nomination() { - let correlation = test_correlation("trusted-upgrade"); - let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\n\ - Connection: X-OpenShell-Request-ID\r\n\ - cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id, UpGrAdE\r\n\ - Upgrade: websocket\r\n\ - X-OpenShell-Request-ID: forged\r\n\r\n"[..]; - let mut client = Vec::new(); - - let outcome = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect("relay response"); - - assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); - let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); - let connection_headers: Vec<_> = delivered - .lines() - .filter(|line| { - line.split_once(':') - .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) - }) - .collect(); - assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive, UpGrAdE"]); - assert!(!delivered.contains("forged"), "{delivered}"); - assert!( - delivered.contains("x-openshell-request-id: trusted-upgrade\r\n"), - "{delivered}" - ); - assert_eq!(request_id_header_count(delivered.as_bytes()), 1); - } - - #[tokio::test] - async fn switching_protocols_rejects_folded_receipt_connection_nomination() { - let correlation = test_correlation("trusted-upgrade"); - let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade,\r\n\tX-OpenShell-Request-ID\r\nUpgrade: websocket\r\n\r\n"[..]; - let mut client = Vec::new(); - - let error = relay_response( - "GET", - &mut upstream, - &mut client, - RelayResponseOptions { - correlation: Some(&correlation), - ..Default::default() - }, - None, - ) - .await - .expect_err("folded upgrade fields must fail closed"); - - assert!( - error - .to_string() - .contains("response header continuation lines are not supported"), - "{error}" - ); - assert!(client.is_empty()); - } - - #[test] - fn http_10_connection_persistence() { - let default_close = "HTTP/1.0 200 OK\r\nServer: test\r\n\r\n"; - assert!(response_is_http_10(default_close)); - assert!(!parse_connection_keep_alive(default_close)); - - let keep_alive = "HTTP/1.0 200 OK\r\nConnection: keep-alive\r\n\r\n"; - assert!(response_is_http_10(keep_alive)); - assert!(parse_connection_keep_alive(keep_alive)); - } - - #[test] - fn response_middleware_preflight_keeps_read_only_content_length() { - let parsed = parse_response_head_for_middleware( - b"HTTP/1.1 206 Partial Content\r\n\ - Content-Length: 5\r\n\ - Content-Encoding: gzip\r\n\ - Content-Range: bytes 0-4/10\r\n\ - Connection: x-hop\r\n\ - X-Hop: omitted\r\n\r\n", - ) - .expect("parse response head"); - - assert_eq!( - parsed.headers, - vec![ - HttpHeader { - name: "content-length".into(), - value: "5".into(), - }, - HttpHeader { - name: "content-encoding".into(), - value: "gzip".into(), - }, - HttpHeader { - name: "content-range".into(), - value: "bytes 0-4/10".into(), - }, - ] - ); - - let nominated = parse_response_head_for_middleware( - b"HTTP/1.1 200 OK\r\nConnection: Content-Length\r\nContent-Length: 5\r\n\r\n", - ) - .expect("parse nominated response head"); - assert!(nominated.headers.is_empty()); - } - - #[test] - fn response_middleware_serializes_only_relay_owned_framing() { - let headers = vec![ - HttpHeader { - name: "content-length".into(), - value: "999".into(), - }, - HttpHeader { - name: "Content-Length".into(), - value: "998".into(), - }, - HttpHeader { - name: "content-type".into(), - value: "text/plain".into(), - }, - ]; - - for (framing, expected) in [ - ( - ResponseFraming::Preserve(BodyLength::ContentLength(5)), - Some("Content-Length: 5\r\n"), - ), - ( - ResponseFraming::ContentLength(9), - Some("Content-Length: 9\r\n"), - ), - (ResponseFraming::Chunked, None), - (ResponseFraming::Preserve(BodyLength::None), None), - ] { - let serialized = String::from_utf8(serialize_response_head( - "HTTP/1.1 200 OK", - &headers, - &[], - framing, - false, - &[], - )) - .expect("serialized response head"); - - assert_eq!( - serialized - .lines() - .filter(|line| line.to_ascii_lowercase().starts_with("content-length:")) - .count(), - usize::from(expected.is_some()) - ); - if let Some(expected) = expected { - assert!(serialized.contains(expected)); - } - if matches!(framing, ResponseFraming::Chunked) { - assert!(serialized.contains("Transfer-Encoding: chunked\r\n")); - } - } - } -} +mod tests; diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response/tests.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response/tests.rs new file mode 100644 index 0000000000..ab331584b7 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response/tests.rs @@ -0,0 +1,3670 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_supervisor_middleware::{ + HttpResponseInvocation, HttpResponseInvocationOutcome as Outcome, NamespacedFinding, +}; + +fn invocation(outcome: Outcome, failed: bool) -> HttpResponseInvocation { + HttpResponseInvocation { + config_name: "guard".to_string(), + implementation: "test/guard".to_string(), + outcome, + sequence: None, + input_size: 0, + output_size: None, + failed, + transformed: false, + stage_disabled: false, + reason_code: None, + failure_category: None, + } +} + +fn finding() -> NamespacedFinding { + NamespacedFinding { + middleware: "guard".to_string(), + finding: openshell_core::proto::Finding { + r#type: "test.finding".to_string(), + label: "Test finding".to_string(), + count: 1, + severity: "high".to_string(), + ..Default::default() + }, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn response_terminal_failure_without_diagnostics_has_no_deny_trace() { + use crate::telemetry::tests::{assert_no_deny, exported}; + + // Platform refusals can lack invocation diagnostics, but must retain + // their terminal failure record. Without an HTTP record, no deny span. + for committed in [false, true] { + let (events, received) = exported("warn", || { + super::emit_http_response_middleware_failure( + "response-policy", + &HttpRequestTarget::default(), + 200, + committed, + None, + ); + }) + .await; + + assert_eq!(events.len(), 1, "only the terminal failure is emitted"); + assert_eq!( + events[0]["finding_info"]["uid"], + "openshell.middleware.http_response_failure" + ); + assert_no_deny(&events, &received); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn response_diagnostic_findings_share_a_deny_span_only_when_delivery_is_denied() { + use crate::telemetry::tests::{assert_deny_resolves, assert_no_deny, exported}; + + for denied in [false, true] { + let (events, received) = exported("warn", || { + let outcome = if denied { + Outcome::BlockDelivery + } else { + Outcome::PassThrough + }; + let diagnostics = openshell_supervisor_middleware::HttpResponseDiagnostics { + invocations: vec![invocation(outcome, false)], + findings: vec![finding()], + ..Default::default() + }; + super::emit_http_response_diagnostics( + "response-policy", + &HttpRequestTarget::default(), + 200, + &diagnostics, + denied, + None, + None, + ); + }) + .await; + // A blocking invocation adds its own block finding. + assert_eq!(events.len(), if denied { 3 } else { 2 }); + if denied { + let blocked = events + .iter() + .find(|event| { + event["finding_info"]["uid"] == "openshell.middleware.http_response_blocked" + }) + .expect("blocked delivery emits its refusal finding"); + assert_eq!(blocked["action_id"], 2); + assert_eq!(blocked["action"], "Denied"); + assert_eq!(received.spans.len(), 1); + assert_deny_resolves(&events, &received, true); + } else { + assert_no_deny(&events, &received); + } + } +} + +#[test] +fn fail_closed_invocations_and_enforced_failures_record_the_denial() { + let target = HttpRequestTarget::default(); + for (outcome, failed, action_id) in [ + (Outcome::BlockDelivery, false, 2), + (Outcome::FailClosed, true, 2), + (Outcome::FailOpen, true, 99), + (Outcome::PassThrough, false, 1), + ] { + let events = super::http_response_middleware_invocation_events( + "p", + &target, + 200, + &[invocation(outcome, failed)], + ); + assert_eq!( + events[0].to_json().unwrap()["action_id"], + action_id, + "{outcome:?}" + ); + } + for denied in [false, true] { + let failure = + super::http_response_middleware_failure_event("p", &target, 200, false, denied) + .to_json() + .unwrap(); + assert_eq!(failure["action_id"] == 2, denied); + } +} + +use super::{ + DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, HttpResponseMiddlewareRelay, RelayResponseOptions, + ResponseFraming, WebSocketResponseValidation, http_response_diagnostic_truncation_event, + parse_connection_keep_alive, parse_response_head_for_middleware, relay_response, + response_is_http_10, response_middleware_failure_denial, response_middleware_observation, + serialize_response_head, validate_http_response_header_block, +}; +use crate::l7::correlation::MiddlewareObservation; +use crate::l7::provider::{BodyLength, L7Request, RelayOutcome}; +use crate::l7::relay::L7EvalContext; +use openshell_core::proto::{ + Decision, ExistingHeaderAction, HeaderMutation, HttpHeader, HttpRequestResult, + HttpRequestTarget, HttpResponseBlockDelivery, HttpResponseBodyMode, HttpResponseBodyResult, + HttpResponseBodySkipRemaining, HttpResponseBodyTransform, HttpResponseEvent, + HttpResponseEventResult, HttpResponsePreflightInspect, HttpResponsePreflightResult, + HttpResponseTrailersResult, MiddlewareBinding, MiddlewareManifest, RequestContext, + SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, WriteHeader, header_mutation, + http_response_body_result, http_response_body_skip_remaining, http_response_body_transform, + http_response_body_unit, http_response_event, http_response_event_result, + http_response_preflight_result, +}; +use std::collections::{HashMap, VecDeque}; +use std::pin::Pin; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +struct StreamingResponseMiddleware { + body_mode: i32, + body_delay: Option, + mutate_headers: bool, +} + +struct BlockingResponseMiddleware; + +struct RequestOnlyMiddleware; + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for RequestOnlyMiddleware { + async fn describe(&self) -> MiddlewareManifest { + let mut manifest = openshell_supervisor_middleware::InProcessMiddleware::describe( + &BlockingResponseMiddleware, + ) + .await; + manifest.name = "test/request-only".into(); + manifest.bindings[0].operation = SupervisorMiddlewareOperation::HttpRequest as i32; + manifest.bindings[0].phase = SupervisorMiddlewarePhase::PreCredentials as i32; + manifest + } + + async fn validate_config(&self, _: &str, _: &prost_types::Struct) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + panic!("request hook must not run on a response") + } +} + +#[tokio::test] +async fn an_absent_response_binding_does_not_create_a_cancelled_phase() { + use tracing_subscriber::layer::SubscriberExt as _; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = + tracing_subscriber::registry().with(openshell_otel::layer(&provider, "no-response-hook")); + let _subscriber = tracing::subscriber::set_default(subscriber); + let correlation = test_correlation("request-only"); + let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new(RequestOnlyMiddleware)); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "request".into(), + implementation: "test/request-only".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut output = Vec::new(); + relay_response( + "GET", + &mut upstream, + &mut output, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(test_response_middleware(&runner, &chain, "request-only")), + ) + .await + .unwrap(); + drop(correlation); + assert!(output.ends_with(b"\r\n\r\nbody")); + assert!( + !exporter + .get_finished_spans() + .unwrap() + .iter() + .any(|span| span.name == "openshell.middleware.response") + ); +} + +enum BodyTestAction { + Block, + ReloadAndSkipTransform(Arc), +} + +struct BodyResponseMiddleware { + action: BodyTestAction, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReloadThenErrorPoint { + Describe, + Preflight, + Finish, +} + +struct ReloadThenErrorResponseMiddleware { + point: ReloadThenErrorPoint, + body_mode: i32, + engine: Arc, +} + +struct PartialThenErrorWriter { + remaining_success: usize, +} + +struct ReloadOnBodyReader { + head: Vec, + head_position: usize, + body: Vec, + body_position: usize, + reloaded: bool, + engine: Arc, +} + +struct SegmentsThenPendingReader { + segments: VecDeque>, + segment_position: usize, + pending_polled: Arc, +} + +impl SegmentsThenPendingReader { + fn new(segments: impl IntoIterator>, pending_polled: Arc) -> Self { + Self { + segments: segments.into_iter().collect(), + segment_position: 0, + pending_polled, + } + } +} + +impl AsyncRead for SegmentsThenPendingReader { + fn poll_read( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + while let Some(segment) = self.segments.front() { + if self.segment_position == segment.len() { + self.segments.pop_front(); + self.segment_position = 0; + continue; + } + let remaining = &segment[self.segment_position..]; + let count = remaining.len().min(buffer.remaining()); + buffer.put_slice(&remaining[..count]); + self.segment_position += count; + return Poll::Ready(Ok(())); + } + self.pending_polled.store(true, Ordering::Release); + Poll::Pending + } +} + +async fn wait_for_pending_read(pending_polled: &AtomicBool) { + tokio::time::timeout(std::time::Duration::from_millis(100), async { + while !pending_polled.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("response relay reached the pending read"); +} + +impl ReloadOnBodyReader { + fn new(head: &[u8], body: &[u8], engine: Arc) -> Self { + Self { + head: head.to_vec(), + head_position: 0, + body: body.to_vec(), + body_position: 0, + reloaded: false, + engine, + } + } +} + +impl AsyncRead for ReloadOnBodyReader { + fn poll_read( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + if self.head_position < self.head.len() { + let remaining = &self.head[self.head_position..]; + let count = remaining.len().min(buffer.remaining()); + buffer.put_slice(&remaining[..count]); + self.head_position += count; + return Poll::Ready(Ok(())); + } + if !self.reloaded { + self.engine + .reload( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("reload test policy"); + self.reloaded = true; + } + if self.body_position < self.body.len() { + let remaining = &self.body[self.body_position..]; + let count = remaining.len().min(buffer.remaining()); + buffer.put_slice(&remaining[..count]); + self.body_position += count; + } + Poll::Ready(Ok(())) + } +} + +#[derive(Default)] +struct ControlledResponseWriter { + bytes: Vec, + zero_first_write: bool, + shutdown_error: bool, +} + +impl AsyncWrite for ControlledResponseWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + if self.zero_first_write { + self.zero_first_write = false; + return Poll::Ready(Ok(0)); + } + self.bytes.extend_from_slice(bytes); + Poll::Ready(Ok(bytes.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll> { + if self.shutdown_error { + Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "test shutdown failure", + ))) + } else { + Poll::Ready(Ok(())) + } + } +} + +impl AsyncWrite for PartialThenErrorWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + if self.remaining_success == 0 { + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "test downstream disconnect", + ))); + } + let written = bytes.len().min(self.remaining_success); + self.remaining_success -= written; + Poll::Ready(Ok(written)) + } + + fn poll_flush(self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } +} + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for BlockingResponseMiddleware { + async fn describe(&self) -> MiddlewareManifest { + MiddlewareManifest { + name: "test/blocking-response".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpResponse as i32, + phase: SupervisorMiddlewarePhase::PreReturn as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + expected_audience: String::new(), + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-blocking-response", + "test", + [], + )), + } + } + + async fn validate_config( + &self, + _middleware_name: &str, + _config: &prost_types::Struct, + ) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _request: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(HttpRequestResult { + decision: Decision::Allow as i32, + ..Default::default() + }) + } + + async fn open_http_response_pre_return( + &self, + mut requests: mpsc::Receiver, + ) -> Result { + let first = requests + .recv() + .await + .ok_or_else(|| tonic::Status::invalid_argument("missing response preflight"))?; + if !matches!(first.event, Some(http_response_event::Event::Preflight(_))) { + return Err(tonic::Status::invalid_argument( + "expected response preflight", + )); + } + let result = HttpResponseEventResult { + result: Some(http_response_event_result::Result::PreflightResult( + HttpResponsePreflightResult { + action: Some(http_response_preflight_result::Action::BlockDelivery( + HttpResponseBlockDelivery {}, + )), + reason_code: "test_block".into(), + ..Default::default() + }, + )), + }; + Ok(Box::pin(futures::stream::iter([Ok(result)]))) + } +} + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for StreamingResponseMiddleware { + async fn describe(&self) -> MiddlewareManifest { + MiddlewareManifest { + name: "test/streaming-response".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpResponse as i32, + phase: SupervisorMiddlewarePhase::PreReturn as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + expected_audience: String::new(), + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-streaming-response", + "test", + [], + )), + } + } + + async fn validate_config( + &self, + _middleware_name: &str, + _config: &prost_types::Struct, + ) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _request: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(HttpRequestResult { + decision: Decision::Allow as i32, + ..Default::default() + }) + } + + async fn open_http_response_pre_return( + &self, + mut requests: mpsc::Receiver, + ) -> Result { + let (sender, receiver) = mpsc::channel(4); + let body_mode = self.body_mode; + let body_delay = self.body_delay; + let mutate_headers = self.mutate_headers; + tokio::spawn(async move { + while let Some(event) = requests.recv().await { + let Some(event) = event.event else { + break; + }; + let result = match event { + http_response_event::Event::Preflight(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::PreflightResult( + HttpResponsePreflightResult { + action: Some(http_response_preflight_result::Action::Inspect( + HttpResponsePreflightInspect { + body_mode, + header_mutations: mutate_headers + .then(|| HeaderMutation { + operation: Some(header_mutation::Operation::Write( + WriteHeader { + name: "x-inspected".into(), + value: "true".into(), + on_existing: ExistingHeaderAction::Overwrite + as i32, + }, + )), + }) + .into_iter() + .collect(), + }, + )), + ..Default::default() + }, + )), + }, + http_response_event::Event::Body(body) => { + if let Some(delay) = body_delay { + tokio::time::sleep(delay).await; + } + let Some(http_response_body_unit::Payload::Data(data)) = body.payload + else { + break; + }; + HttpResponseEventResult { + result: Some(http_response_event_result::Result::BodyResult( + HttpResponseBodyResult { + sequence: body.sequence, + action: Some(http_response_body_result::Action::Transform( + HttpResponseBodyTransform { + replacement: Some( + http_response_body_transform::Replacement::Data( + data, + ), + ), + }, + )), + ..Default::default() + }, + )), + } + } + http_response_event::Event::Trailers(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::TrailersResult( + HttpResponseTrailersResult::default(), + )), + }, + http_response_event::Event::SessionEnd(_) => break, + }; + if sender.send(Ok(result)).await.is_err() { + break; + } + } + }); + Ok(Box::pin(ReceiverStream::new(receiver))) + } +} + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for BodyResponseMiddleware { + async fn describe(&self) -> MiddlewareManifest { + MiddlewareManifest { + name: "test/body-response".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpResponse as i32, + phase: SupervisorMiddlewarePhase::PreReturn as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + expected_audience: String::new(), + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-body-response", + "test", + [], + )), + } + } + + async fn validate_config( + &self, + _middleware_name: &str, + _config: &prost_types::Struct, + ) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _request: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(HttpRequestResult { + decision: Decision::Allow as i32, + ..Default::default() + }) + } + + async fn open_http_response_pre_return( + &self, + mut requests: mpsc::Receiver, + ) -> Result { + let (sender, receiver) = mpsc::channel(4); + let action = match &self.action { + BodyTestAction::Block => BodyTestAction::Block, + BodyTestAction::ReloadAndSkipTransform(engine) => { + BodyTestAction::ReloadAndSkipTransform(Arc::clone(engine)) + } + }; + tokio::spawn(async move { + while let Some(event) = requests.recv().await { + let Some(event) = event.event else { + break; + }; + let result = match event { + http_response_event::Event::Preflight(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::PreflightResult( + HttpResponsePreflightResult { + action: Some(http_response_preflight_result::Action::Inspect( + HttpResponsePreflightInspect { + body_mode: HttpResponseBodyMode::StreamBytes as i32, + header_mutations: Vec::new(), + }, + )), + ..Default::default() + }, + )), + }, + http_response_event::Event::Body(body) => { + let Some(http_response_body_unit::Payload::Data(data)) = body.payload + else { + break; + }; + let action = match &action { + BodyTestAction::Block => { + http_response_body_result::Action::BlockDelivery( + HttpResponseBlockDelivery {}, + ) + } + BodyTestAction::ReloadAndSkipTransform(engine) => { + engine + .reload( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("reload test policy"); + http_response_body_result::Action::SkipRemaining( + HttpResponseBodySkipRemaining { + current: Some( + http_response_body_skip_remaining::Current::Transform( + HttpResponseBodyTransform { + replacement: Some( + http_response_body_transform::Replacement::Data(data), + ), + }, + ), + ), + }, + ) + } + }; + HttpResponseEventResult { + result: Some(http_response_event_result::Result::BodyResult( + HttpResponseBodyResult { + sequence: body.sequence, + action: Some(action), + reason_code: "test_body_action".into(), + ..Default::default() + }, + )), + } + } + http_response_event::Event::Trailers(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::TrailersResult( + HttpResponseTrailersResult::default(), + )), + }, + http_response_event::Event::SessionEnd(_) => break, + }; + if sender.send(Ok(result)).await.is_err() { + break; + } + } + }); + Ok(Box::pin(ReceiverStream::new(receiver))) + } +} + +impl ReloadThenErrorResponseMiddleware { + fn reload_policy(&self) { + self.engine + .reload( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("reload test policy"); + } +} + +#[tonic::async_trait] +impl openshell_supervisor_middleware::InProcessMiddleware for ReloadThenErrorResponseMiddleware { + async fn describe(&self) -> MiddlewareManifest { + if self.point == ReloadThenErrorPoint::Describe { + self.reload_policy(); + } + MiddlewareManifest { + name: "test/reload-then-error-response".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpResponse as i32, + phase: SupervisorMiddlewarePhase::PreReturn as i32, + max_payload_bytes: if self.point == ReloadThenErrorPoint::Describe { + 0 + } else { + 4096 + }, + request_timeout: None, + }], + expected_audience: String::new(), + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-reload-then-error-response", + "test", + [], + )), + } + } + + async fn validate_config( + &self, + _middleware_name: &str, + _config: &prost_types::Struct, + ) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _request: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(HttpRequestResult { + decision: Decision::Allow as i32, + ..Default::default() + }) + } + + async fn open_http_response_pre_return( + &self, + mut requests: mpsc::Receiver, + ) -> Result { + let first = requests + .recv() + .await + .ok_or_else(|| tonic::Status::invalid_argument("missing response preflight"))?; + if !matches!(first.event, Some(http_response_event::Event::Preflight(_))) { + return Err(tonic::Status::invalid_argument( + "expected response preflight", + )); + } + if self.point == ReloadThenErrorPoint::Preflight { + self.reload_policy(); + return Err(tonic::Status::internal( + "test preflight failure after policy reload", + )); + } + + let (sender, receiver) = mpsc::channel(4); + sender + .send(Ok(HttpResponseEventResult { + result: Some(http_response_event_result::Result::PreflightResult( + HttpResponsePreflightResult { + action: Some(http_response_preflight_result::Action::Inspect( + HttpResponsePreflightInspect { + body_mode: self.body_mode, + header_mutations: Vec::new(), + }, + )), + ..Default::default() + }, + )), + })) + .await + .map_err(|_| tonic::Status::unavailable("test response stream closed"))?; + + let point = self.point; + let engine = Arc::clone(&self.engine); + tokio::spawn(async move { + while let Some(event) = requests.recv().await { + let Some(event) = event.event else { + break; + }; + let result = match event { + http_response_event::Event::Body(body) + if point == ReloadThenErrorPoint::Finish && body.end_of_stream => + { + engine + .reload( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("reload test policy"); + let _ = sender + .send(Err(tonic::Status::internal( + "test finish failure after policy reload", + ))) + .await; + break; + } + http_response_event::Event::Body(body) => { + let Some(http_response_body_unit::Payload::Data(data)) = body.payload + else { + break; + }; + HttpResponseEventResult { + result: Some(http_response_event_result::Result::BodyResult( + HttpResponseBodyResult { + sequence: body.sequence, + action: Some(http_response_body_result::Action::Transform( + HttpResponseBodyTransform { + replacement: Some( + http_response_body_transform::Replacement::Data( + data, + ), + ), + }, + )), + ..Default::default() + }, + )), + } + } + http_response_event::Event::Trailers(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::TrailersResult( + HttpResponseTrailersResult::default(), + )), + }, + http_response_event::Event::Preflight(_) + | http_response_event::Event::SessionEnd(_) => break, + }; + if sender.send(Ok(result)).await.is_err() { + break; + } + } + }); + Ok(Box::pin(ReceiverStream::new(receiver))) + } +} + +fn test_correlation_for_method( + request_id: &str, + method: &str, +) -> crate::l7::correlation::RequestCorrelation { + let request = L7Request { + action: method.into(), + target: "/data".into(), + query_params: HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + request_id.into(), + ) +} + +fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { + test_correlation_for_method(request_id, "GET") +} + +fn test_response_middleware<'a>( + runner: &'a openshell_supervisor_middleware::ChainRunner, + chain: &'a [openshell_supervisor_middleware::ChainEntry], + request_id: &str, +) -> HttpResponseMiddlewareRelay<'a> { + HttpResponseMiddlewareRelay { + chain, + runner, + request_context: RequestContext { + request_id: request_id.into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + } +} + +fn response_audit_event_for( + capture: &openshell_ocsf::test_support::OcsfCapture, + request_id: &str, +) -> serde_json::Value { + let mut events = capture + .json_events() + .into_iter() + .filter(|event| { + event["class_uid"] == 4002 && event["metadata"]["correlation_uid"] == request_id + }) + .collect::>(); + let terminal = events + .iter() + .position(|event| event["firewall_rule"]["type"] == "response-correlation"); + if let Some(index) = terminal { + return events.remove(index); + } + assert_eq!( + events.len(), + 1, + "one primary record when no post-header exception exists" + ); + events.pop().expect("primary event") +} + +fn request_id_header_count(response: &[u8]) -> usize { + String::from_utf8_lossy(response) + .lines() + .filter(|line| { + line.split_once(':').is_some_and(|(name, _)| { + name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) + }) + }) + .count() +} + +#[tokio::test] +async fn allowed_final_response_carries_platform_receipt() { + let correlation = test_correlation("receipt-1"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + assert_eq!(request_id_header_count(&client), 1); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); +} + +#[tokio::test] +async fn cancellation_before_final_head_emits_one_incomplete_terminal_event() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-cancel-before-head"); + let (mut upstream, _upstream_peer) = tokio::io::duplex(64); + let mut client = Vec::new(); + + tokio::time::timeout( + std::time::Duration::from_millis(10), + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ), + ) + .await + .expect_err("relay remains pending before final head"); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-cancel-before-head"); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!(event["unmapped"]["response_selection_cause"], "cancelled"); + assert_eq!(event["unmapped"]["delivery_started"], false); + assert_eq!(event["unmapped"]["partial_delivery"], false); +} + +#[tokio::test] +async fn unsolicited_switching_protocols_is_a_protocol_failure() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-unsolicited-101"); + let mut upstream = + &b"HTTP/1.1 101 Switching Protocols\r\nConnection: upgrade\r\nUpgrade: websocket\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + client_requested_upgrade: false, + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("unsolicited 101 must fail"); + assert!(client.is_empty()); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-unsolicited-101"); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_protocol_error" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert!(event.get("http_response").is_none()); + assert_eq!(event["unmapped"]["delivery_started"], false); +} + +#[tokio::test] +async fn invalid_websocket_handshake_is_a_correlated_upstream_protocol_failure() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let request_id = "response-invalid-websocket-accept"; + let correlation = test_correlation(request_id); + let validation = WebSocketResponseValidation { + expected_accept: "expected-accept".into(), + expected_extension: None, + offered_subprotocols: Vec::new(), + }; + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: invalid\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + client_requested_upgrade: true, + websocket: Some(validation), + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("invalid upstream WebSocket handshake must fail"); + assert!( + error + .downcast_ref::() + .is_some_and(|error| matches!( + error, + super::RelayError::Protocol { + code: "invalid_websocket_handshake", + .. + } + )), + "{error}" + ); + assert!(client.is_empty(), "invalid 101 response reached the client"); + drop(correlation); + + let event = response_audit_event_for(&capture, request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_protocol_error" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 101); + assert!(event.get("http_response").is_none()); + assert!( + event["unmapped"] + .get("effective_http_status_code") + .is_none() + ); + assert_eq!(event["unmapped"]["delivery_started"], false); + assert_eq!(event["unmapped"]["partial_delivery"], false); +} + +#[tokio::test] +async fn whole_body_cancellation_before_selection_has_no_effective_status() { + use tokio::io::AsyncWriteExt as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-whole-body-cancel"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::WholeBodyBytes as i32, + body_delay: None, + mutate_headers: true, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = test_response_middleware(&runner, &chain, "response-whole-body-cancel"); + let (mut upstream, mut upstream_peer) = tokio::io::duplex(256); + upstream_peer + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nbody") + .await + .expect("seed partial response"); + let mut client = Vec::new(); + + tokio::time::timeout( + std::time::Duration::from_millis(20), + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ), + ) + .await + .expect_err("whole-body response remains pending"); + assert!(client.is_empty()); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-whole-body-cancel"); + assert_eq!(event["unmapped"]["response_selection_cause"], "cancelled"); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert!(event.get("http_response").is_none()); + assert!( + event["unmapped"] + .get("effective_http_status_code") + .is_none() + ); + assert_eq!(event["unmapped"]["delivery_started"], false); +} + +#[tokio::test] +async fn whole_body_fail_open_timeout_selects_upstream_response_on_commit() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let request_id = "response-whole-body-fail-open-timeout"; + let correlation = test_correlation(request_id); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::WholeBodyBytes as i32, + body_delay: Some(std::time::Duration::from_millis(75)), + mutate_headers: true, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailOpen, + }]; + let mut middleware = test_response_middleware(&runner, &chain, request_id); + middleware.whole_body_timeout = std::time::Duration::from_millis(10); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("whole-body timeout fails open"); + assert!(matches!(outcome, RelayOutcome::Reusable)); + assert!(client.ends_with(b"hello")); + drop(correlation); + + let event = response_audit_event_for(&capture, request_id); + assert!(event["unmapped"]["response_selection_cause"].is_null()); + assert_eq!(event["unmapped"]["middleware_fail_open"], true); + assert!( + event["unmapped"]["response_delivery_outcome"].is_null(), + "completion is not claimed by the primary record" + ); + assert!( + event["unmapped"]["upstream_response_code"].is_null(), + "unchanged upstream status is not duplicated" + ); + assert_eq!(event["http_response"]["code"], 200); + assert!( + event["unmapped"]["delivery_started"].is_null(), + "completion is not claimed by the primary record" + ); + assert!( + event["unmapped"]["partial_delivery"].is_null(), + "completion is not claimed by the primary record" + ); +} + +#[tokio::test] +async fn accepted_early_hints_are_separate_from_cancelled_final_delivery() { + use tokio::io::AsyncWriteExt as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-early-hints-cancel"); + let (mut upstream, mut upstream_peer) = tokio::io::duplex(256); + upstream_peer + .write_all(b"HTTP/1.1 103 Early Hints\r\nLink: \r\n\r\n") + .await + .expect("seed early hints"); + let mut client = Vec::new(); + + tokio::time::timeout( + std::time::Duration::from_millis(10), + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ), + ) + .await + .expect_err("relay remains pending after early hints"); + assert!(client.starts_with(b"HTTP/1.1 103 Early Hints\r\n")); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-early-hints-cancel"); + assert_eq!(event["unmapped"]["interim_delivery_started"], true); + assert!( + event["unmapped"]["interim_bytes_forwarded"] + .as_u64() + .is_some_and(|bytes| bytes > 0) + ); + assert_eq!(event["unmapped"]["delivery_started"], false); + assert_eq!(event["unmapped"]["partial_delivery"], false); +} + +#[tokio::test] +async fn response_head_reload_during_early_hints_is_not_forwarded() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let engine = Arc::new( + crate::opa::OpaEngine::from_strings( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("test policy"), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .expect("generation guard"); + let request_id = "response-early-hints-reload-race"; + let correlation = test_correlation(request_id); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::HeadersOnly as i32, + body_delay: None, + mutate_headers: false, + })); + let chain = Vec::new(); + let mut middleware = test_response_middleware(&runner, &chain, request_id); + middleware.generation_guard = Some(&generation_guard); + let mut upstream = ReloadOnBodyReader::new( + b"", + b"HTTP/1.1 103 Early Hints\r\nLink: \r\n\r\n", + Arc::clone(&engine), + ); + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("policy reload must win before forwarding early hints"); + assert!( + error + .downcast_ref::() + .is_some_and(|error| matches!(error, super::RelayError::PolicyReloaded(_))), + "{error}" + ); + assert!(client.is_empty(), "stale early hints reached the client"); + drop(correlation); + + let event = response_audit_event_for(&capture, request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "policy_reloaded" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!(event["unmapped"]["interim_delivery_started"], false); + assert_eq!(event["unmapped"]["delivery_started"], false); +} + +#[tokio::test] +async fn idle_before_final_response_head_wakes_on_policy_reload() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let engine = Arc::new( + crate::opa::OpaEngine::from_strings( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("test policy"), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .expect("generation guard"); + let request_id = "response-idle-before-head-reload"; + let correlation = test_correlation(request_id); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::HeadersOnly as i32, + body_delay: None, + mutate_headers: false, + })); + let chain = Vec::new(); + let mut middleware = test_response_middleware(&runner, &chain, request_id); + middleware.generation_guard = Some(&generation_guard); + let pending_polled = Arc::new(AtomicBool::new(false)); + let mut upstream = + SegmentsThenPendingReader::new(Vec::>::new(), Arc::clone(&pending_polled)); + let mut client = Vec::new(); + let mut relay = Box::pin(relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + )); + + tokio::select! { + result = &mut relay => panic!("idle response finished before reload: {result:?}"), + () = wait_for_pending_read(&pending_polled) => {} + } + engine + .reload( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("reload test policy"); + let error = tokio::time::timeout(std::time::Duration::from_millis(100), &mut relay) + .await + .expect("policy reload wakes the response-head read") + .expect_err("policy reload stops the response"); + assert!( + error + .downcast_ref::() + .is_some_and(|error| matches!(error, super::RelayError::PolicyReloaded(_))), + "{error}" + ); + drop(relay); + assert!(client.is_empty()); + drop(correlation); + + let event = response_audit_event_for(&capture, request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "policy_reloaded" + ); + assert_eq!(event["unmapped"]["delivery_started"], false); + assert_eq!(event["unmapped"]["partial_delivery"], false); +} + +#[tokio::test] +async fn framed_response_reads_wake_on_policy_reload_without_stale_body_bytes() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + + for (case_name, segments) in [ + ( + "fixed-payload", + vec![b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n".to_vec()], + ), + ( + "chunk-size", + vec![b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec()], + ), + ( + "chunk-payload", + vec![b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\n".to_vec()], + ), + ( + "chunk-terminator", + vec![b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody".to_vec()], + ), + ( + "chunk-trailer", + vec![b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n".to_vec()], + ), + ] { + let engine = Arc::new( + crate::opa::OpaEngine::from_strings( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("test policy"), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .expect("generation guard"); + let request_id = format!("response-stalled-{case_name}-reload"); + let correlation = test_correlation(&request_id); + let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( + StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::HeadersOnly as i32, + body_delay: None, + mutate_headers: false, + }, + )); + let chain = Vec::new(); + let mut middleware = test_response_middleware(&runner, &chain, &request_id); + middleware.generation_guard = Some(&generation_guard); + let pending_polled = Arc::new(AtomicBool::new(false)); + let mut upstream = SegmentsThenPendingReader::new(segments, Arc::clone(&pending_polled)); + let mut client = Vec::new(); + let mut relay = Box::pin(relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + )); + + tokio::select! { + result = &mut relay => { + panic!("{case_name} response finished before reload: {result:?}") + } + () = wait_for_pending_read(&pending_polled) => {} + } + engine + .reload( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("reload test policy"); + let error = tokio::time::timeout(std::time::Duration::from_millis(100), &mut relay) + .await + .expect("policy reload wakes the framed response read") + .expect_err("policy reload stops the response"); + assert!( + error + .downcast_ref::() + .is_some_and(|error| matches!(error, super::RelayError::PolicyReloaded(_))), + "{case_name}: {error}" + ); + drop(relay); + let head_end = client + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("final response head delivered before body acquisition"); + assert_eq!( + &client[head_end..], + b"", + "{case_name} leaked stale body or chunk framing bytes" + ); + drop(correlation); + + let event = response_audit_event_for(&capture, &request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "policy_reloaded" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!(event["unmapped"]["delivery_started"], true); + assert_eq!(event["unmapped"]["partial_delivery"], true); + } +} + +#[tokio::test] +async fn shutdown_failure_overrides_delivery_but_retains_upstream_selection() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-shutdown-failure"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"[..]; + let mut client = ControlledResponseWriter { + shutdown_error: true, + ..Default::default() + }; + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("shutdown failure"); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-shutdown-failure"); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_response" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "downstream_io_error" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert_eq!(event["http_response"]["code"], 200); + assert_eq!(event["unmapped"]["delivery_started"], true); + assert_eq!(event["unmapped"]["partial_delivery"], true); +} + +#[tokio::test] +async fn zero_byte_first_write_does_not_claim_delivery_started() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-zero-write"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = ControlledResponseWriter { + zero_first_write: true, + ..Default::default() + }; + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("write zero"); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-zero-write"); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "downstream_io_error" + ); + assert_eq!(event["unmapped"]["delivery_started"], false); + assert_eq!(event["unmapped"]["partial_delivery"], false); +} + +#[tokio::test] +async fn content_length_overflow_is_rejected_before_final_head() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-content-length-overflow"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbodyEXTRA"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("content-length overflow"); + assert!(client.is_empty()); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-content-length-overflow"); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_protocol_error" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert!( + event["unmapped"] + .get("effective_http_status_code") + .is_none() + ); + assert_eq!(event["unmapped"]["delivery_started"], false); +} + +#[tokio::test] +async fn malformed_chunk_is_typed_as_protocol_failure_after_head_delivery() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-malformed-chunk"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nZ\r\nbody\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("malformed chunk"); + assert!(client.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert!(!client.ends_with(b"body\r\n")); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-malformed-chunk"); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_protocol_error" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!(event["unmapped"]["delivery_started"], true); + assert_eq!(event["unmapped"]["partial_delivery"], true); +} + +#[tokio::test] +async fn complete_fixed_chunked_and_sse_responses_record_terminal_delivery() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer( + &provider, + "response-correlation-test", + )) + .with(ocsf.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + + for (request_id, response) in [ + ( + "response-fixed-complete", + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody".as_slice(), + ), + ( + "response-chunked-complete", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" + .as_slice(), + ), + ( + "response-sse-complete", + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\r\ndata: ok\n\n".as_slice(), + ), + ] { + let correlation = test_correlation(request_id); + let mut upstream = response; + let mut client = Vec::new(); + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("complete response relay"); + drop(correlation); + } + provider.force_flush().expect("flush spans"); + + let spans = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .filter(|span| span.name == "openshell.response.deliver") + .map(|span| { + span.attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>() + }) + .collect::>(); + assert_eq!(spans.len(), 3); + for attributes in spans { + assert_eq!( + attributes["openshell.upstream.http.response.status_code"], + "200" + ); + assert_eq!(attributes["http.response.status_code"], "200"); + assert_eq!(attributes["openshell.response.delivery_started"], "true"); + assert_eq!(attributes["openshell.response.partial_delivery"], "false"); + assert_eq!( + attributes["openshell.response.delivery_outcome"], + "complete" + ); + } + assert!( + !ocsf + .json_events() + .iter() + .any(|event| event["firewall_rule"]["type"] == "response-correlation"), + "routine completion remains in sampled spans" + ); + let primary = ocsf + .json_events() + .into_iter() + .filter(|event| event["class_uid"] == 4002) + .collect::>(); + assert_eq!(primary.len(), 3); + assert!( + primary + .iter() + .all(|event| event["http_response"]["code"] == 200) + ); +} + +#[tokio::test] +async fn downstream_disconnect_after_partial_head_write_is_correlated() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "response-correlation-test", + )); + let _subscriber = tracing::subscriber::set_default(subscriber); + let correlation = test_correlation("response-client-disconnect"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = PartialThenErrorWriter { + remaining_success: 8, + }; + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("downstream disconnect"); + drop(correlation); + provider.force_flush().expect("flush spans"); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.response.deliver") + .expect("HTTP enforcement span"); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!(attributes["openshell.response.delivery_started"], "true"); + assert_eq!(attributes["openshell.response.partial_delivery"], "true"); + assert_eq!( + attributes["openshell.response.delivery_outcome"], + "downstream_io_error" + ); +} + +#[tokio::test] +async fn truncated_fixed_response_is_not_reported_complete() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "response-correlation-test", + )); + let _subscriber = tracing::subscriber::set_default(subscriber); + let correlation = test_correlation("response-upstream-truncated"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nbody"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("truncated upstream response"); + drop(correlation); + provider.force_flush().expect("flush spans"); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.response.deliver") + .expect("HTTP enforcement span"); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!( + attributes["openshell.response.delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!( + attributes["openshell.response.selection_cause"], + "upstream_truncated" + ); + assert_eq!(attributes["openshell.response.partial_delivery"], "true"); +} + +#[tokio::test] +async fn middleware_truncated_fixed_and_chunked_bodies_select_complete_502() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + + for (request_id, response) in [ + ( + "response-middleware-fixed-truncated", + b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nbody".as_slice(), + ), + ( + "response-middleware-chunked-truncated", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbo".as_slice(), + ), + ] { + let correlation = test_correlation(request_id); + let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( + StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::WholeBodyBytes as i32, + body_delay: None, + mutate_headers: true, + }, + )); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = test_response_middleware(&runner, &chain, request_id); + let mut upstream = response; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("pre-commit truncation gets a bounded fallback"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + drop(correlation); + + let event = response_audit_event_for(&capture, request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_truncated" + ); + assert!( + event["unmapped"]["response_delivery_outcome"].is_null(), + "completion is not claimed by the primary record" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert_eq!(event["http_response"]["code"], 502); + assert_eq!(event["http_response"]["code"], 502); + assert!( + event["unmapped"]["partial_delivery"].is_null(), + "completion is not claimed by the primary record" + ); + } +} + +#[tokio::test] +async fn fallback_disconnect_preserves_the_precommit_upstream_cause() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-fallback-disconnect"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::WholeBodyBytes as i32, + body_delay: None, + mutate_headers: true, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = test_response_middleware(&runner, &chain, "response-fallback-disconnect"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nbody"[..]; + let mut client = ControlledResponseWriter { + zero_first_write: true, + ..Default::default() + }; + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("fallback head is not accepted downstream"); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-fallback-disconnect"); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_truncated" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], + "downstream_io_error" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert_eq!(event["http_response"]["code"], 502); + assert_eq!(event["unmapped"]["delivery_started"], false); + assert_eq!(event["unmapped"]["partial_delivery"], false); +} + +#[tokio::test(start_paused = true)] +async fn close_delimited_idle_timeout_is_not_reported_complete() { + use tokio::io::AsyncWriteExt as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "response-correlation-test", + )); + let _subscriber = tracing::subscriber::set_default(subscriber); + let correlation = test_correlation("response-idle-timeout"); + let (mut upstream, mut upstream_peer) = tokio::io::duplex(256); + upstream_peer + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nbody") + .await + .expect("seed response"); + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("idle timeout consumes close-delimited response"); + drop(upstream_peer); + drop(correlation); + provider.force_flush().expect("flush spans"); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.response.deliver") + .expect("HTTP enforcement span"); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!( + attributes["openshell.response.delivery_outcome"], + "cancelled_or_incomplete" + ); + assert_eq!( + attributes["openshell.response.selection_cause"], + "upstream_idle_timeout" + ); + assert_eq!(attributes["openshell.http.outcome"], "upstream_error"); + assert_eq!(attributes["openshell.response.partial_delivery"], "true"); +} + +#[tokio::test] +async fn branch08_response_middleware_denial_records_that_upstream_was_reached() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer( + &provider, + "response-correlation-test", + )) + .with(ocsf.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation_for_method("response-denied", "POST"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BlockingResponseMiddleware)); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/blocking-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "response-denied".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "POST".into(), + path: "/mutate".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "POST", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("middleware denial response"); + + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 403 Forbidden\r\n")); + assert!( + String::from_utf8_lossy(&client).contains("x-openshell-request-id: response-denied\r\n") + ); + drop(correlation); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.http.request") + .expect("HTTP enforcement span"); + assert!(matches!( + span.status, + opentelemetry::trace::Status::Error { .. } + )); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!(attributes["openshell.target.reached"], "true"); + assert_eq!(attributes["openshell.http.outcome"], "denied"); + assert_eq!(attributes["http.response.status_code"], "403"); + assert!(!attributes.contains_key("openshell.error.stage")); +} + +#[tokio::test] +async fn fully_delivered_middleware_failure_keeps_upstream_and_effective_statuses() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation("response-middleware-failure"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BlockingResponseMiddleware)); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/unavailable-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "response-middleware-failure".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("canonical middleware failure response"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + drop(correlation); + + let event = response_audit_event_for(&capture, "response-middleware-failure"); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "middleware_failed" + ); + assert!( + event["unmapped"]["response_delivery_outcome"].is_null(), + "completion is not claimed by the primary record" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert_eq!(event["http_response"]["code"], 502); + assert!( + event["unmapped"]["delivery_started"].is_null(), + "completion is not claimed by the primary record" + ); + assert!( + event["unmapped"]["partial_delivery"].is_null(), + "completion is not claimed by the primary record" + ); +} + +#[tokio::test] +async fn bodiless_middleware_denial_and_failure_select_canonical_statuses() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + + for (action_name, denied) in [("denied", true), ("failed", false)] { + for (case_name, method, response, upstream_status) in [ + ( + "head", + "HEAD", + b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n".as_slice(), + 200, + ), + ( + "204", + "GET", + b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n".as_slice(), + 204, + ), + ( + "304", + "GET", + b"HTTP/1.1 304 Not Modified\r\nContent-Length: 0\r\n\r\n".as_slice(), + 304, + ), + ] { + let request_id = format!("response-bodiless-{case_name}-{action_name}"); + let correlation = test_correlation_for_method(&request_id, method); + let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( + BlockingResponseMiddleware, + )); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: if denied { + "test/blocking-response".into() + } else { + "test/unavailable-response".into() + }, + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let mut middleware = test_response_middleware(&runner, &chain, &request_id); + middleware.target.method = method.into(); + let mut upstream = response; + let mut client = Vec::new(); + + let outcome = relay_response( + method, + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("bodiless middleware result gets a canonical response"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + let effective_status = if denied { 403 } else { 502 }; + let expected_head = if denied { + b"HTTP/1.1 403 Forbidden\r\n".as_slice() + } else { + b"HTTP/1.1 502 Bad Gateway\r\n".as_slice() + }; + assert!( + client.starts_with(expected_head), + "{case_name}/{action_name}: {}", + String::from_utf8_lossy(&client) + ); + drop(correlation); + + let event = response_audit_event_for(&capture, &request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + if denied { + "middleware_denied" + } else { + "middleware_failed" + } + ); + assert!( + event["unmapped"]["response_delivery_outcome"].is_null(), + "completion is not claimed by the primary record" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], upstream_status); + assert_eq!(event["http_response"]["code"], effective_status); + assert!( + !capture + .json_events() + .iter() + .any( + |event| event["firewall_rule"]["type"] == "middleware-correlation" + && event["metadata"]["correlation_uid"] == request_id + ) + ); + } + } +} + +#[tokio::test] +async fn response_finish_block_retains_an_explicit_denial() { + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BodyResponseMiddleware { + action: BodyTestAction::Block, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/body-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let outcome = runner + .preflight_http_response( + &chain, + openshell_supervisor_middleware::HttpResponsePreflightInput { + context: RequestContext::default(), + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + status_code: 200, + declared_body_length: Some(0), + headers: Vec::new(), + connection_nominated_headers: Vec::new(), + }, + ) + .await + .expect("response preflight"); + let failure = outcome + .session + .expect("streaming response session") + .finish(Vec::new()) + .await + .expect_err("final body event is denied"); + + let denial = response_middleware_failure_denial(&failure).expect("explicit denial"); + assert_eq!(denial.config_name, "response"); + assert_eq!(denial.reason_code.as_deref(), Some("test_body_action")); +} + +#[tokio::test] +async fn chain_description_failure_is_middleware_failed_not_upstream_protocol() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let request_id = "response-describe-failure"; + let correlation = test_correlation(request_id); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::HeadersOnly as i32, + body_delay: None, + mutate_headers: false, + })); + let chain = (0..=openshell_supervisor_middleware::MAX_MIDDLEWARE_CHAIN_STAGES) + .map(|index| openshell_supervisor_middleware::ChainEntry { + name: format!("response-{index}"), + implementation: "test/streaming-response".into(), + order: i32::try_from(index).expect("bounded test index"), + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }) + .collect::>(); + let middleware = test_response_middleware(&runner, &chain, request_id); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("description failure gets a canonical 502"); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + drop(correlation); + + let event = response_audit_event_for(&capture, request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "middleware_failed" + ); + assert!( + event["unmapped"]["response_delivery_outcome"].is_null(), + "completion is not claimed by the primary record" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert_eq!(event["http_response"]["code"], 502); + assert!(capture.json_events().iter().any(|event| { + event["finding_info"]["uid"] == "openshell.middleware.http_response_failure" + && event["metadata"]["correlation_uid"] == request_id + })); +} + +#[tokio::test] +async fn policy_reload_wins_over_describe_preflight_and_finish_errors() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + + for (case_name, point, method, body_mode) in [ + ( + "describe", + ReloadThenErrorPoint::Describe, + "GET", + HttpResponseBodyMode::HeadersOnly, + ), + ( + "preflight", + ReloadThenErrorPoint::Preflight, + "GET", + HttpResponseBodyMode::HeadersOnly, + ), + ( + "bodiless-finish", + ReloadThenErrorPoint::Finish, + "HEAD", + HttpResponseBodyMode::WholeBodyBytes, + ), + ( + "normal-finish", + ReloadThenErrorPoint::Finish, + "GET", + HttpResponseBodyMode::WholeBodyBytes, + ), + ( + "normal-finish-streaming", + ReloadThenErrorPoint::Finish, + "GET", + HttpResponseBodyMode::StreamBytes, + ), + ] { + let engine = Arc::new( + crate::opa::OpaEngine::from_strings( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("test policy"), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .expect("generation guard"); + let request_id = format!("response-reload-then-error-{case_name}"); + let correlation = test_correlation_for_method(&request_id, method); + let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( + ReloadThenErrorResponseMiddleware { + point, + body_mode: body_mode as i32, + engine: Arc::clone(&engine), + }, + )); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/reload-then-error-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let mut middleware = test_response_middleware(&runner, &chain, &request_id); + middleware.generation_guard = Some(&generation_guard); + // Keep the middleware target body-capable in the HEAD case so the + // test creates a session and reaches the relay's bodiless finish. + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + method, + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("policy reload must override the middleware error"); + assert!( + error + .downcast_ref::() + .is_some_and(|error| matches!(error, super::RelayError::PolicyReloaded(_))), + "{case_name}: {error}" + ); + let committed = case_name == "normal-finish-streaming"; + if committed { + assert!( + client.starts_with(b"HTTP/1.1 200 OK\r\n"), + "{case_name}: expected the response head to be committed" + ); + let head_end = client + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + .expect("committed response head"); + assert_eq!( + &client[head_end..], + b"", + "{case_name}: stale response body reached the client" + ); + } else { + assert!( + client.is_empty(), + "{case_name}: stale middleware result reached the client" + ); + } + drop(correlation); + + let event = response_audit_event_for(&capture, &request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], "policy_reloaded", + "{case_name}" + ); + assert_eq!( + event["unmapped"]["response_delivery_outcome"], "cancelled_or_incomplete", + "{case_name}" + ); + assert_eq!( + event["unmapped"]["upstream_response_code"], 200, + "{case_name}" + ); + if committed { + assert_eq!(event["http_response"]["code"], 200, "{case_name}"); + assert_eq!(event["http_response"]["code"], 200, "{case_name}"); + } else { + assert!(event.get("http_response").is_none(), "{case_name}"); + assert!( + event["unmapped"] + .get("effective_http_status_code") + .is_none(), + "{case_name}" + ); + } + assert_eq!( + event["unmapped"]["delivery_started"], committed, + "{case_name}" + ); + assert_eq!( + event["unmapped"]["partial_delivery"], committed, + "{case_name}" + ); + assert!(!capture.json_events().iter().any(|event| { + event["finding_info"]["uid"] == "openshell.middleware.http_response_failure" + && event["metadata"]["correlation_uid"] == request_id + })); + if case_name == "preflight" { + assert!(event["unmapped"]["middleware_fail_closed"].is_null()); + } + } +} + +#[tokio::test] +async fn normalized_head_parse_failure_is_upstream_protocol_not_middleware_failure() { + use tracing_subscriber::layer::SubscriberExt as _; + + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(capture.clone()) + .with(openshell_otel::layer(&provider, "response-head-failure")); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let request_id = "response-head-parse-failure"; + let correlation = test_correlation(request_id); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::HeadersOnly as i32, + body_delay: None, + mutate_headers: false, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = test_response_middleware(&runner, &chain, request_id); + let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: Content-Length\r\n\r\n0\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("malformed normalized head gets a canonical 502"); + assert!(client.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + drop(correlation); + assert!( + !exporter + .get_finished_spans() + .unwrap() + .iter() + .any(|span| { span.name == "openshell.middleware.response" }), + "invalid upstream headers must not invent a cancelled middleware phase" + ); + + let event = response_audit_event_for(&capture, request_id); + assert_eq!( + event["unmapped"]["response_selection_cause"], + "upstream_protocol_error" + ); + assert!( + event["unmapped"]["response_delivery_outcome"].is_null(), + "completion is not claimed by the primary record" + ); + assert_eq!(event["unmapped"]["upstream_response_code"], 200); + assert_eq!(event["http_response"]["code"], 502); + assert!(!capture.json_events().iter().any(|event| { + event["finding_info"]["uid"] == "openshell.middleware.http_response_failure" + && event["metadata"]["correlation_uid"] == request_id + })); + let events = capture.json_events(); + assert!( + !events.iter().any(|event| { + event["firewall_rule"]["type"] == "middleware-correlation" + && event["metadata"]["correlation_uid"] == request_id + }), + "{events:#?}" + ); +} + +#[tokio::test] +async fn streaming_response_middleware_aggregates_header_and_body_mutations() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "response-correlation-test", + )); + let correlation = tracing::subscriber::with_default(subscriber, || { + test_correlation_for_method("response-streaming", "GET") + }); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::StreamBytes as i32, + body_delay: None, + mutate_headers: true, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "response-streaming".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let body = vec![b'a'; 9_000]; + let mut response = + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()).into_bytes(); + response.extend_from_slice(&body); + let mut upstream = response.as_slice(); + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("streaming middleware response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + assert!(String::from_utf8_lossy(&client).contains("x-inspected: true\r\n")); + assert!( + String::from_utf8_lossy(&client).contains("x-openshell-request-id: response-streaming\r\n") + ); + drop(correlation); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.middleware.response") + .expect("HTTP enforcement span"); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!(attributes["openshell.middleware.outcome"], "allowed"); + assert_eq!(attributes["openshell.middleware.mutated"], "true"); + assert_eq!(attributes["openshell.middleware.fail_open"], "false"); + assert!(!attributes.contains_key("openshell.request.id")); +} + +#[tokio::test] +async fn streaming_body_denial_records_partial_delivery() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer( + &provider, + "response-correlation-test", + )) + .with(ocsf.clone()); + let _subscriber = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let correlation = test_correlation_for_method("response-partial-denial", "GET"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BodyResponseMiddleware { + action: BodyTestAction::Block, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/body-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "response-partial-denial".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("streaming body denial follows response commitment"); + assert!(error.to_string().contains("stopped delivery"), "{error}"); + assert!(client.starts_with(b"HTTP/1.1 200 OK\r\n")); + drop(correlation); + provider.force_flush().expect("flush spans"); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.middleware.response") + .expect("HTTP enforcement span"); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!(attributes["openshell.middleware.outcome"], "denied"); + assert_eq!(attributes["openshell.response.delivery_started"], "true"); + assert_eq!(attributes["openshell.response.partial_delivery"], "true"); + assert!( + !ocsf + .json_events() + .iter() + .any(|event| event["firewall_rule"]["type"] == "middleware-correlation"), + "routine middleware aggregates are not separate audit records" + ); +} + +#[tokio::test] +async fn stale_generation_preserves_skip_transform_observation() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "response-correlation-test", + )); + let correlation = tracing::subscriber::with_default(subscriber, || { + test_correlation_for_method("response-stale-transform", "GET") + }); + let engine = Arc::new( + crate::opa::OpaEngine::from_strings( + include_str!("../../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .expect("test policy"), + ); + let generation_guard = engine + .generation_guard(engine.current_generation()) + .expect("generation guard"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BodyResponseMiddleware { + action: BodyTestAction::ReloadAndSkipTransform(Arc::clone(&engine)), + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/body-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "response-stale-transform".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: Some(&generation_guard), + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("policy reload must stop the stale response"); + assert!( + error.to_string().contains("policy generation is stale"), + "{error}" + ); + drop(correlation); + provider.force_flush().expect("flush spans"); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.middleware.response") + .expect("HTTP enforcement span"); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!(attributes["openshell.middleware.outcome"], "incomplete"); + assert_eq!(attributes["openshell.middleware.mutated"], "true"); + assert_eq!(attributes["openshell.response.delivery_started"], "true"); + assert_eq!(attributes["openshell.response.partial_delivery"], "true"); +} + +#[test] +fn response_middleware_observation_maps_bounded_outcomes() { + let invocation = |outcome| HttpResponseInvocation { + config_name: "not-exported".into(), + implementation: "not-exported".into(), + outcome, + sequence: None, + input_size: 17, + output_size: None, + transformed: outcome + == openshell_supervisor_middleware::HttpResponseInvocationOutcome::Transform, + failed: false, + stage_disabled: false, + reason_code: None, + failure_category: None, + }; + let mut invocations = vec![ + invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::Transform), + invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::FailOpen), + invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::FailClosed), + ]; + let mut skip_transform = + invocation(openshell_supervisor_middleware::HttpResponseInvocationOutcome::SkipRemaining); + skip_transform.transformed = true; + invocations.push(skip_transform); + let observation = response_middleware_observation(&invocations, false, true); + + assert_eq!( + observation, + MiddlewareObservation { + mutated: true, + fail_open: true, + fail_closed: true, + admission_exhausted: true, + ..Default::default() + } + ); +} + +#[test] +fn response_diagnostic_truncation_event_is_bounded() { + let event = http_response_diagnostic_truncation_event( + "test-policy", + &HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/private".into(), + query: "secret=value".into(), + }, + ) + .to_json() + .expect("serialize truncation event"); + + assert_eq!(event["severity"], "Low"); + assert_eq!( + event["finding_info"]["uid"], + "openshell.middleware.http_response_diagnostics_truncated" + ); + assert_eq!(event["unmapped"]["phase"], "pre_return"); + let encoded = serde_json::to_string(&event).expect("encode truncation event"); + assert!(!encoded.contains("/private")); + assert!(!encoded.contains("secret=value")); +} + +#[tokio::test] +async fn upstream_receipt_is_replaced_by_platform_receipt() { + let correlation = test_correlation("trusted"); + let mut upstream = + &b"HTTP/1.1 200 OK\r\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: trusted\r\n")); + assert_eq!(request_id_header_count(&client), 1); +} + +#[tokio::test] +async fn ordinary_response_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let mut upstream = &b"HTTP/1.1 200 OK\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id\r\n\ + X-OpenShell-Request-ID: forged-head\r\n\ + Trailer: x-openshell-request-id, X-Checksum\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 4\r\nbody\r\n\ + 0\r\n\ + x-OPENshell-request-ID: forged-trailer\r\n\ + X-Checksum: ok\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let head = delivered + .split_once("\r\n\r\n") + .expect("response head terminator") + .0; + let connection_headers: Vec<_> = head + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!(head.contains("Trailer: X-Checksum"), "{head}"); + assert!(delivered.contains("0\r\nX-Checksum: ok\r\n\r\n")); + assert!(head.contains("x-openshell-request-id: trusted"), "{head}"); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); +} + +#[tokio::test] +async fn ordinary_response_rejects_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("folded response fields must fail closed"); + + assert!( + error + .to_string() + .contains("response header continuation lines are not supported"), + "{error}" + ); + assert!(client.is_empty()); +} + +#[tokio::test] +async fn ordinary_response_rejects_bare_lf_receipt_smuggling() { + let correlation = test_correlation("trusted"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("bare LF response fields must fail closed"); + + assert!(error.to_string().contains("bare LF"), "{error}"); + assert!(client.is_empty()); +} + +#[test] +fn response_head_validation_rejects_malformed_syntax() { + let cases: &[(&str, &[u8])] = &[ + ("incomplete block", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n"), + ("bare LF", b"HTTP/1.1 200 OK\r\nX-Test: yes\n\r\n"), + ("bare CR", b"HTTP/1.1 200 OK\r\nX-Test: yes\rX: no\r\n\r\n"), + ( + "obsolete fold", + b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n continued\r\n\r\n", + ), + ("unsupported protocol", b"HTTP/2 200 OK\r\n\r\n"), + ("invalid status range", b"HTTP/1.1 099 Nope\r\n\r\n"), + ("non-numeric status", b"HTTP/1.1 2O0 Nope\r\n\r\n"), + ("four-digit status", b"HTTP/1.1 2000 Nope\r\n\r\n"), + ("missing reason separator", b"HTTP/1.1 200\r\n\r\n"), + ("malformed field", b"HTTP/1.1 200 OK\r\nX-Test\r\n\r\n"), + ("empty field name", b"HTTP/1.1 200 OK\r\n: value\r\n\r\n"), + ( + "non-token field name", + b"HTTP/1.1 200 OK\r\nBad Name: value\r\n\r\n", + ), + ( + "whitespace before colon", + b"HTTP/1.1 200 OK\r\nX-Test : value\r\n\r\n", + ), + ( + "field control byte", + b"HTTP/1.1 200 OK\r\nX-Test: value\x0bmore\r\n\r\n", + ), + ( + "field delete byte", + b"HTTP/1.1 200 OK\r\nX-Test: value\x7f\r\n\r\n", + ), + ("reason control byte", b"HTTP/1.1 200 O\x00K\r\n\r\n"), + ( + "early empty line", + b"HTTP/1.1 200 OK\r\n\r\nX-Smuggled: yes\r\n\r\n", + ), + ]; + + for (name, input) in cases { + assert!( + validate_http_response_header_block(input).is_err(), + "{name} unexpectedly passed validation" + ); + } +} + +#[test] +fn response_head_validation_accepts_supported_informational_and_upgrade_responses() { + for response in [ + &b"HTTP/1.0 200 OK\r\nX-Test: yes\r\n\r\n"[..], + &b"HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n"[..], + &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"[..], + ] { + validate_http_response_header_block(response).expect("valid HTTP/1.x response head"); + } +} + +#[tokio::test] +async fn response_middleware_rejects_forged_request_id_trailer() { + let correlation = test_correlation("trusted"); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(StreamingResponseMiddleware { + body_mode: HttpResponseBodyMode::StreamBytes as i32, + body_delay: None, + mutate_headers: true, + })); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "middleware-request".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("platform correlation trailers must fail closed"); + + assert!( + error.to_string().contains("protected field name"), + "{error}" + ); + let delivered = String::from_utf8_lossy(&client); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(&client), 1); +} + +#[test] +fn response_middleware_rejects_declared_request_id_trailer() { + let error = parse_response_head_for_middleware( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: X-OpenShell-Request-ID\r\n\r\n", + ) + .expect_err("platform correlation trailer declarations must fail closed"); + + assert!( + error.to_string().contains("protected trailer field"), + "{error}" + ); +} + +#[tokio::test] +async fn informational_responses_are_not_decorated_but_upgrade_receives_a_receipt() { + let correlation = test_correlation("receipt-1"); + let mut upstream = + &b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + assert!(client.starts_with(b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n")); + assert_eq!(request_id_header_count(&client), 1); + + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + assert_eq!(request_id_header_count(&client), 1); + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); +} + +#[tokio::test] +async fn switching_protocols_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id, UpGrAdE\r\n\ + Upgrade: websocket\r\n\ + X-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let connection_headers: Vec<_> = delivered + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive, UpGrAdE"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted-upgrade\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); +} + +#[tokio::test] +async fn switching_protocols_rejects_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade,\r\n\tX-OpenShell-Request-ID\r\nUpgrade: websocket\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("folded upgrade fields must fail closed"); + + assert!( + error + .to_string() + .contains("response header continuation lines are not supported"), + "{error}" + ); + assert!(client.is_empty()); +} + +#[test] +fn http_10_connection_persistence() { + let default_close = "HTTP/1.0 200 OK\r\nServer: test\r\n\r\n"; + assert!(response_is_http_10(default_close)); + assert!(!parse_connection_keep_alive(default_close)); + + let keep_alive = "HTTP/1.0 200 OK\r\nConnection: keep-alive\r\n\r\n"; + assert!(response_is_http_10(keep_alive)); + assert!(parse_connection_keep_alive(keep_alive)); +} + +#[test] +fn response_middleware_preflight_keeps_read_only_content_length() { + let parsed = parse_response_head_for_middleware( + b"HTTP/1.1 206 Partial Content\r\n\ + Content-Length: 5\r\n\ + Content-Encoding: gzip\r\n\ + Content-Range: bytes 0-4/10\r\n\ + Connection: x-hop\r\n\ + X-Hop: omitted\r\n\r\n", + ) + .expect("parse response head"); + + assert_eq!( + parsed.headers, + vec![ + HttpHeader { + name: "content-length".into(), + value: "5".into(), + }, + HttpHeader { + name: "content-encoding".into(), + value: "gzip".into(), + }, + HttpHeader { + name: "content-range".into(), + value: "bytes 0-4/10".into(), + }, + ] + ); + + let nominated = parse_response_head_for_middleware( + b"HTTP/1.1 200 OK\r\nConnection: Content-Length\r\nContent-Length: 5\r\n\r\n", + ) + .expect("parse nominated response head"); + assert!(nominated.headers.is_empty()); +} + +#[test] +fn response_middleware_serializes_only_relay_owned_framing() { + let headers = vec![ + HttpHeader { + name: "content-length".into(), + value: "999".into(), + }, + HttpHeader { + name: "Content-Length".into(), + value: "998".into(), + }, + HttpHeader { + name: "content-type".into(), + value: "text/plain".into(), + }, + ]; + + for (framing, expected) in [ + ( + ResponseFraming::Preserve(BodyLength::ContentLength(5)), + Some("Content-Length: 5\r\n"), + ), + ( + ResponseFraming::ContentLength(9), + Some("Content-Length: 9\r\n"), + ), + (ResponseFraming::Chunked, None), + (ResponseFraming::Preserve(BodyLength::None), None), + ] { + let serialized = String::from_utf8(serialize_response_head( + "HTTP/1.1 200 OK", + &headers, + &[], + framing, + false, + &[], + )) + .expect("serialized response head"); + + assert_eq!( + serialized + .lines() + .filter(|line| line.to_ascii_lowercase().starts_with("content-length:")) + .count(), + usize::from(expected.is_some()) + ); + if let Some(expected) = expected { + assert!(serialized.contains(expected)); + } + if matches!(framing, ResponseFraming::Chunked) { + assert!(serialized.contains("Transfer-Encoding: chunked\r\n")); + } + } +} diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index d153a01d25..ecbbb5acd3 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -94,9 +94,10 @@ the mechanism, placement and outcome without credential values. The primary OCSF record includes notable withholding, denial, unavailability or stale state. The primary HTTP Activity record describes the final response headers accepted -by the client connection. Response-body completion is recorded separately in the -trace. A successful header write does not establish that the caller received a complete -body or that the external service committed an operation. Correlation attributes +by the client connection. An `openshell.response.deliver` child records body +completion separately; a later delivery failure adds a terminal OCSF record with +the same request ID. A successful header write does not establish that the caller +received a complete body or that the external service committed an operation. Correlation attributes exclude credentials, query values, bodies and MCP arguments. ## Next Steps