diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 29b183dfd3..956bdd0df1 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -100,3 +100,9 @@ emits a bounded truncation finding. The middleware crate's Rust `ChainOutcome` now includes a typed termination reason. Rust consumers constructing that result must supply the new field; middleware protobuf messages and RPC signatures are unchanged. + +Credential preparation has its own child spans. They describe the mechanism, +placement and bounded result, never token material. The primary audit record +includes notable credential outcomes such as withholding or unavailable material; +successful preparation does not create another routine HTTP record. A completed +preparation span does not establish that the target received the request. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs index d723f8e738..c610d41a66 100644 --- a/crates/openshell-supervisor-network/src/l7/correlation.rs +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -3,6 +3,7 @@ //! Per-request correlation for HTTP enforcement events. +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Mutex; use std::{ future::Future, @@ -37,6 +38,186 @@ struct RequestState { outcome: Option<&'static str>, policy_span: Option, middleware: [Option; 2], + credential: CredentialState, + credential_span: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum CredentialMechanism { + StaticPlaceholder, + TokenGrant, + SigV4, +} + +impl CredentialMechanism { + const ALL: [Self; 3] = [Self::StaticPlaceholder, Self::TokenGrant, Self::SigV4]; + + fn as_str(self) -> &'static str { + match self { + Self::StaticPlaceholder => "static_placeholder", + Self::TokenGrant => "token_grant", + Self::SigV4 => "sigv4", + } + } + + fn span_outcome_field(self) -> &'static str { + match self { + Self::StaticPlaceholder => "openshell.credential.static_placeholder.outcome", + Self::TokenGrant => "openshell.credential.token_grant.outcome", + Self::SigV4 => "openshell.credential.sigv4.outcome", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum CredentialSurface { + Path, + Query, + Header, + Body, + Signature, +} + +impl CredentialSurface { + fn as_str(self) -> &'static str { + match self { + Self::Path => "path", + Self::Query => "query", + Self::Header => "header", + Self::Body => "body", + Self::Signature => "signature", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CredentialOutcome { + Prepared, + Withheld, + Denied, + Unavailable, + Stale, + Incomplete, +} + +impl CredentialOutcome { + fn as_str(self) -> &'static str { + match self { + Self::Prepared => "prepared", + Self::Withheld => "withheld", + Self::Denied => "denied", + Self::Unavailable => "unavailable", + Self::Stale => "stale", + Self::Incomplete => "incomplete", + } + } + + fn precedence(self) -> u8 { + match self { + Self::Incomplete => 0, + Self::Withheld => 1, + Self::Prepared => 2, + Self::Unavailable => 3, + Self::Stale => 4, + Self::Denied => 5, + } + } +} + +#[derive(Debug, Default)] +struct CredentialState { + selected: bool, + mechanisms: BTreeMap, + surfaces: BTreeSet, + outcome: Option, +} + +impl CredentialState { + fn observe(&mut self, mechanism: CredentialMechanism, surface: CredentialSurface) { + self.selected = true; + self.mechanisms + .entry(mechanism) + .or_insert(CredentialOutcome::Incomplete); + self.surfaces.insert(surface); + } + + fn complete(&mut self, outcome: CredentialOutcome) { + if !self.selected && outcome != CredentialOutcome::Stale { + return; + } + self.selected = true; + + merge_optional_credential_outcome(&mut self.outcome, outcome); + if outcome == CredentialOutcome::Stale { + for current in self.mechanisms.values_mut() { + merge_credential_outcome(current, outcome); + } + } + } + + fn complete_mechanism(&mut self, mechanism: CredentialMechanism, outcome: CredentialOutcome) { + let Some(current) = self.mechanisms.get_mut(&mechanism) else { + return; + }; + + merge_optional_credential_outcome(&mut self.outcome, outcome); + merge_credential_outcome(current, outcome); + } + + fn snapshot(&self) -> Option { + self.selected.then(|| { + let has_incomplete = self + .mechanisms + .values() + .any(|outcome| *outcome == CredentialOutcome::Incomplete); + let outcome = match self.outcome { + Some( + outcome @ (CredentialOutcome::Denied + | CredentialOutcome::Stale + | CredentialOutcome::Unavailable), + ) => outcome, + _ if has_incomplete => CredentialOutcome::Incomplete, + Some(outcome) => outcome, + None => CredentialOutcome::Incomplete, + }; + + CredentialSnapshot { + mechanisms: sorted_values(self.mechanisms.keys().map(|value| value.as_str())), + surfaces: sorted_values(self.surfaces.iter().map(|value| value.as_str())), + outcome, + mechanism_outcomes: self.mechanisms.clone(), + } + }) + } +} + +fn merge_credential_outcome(current: &mut CredentialOutcome, outcome: CredentialOutcome) { + if outcome.precedence() > current.precedence() { + *current = outcome; + } +} + +fn merge_optional_credential_outcome( + current: &mut Option, + outcome: CredentialOutcome, +) { + if current.is_none_or(|current| outcome.precedence() > current.precedence()) { + *current = Some(outcome); + } +} + +#[derive(Debug, PartialEq, Eq)] +struct CredentialSnapshot { + mechanisms: String, + surfaces: String, + outcome: CredentialOutcome, + mechanism_outcomes: BTreeMap, +} + +fn sorted_values(values: impl Iterator) -> String { + let mut values = values.collect::>(); + values.sort_unstable(); + values.join(",") } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -155,6 +336,7 @@ impl RequestCorrelation { "server.port" = ctx.port, "openshell.policy.name" = %ctx.policy_name, "openshell.target.reached" = false, + "openshell.request.delivery_started" = false, "openshell.policy.decision" = tracing::field::Empty, "openshell.enforcement.action" = tracing::field::Empty, "openshell.http.outcome" = tracing::field::Empty, @@ -178,6 +360,8 @@ impl RequestCorrelation { outcome: None, policy_span: None, middleware: [None, None], + credential: CredentialState::default(), + credential_span: None, })), } } @@ -268,6 +452,118 @@ impl RequestCorrelation { self.span.record("openshell.target.reached", true); } + pub(crate) fn record_request_delivery_started(&self) { + self.span.record("openshell.request.delivery_started", true); + self.finish_credential_span(); + } + + fn credential_span(&self, state: &mut RequestState) { + state.credential_span.get_or_insert_with(|| { + self.span + .with_subscriber(|(_, dispatch)| { + tracing::dispatcher::with_default(dispatch, || { + tracing::info_span!(parent: &self.span, "openshell.credential.prepare", + otel.status_code = tracing::field::Empty, + "openshell.credential.outcome" = tracing::field::Empty, + "openshell.credential.mechanisms" = tracing::field::Empty, + "openshell.credential.surfaces" = tracing::field::Empty, + "openshell.credential.static_placeholder.outcome" = tracing::field::Empty, + "openshell.credential.token_grant.outcome" = tracing::field::Empty, + "openshell.credential.sigv4.outcome" = tracing::field::Empty) + }) + }) + .unwrap_or_else(tracing::Span::none) + }); + } + + pub(crate) fn observe_credential( + &self, + mechanism: CredentialMechanism, + surface: CredentialSurface, + ) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.credential.observe(mechanism, surface); + self.credential_span(&mut state); + } + + pub(crate) fn has_credential_mechanism(&self, mechanism: CredentialMechanism) -> bool { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .credential + .mechanisms + .contains_key(&mechanism) + } + + pub(crate) fn complete_credential(&self, outcome: CredentialOutcome) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + state.credential.complete(outcome); + if state.credential.selected { + self.credential_span(&mut state); + } + } + + pub(crate) fn complete_credential_mechanism( + &self, + mechanism: CredentialMechanism, + outcome: CredentialOutcome, + ) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .credential + .complete_mechanism(mechanism, outcome); + } + + pub(crate) async fn within_credential(&self, future: F) -> F::Output { + let span = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .credential_span + .clone() + .unwrap_or_else(tracing::Span::none); + future.instrument(span).await + } + + fn finish_credential_span(&self) { + let (span, snapshot) = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(span) = state.credential_span.take() else { + return; + }; + (span, state.credential.snapshot()) + }; + if let Some(snapshot) = snapshot { + span.record("openshell.credential.outcome", snapshot.outcome.as_str()); + span.record( + "openshell.credential.mechanisms", + snapshot.mechanisms.as_str(), + ); + span.record("openshell.credential.surfaces", snapshot.surfaces.as_str()); + for mechanism in CredentialMechanism::ALL { + if let Some(outcome) = snapshot.mechanism_outcomes.get(&mechanism) { + span.record(mechanism.span_outcome_field(), outcome.as_str()); + } + } + if !matches!( + snapshot.outcome, + CredentialOutcome::Prepared | CredentialOutcome::Withheld + ) { + openshell_otel::mark_error(&span); + } + } + } + pub(crate) fn record_error_stage(&self, _stage: &str) { openshell_otel::mark_error(&self.span); } @@ -470,6 +766,7 @@ impl RequestCorrelation { } fn emit_primary(&self, incomplete: bool) { + self.finish_credential_span(); let (mut event, status, upstream_status, outcome) = { let mut state = self .state @@ -478,6 +775,16 @@ impl RequestCorrelation { let Some(primary) = state.primary.take() else { return; }; + let mut primary = primary; + if let Some(snapshot) = state.credential.snapshot() + && snapshot.outcome != CredentialOutcome::Prepared + && let OcsfEvent::HttpActivity(http) = &mut primary + { + http.base + .unmapped + .get_or_insert_with(|| serde_json::json!({}))["credential_outcome"] = + snapshot.outcome.as_str().into(); + } (primary, state.status, state.upstream_status, state.outcome) }; if let OcsfEvent::HttpActivity(http) = &mut event { @@ -526,6 +833,49 @@ pub fn emit_ocsf(correlation: Option<&RequestCorrelation>, event: OcsfEvent) { } } +#[cfg(test)] +pub fn test_request_attributes( + spans: &[opentelemetry_sdk::trace::SpanData], + request_id: Option<&str>, +) -> BTreeMap { + let root = spans + .iter() + .find(|span| { + span.name == "openshell.http.request" + && request_id.is_none_or(|id| { + span.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openshell.request.id" + && attribute.value.to_string() == id + }) + }) + }) + .expect("request root"); + assert!( + root.attributes + .iter() + .all(|attribute| !attribute.key.as_str().starts_with("openshell.credential.")), + "credential detail belongs on a child span" + ); + let mut attributes = root + .attributes + .iter() + .map(|attribute| (attribute.key.to_string(), attribute.value.to_string())) + .collect::>(); + for child in spans.iter().filter(|span| { + span.name == "openshell.credential.prepare" + && span.parent_span_id == root.span_context.span_id() + }) { + assert_eq!(child.span_context.trace_id(), root.span_context.trace_id()); + attributes.extend( + child + .attributes + .iter() + .map(|attribute| (attribute.key.to_string(), attribute.value.to_string())), + ); + } + attributes +} + pub fn emit_phase_events( correlation: Option<&RequestCorrelation>, phase: MiddlewarePhase, @@ -856,7 +1206,7 @@ mod tests { .attributes .iter() .map(|attribute| (attribute.key.as_str(), attribute.value.to_string())) - .collect::>(); + .collect::>(); assert_eq!(attributes["openshell.middleware.outcome"], "denied"); assert_eq!( attributes["openshell.middleware.diagnostics_truncated"], @@ -942,7 +1292,7 @@ mod tests { .attributes .iter() .map(|kv| (kv.key.as_str().to_string(), kv.value.to_string())) - .collect::>(); + .collect::>(); assert_eq!(attributes["openshell.policy.decision"], "deny"); assert_eq!(attributes["openshell.enforcement.action"], "block"); assert_eq!(attributes["openshell.http.outcome"], "denied"); @@ -1026,7 +1376,7 @@ mod tests { .attributes .into_iter() .map(|kv| (kv.key.as_str().to_string(), kv.value.to_string())) - .collect::>(); + .collect::>(); assert_eq!(attributes["openshell.http.outcome"], "incomplete"); assert!(!attributes.contains_key("openshell.error.stage")); } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 87d95c2d66..be3f8426e3 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -7,7 +7,9 @@ //! Parses each request within the tunnel, evaluates it against OPA policy, //! and either forwards or denies the request. -use crate::l7::correlation::RequestCorrelation; +use crate::l7::correlation::{ + CredentialMechanism, CredentialOutcome, CredentialSurface, RequestCorrelation, +}; use crate::l7::middleware::{ MiddlewareApplyResult, UninspectableTrafficGate, apply_middleware_chain_with_request_id, emit_middleware_uninspectable, middleware_network_input, uninspectable_traffic_gate, @@ -30,8 +32,6 @@ use openshell_ocsf::{ ActionId, ActivityId, DetectionFindingBuilder, DispositionId, Endpoint, FindingInfo, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, SeverityId, StatusId, Url as OcsfUrl, }; -#[cfg(test)] -use std::collections::BTreeMap; use std::collections::HashSet; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; @@ -165,6 +165,7 @@ fn credential_generation_guard( /// /// A refused grant returns a [`TokenGrantInjectionError`](crate::l7::token_grant_injection::TokenGrantInjectionError) /// inside the report, so callers can tell it apart from other failures. +#[allow(clippy::too_many_arguments)] pub(crate) async fn prepare_inspected_request( req: crate::l7::provider::L7Request, ctx: &L7EvalContext, @@ -173,6 +174,7 @@ pub(crate) async fn prepare_inspected_request( request_info: &L7RequestInfo, request_id: &str, observation_context: Option<&openshell_core::endpoint_status::EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result<(crate::l7::provider::L7Request, L7EvalContext)> { let mut scoped = ctx.clone(); let snapshot = crate::l7::token_grant_injection::pin_live_credentials(&mut scoped); @@ -224,6 +226,7 @@ pub(crate) async fn prepare_inspected_request( observation_context.filter(|context| { context.matches_config(&config.policy_hash, Some(snapshot.revision)) }), + correlation, ) .await? } else { @@ -232,8 +235,13 @@ pub(crate) async fn prepare_inspected_request( if engine.is_stale() { return Err(miette!("policy changed during token grant resolution")); } - if let Some(guard) = credential_generation_guard(&scoped) { - guard.ensure_current()?; + if let Some(guard) = credential_generation_guard(&scoped) + && let Err(error) = guard.ensure_current() + { + if let Some(correlation) = correlation { + correlation.complete_credential(CredentialOutcome::Stale); + } + return Err(error); } let scoped = scoped_context_for_request(&scoped, &req).unwrap_or(scoped); Ok((req, scoped)) @@ -752,6 +760,31 @@ where events.push(build_credential_endpoint_mismatch_finding(ctx)); } if let Some(correlation) = correlation { + let mechanism = if correlation.has_credential_mechanism(CredentialMechanism::SigV4) { + CredentialMechanism::SigV4 + } else { + CredentialMechanism::StaticPlaceholder + }; + let static_surface = match error.location { + "path" => Some(CredentialSurface::Path), + "query_param" => Some(CredentialSurface::Query), + "header" => Some(CredentialSurface::Header), + _ => None, + }; + if mechanism == CredentialMechanism::StaticPlaceholder + && !correlation.has_credential_mechanism(CredentialMechanism::StaticPlaceholder) + && let Some(surface) = static_surface + { + correlation.observe_credential(CredentialMechanism::StaticPlaceholder, surface); + } + correlation.complete_credential_mechanism( + mechanism, + if endpoint_mismatch || placement_mismatch { + CredentialOutcome::Denied + } else { + CredentialOutcome::Unavailable + }, + ); correlation.record_enforcement_action("block"); correlation.record_response_status(if endpoint_mismatch || placement_mismatch { 403 @@ -794,6 +827,14 @@ pub(crate) async fn reject_body_credential( "message": "A credential placeholder in the request body cannot be forwarded. Remove the reference from conversation history or restore provider access; body credential rewriting is disabled." }}); if let Some(correlation) = correlation { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Body, + ); + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Denied, + ); correlation.record_policy_decision("deny"); correlation.record_enforcement_action("block"); correlation.record_response_status(403); @@ -824,6 +865,10 @@ async fn reject_credential_unavailable( "message": "Credential material is unavailable for this request", }); if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); correlation.record_enforcement_action("block"); correlation.record_response_status(500); correlation.record_final_http_outcome("platform_error"); @@ -892,6 +937,7 @@ where request_info, request_id, observation_context, + correlation, ) .await { @@ -1044,6 +1090,7 @@ where } if error.generation_is_stale() { if let Some(correlation) = correlation { + correlation.complete_credential(CredentialOutcome::Stale); correlation.record_enforcement_action("block"); correlation.record_final_http_outcome("platform_error"); correlation.record_error_stage("credential_generation"); @@ -4130,8 +4177,15 @@ where return Ok(()); } }; + let mut correlation = (ctx.provider_credentials.is_some() || ctx.secret_resolver.is_some()) + .then(|| RequestCorrelation::new(&req, ctx)); + let fallback_request_id = uuid::Uuid::new_v4().to_string(); + if let Some(correlation) = correlation.as_ref() { + correlation.record_policy_decision("not_applicable"); + } if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action, None).await?; + reject_request_authority_mismatch(client, ctx, &req.action, correlation.as_ref()) + .await?; return Ok(()); } if close_if_stale(generation_guard, ctx) { @@ -4144,7 +4198,14 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error, None).await?; + reject_credential_resolution( + client, + ctx, + &req.action, + &error, + correlation.as_ref(), + ) + .await?; return Ok(()); } }; @@ -4171,8 +4232,6 @@ where egress_emit!(event); } - let fallback_request_id = uuid::Uuid::new_v4().to_string(); - let mut correlation = None; let mut response_selection = None; let req = if let Some(engine) = middleware_engine { let input = middleware_network_input(ctx); @@ -4180,10 +4239,10 @@ where if generation != generation_guard.captured_generation() { return Ok(()); } - if !chain.is_empty() { - let selected = RequestCorrelation::new(&req, ctx); - selected.record_policy_decision("not_applicable"); - correlation = Some(selected); + if !chain.is_empty() && correlation.is_none() { + let request = RequestCorrelation::new(&req, ctx); + request.record_policy_decision("not_applicable"); + correlation = Some(request); } let request_id = correlation .as_ref() @@ -4282,6 +4341,7 @@ where &snapshot, request_id, observation_context.as_ref(), + correlation.as_ref(), ) .await } @@ -4452,10 +4512,12 @@ mod tests { use openshell_core::token_grant_failure::{ TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, }; - use std::collections::HashMap as TestHashMap; + use std::collections::{BTreeMap, HashMap as TestHashMap}; use std::fmt::Write as _; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; fn relay_grant_failure() -> TokenGrantFailure { TokenGrantFailure::new( @@ -4464,6 +4526,219 @@ mod tests { ) } + async fn capture_correlated_credential_rejection( + mechanism: CredentialMechanism, + surface: CredentialSurface, + error_location: &'static str, + ) -> BTreeMap { + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "credential-rejection-correlation-test", + )); + let request = crate::l7::provider::L7Request { + action: "GET".into(), + target: "/credential".into(), + query_params: TestHashMap::new(), + raw_header: b"GET /credential HTTP/1.1\r\nHost: api.example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let context = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "credential-test".into(), + ..Default::default() + }; + + async { + tracing::callsite::rebuild_interest_cache(); + let correlation = RequestCorrelation::with_request_id( + &request, + &context, + "00000000-0000-4000-8000-000000000031".into(), + ); + correlation.observe_credential(mechanism, surface); + let mut client = tokio::io::sink(); + reject_credential_resolution( + &mut client, + &context, + &request.action, + &secrets::UnresolvedPlaceholderError::unavailable(error_location), + Some(&correlation), + ) + .await + .expect("write local credential rejection"); + } + .with_subscriber(subscriber) + .await; + provider.force_flush().expect("flush spans"); + + crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + None, + ) + } + + #[tokio::test] + async fn credential_rejection_preserves_authoritative_prescanned_surface() { + let attributes = capture_correlated_credential_rejection( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Path, + "header", + ) + .await; + + assert_eq!(attributes["openshell.credential.outcome"], "unavailable"); + assert_eq!( + attributes["openshell.credential.mechanisms"], + "static_placeholder" + ); + assert_eq!(attributes["openshell.credential.surfaces"], "path"); + assert_eq!( + attributes["openshell.credential.static_placeholder.outcome"], + "unavailable" + ); + } + + #[tokio::test] + async fn sigv4_resolution_rejection_is_not_attributed_to_static_placeholders() { + let attributes = capture_correlated_credential_rejection( + CredentialMechanism::SigV4, + CredentialSurface::Signature, + "sigv4", + ) + .await; + + assert_eq!(attributes["openshell.credential.outcome"], "unavailable"); + assert_eq!(attributes["openshell.credential.mechanisms"], "sigv4"); + assert_eq!(attributes["openshell.credential.surfaces"], "signature"); + assert_eq!( + attributes["openshell.credential.sigv4.outcome"], + "unavailable" + ); + assert!(!attributes.contains_key("openshell.credential.static_placeholder.outcome")); + } + + #[tokio::test] + async fn policy_only_staleness_does_not_emit_credential_state() { + const DATA: &str = r#" +network_policies: + rest_api: + name: rest_api + endpoints: + - host: api.example.test + port: 8080 + protocol: rest + enforcement: enforce + rules: + - allow: + method: GET + path: "/v1/**" + binaries: + - { path: /usr/bin/curl } +"#; + const REQUEST_ID: &str = "00000000-0000-4000-8000-000000000032"; + + let engine = OpaEngine::from_strings(TEST_POLICY, DATA).unwrap(); + let input = NetworkInput { + host: "api.example.test".into(), + port: 8080, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&endpoint_config.unwrap()).unwrap(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + engine.reload(TEST_POLICY, DATA).unwrap(); + + let request = crate::l7::provider::L7Request { + action: "GET".into(), + target: "/v1/data".into(), + query_params: TestHashMap::new(), + raw_header: b"GET /v1/data HTTP/1.1\r\nHost: api.example.test:8080\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let request_info = L7RequestInfo { + action: request.action.clone(), + target: request.target.clone(), + query_params: TestHashMap::new(), + graphql: None, + jsonrpc: None, + }; + let context = L7EvalContext { + host: "api.example.test".into(), + port: 8080, + request_default_port: Some(8080), + policy_name: "rest_api".into(), + binary_path: "/usr/bin/curl".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer( + &provider, + "policy-staleness-correlation-test", + )) + .with(ocsf.clone()); + + async { + tracing::callsite::rebuild_interest_cache(); + let correlation = + RequestCorrelation::with_request_id(&request, &context, REQUEST_ID.into()); + let Err(error) = prepare_inspected_request( + request, + &context, + &tunnel_engine, + &config, + &request_info, + REQUEST_ID, + None, + Some(&correlation), + ) + .await + else { + panic!("stale policy generation must reject the request"); + }; + assert!( + error + .to_string() + .contains("policy changed during token grant resolution") + ); + correlation.record_final_http_outcome("platform_error"); + } + .with_subscriber(subscriber) + .await; + provider.force_flush().expect("flush spans"); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.http.request") + .expect("HTTP enforcement span"); + assert!( + span.attributes + .iter() + .all(|attribute| !attribute.key.as_str().starts_with("openshell.credential.")) + ); + assert!( + ocsf.json_events() + .iter() + .all(|event| { event["firewall_rule"]["type"] != "credential-correlation" }) + ); + } + /// Assert the generic refusal and return its request ID header value. pub(super) fn grant_refusal_request_id(response: &str) -> String { assert!( @@ -5623,10 +5898,15 @@ mod tests { #[tokio::test] async fn implicit_sigv4_endpoint_mismatch_returns_typed_403() { + const ACCESS_KEY: &str = "AKIA_MISMATCH_SENTINEL"; + const SECRET_KEY: &str = "sigv4-mismatch-secret-sentinel"; + const SESSION_TOKEN: &str = "sigv4-mismatch-session-sentinel"; + const REGION: &str = "us-mismatch-1"; + const SERVICE: &str = "mismatch-service"; let (_state, resolver) = endpoint_mismatch_resolver(TestHashMap::from([ - ("AWS_ACCESS_KEY_ID".to_string(), "access".to_string()), - ("AWS_SECRET_ACCESS_KEY".to_string(), "secret".to_string()), - ("AWS_SESSION_TOKEN".to_string(), "session".to_string()), + ("AWS_ACCESS_KEY_ID".to_string(), ACCESS_KEY.to_string()), + ("AWS_SECRET_ACCESS_KEY".to_string(), SECRET_KEY.to_string()), + ("AWS_SESSION_TOKEN".to_string(), SESSION_TOKEN.to_string()), ])); let request = crate::l7::provider::L7Request { action: "GET".to_string(), @@ -5638,19 +5918,52 @@ mod tests { body_length: crate::l7::provider::BodyLength::ContentLength(0), }; - assert_credential_relay_rejected( - request, - resolver.as_ref(), - crate::l7::rest::RelayRequestOptions { - credential_signing: crate::l7::CredentialSigning::SigV4NoBody, - signing_service: "execute-api", - signing_region: "us-west-2", - host: "denied.example.test", - port: 443, - ..Default::default() - }, - ) + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "sigv4-endpoint-mismatch-correlation-test", + )); + + async { + tracing::callsite::rebuild_interest_cache(); + assert_credential_relay_rejected( + request, + resolver.as_ref(), + crate::l7::rest::RelayRequestOptions { + credential_signing: crate::l7::CredentialSigning::SigV4NoBody, + signing_service: SERVICE, + signing_region: REGION, + host: "denied.example.test", + port: 443, + ..Default::default() + }, + ) + .await; + } + .with_subscriber(subscriber) .await; + provider.force_flush().expect("flush spans"); + + let attributes = crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + None, + ); + assert_eq!(attributes["openshell.credential.outcome"], "denied"); + assert_eq!(attributes["openshell.credential.mechanisms"], "sigv4"); + assert_eq!(attributes["openshell.credential.surfaces"], "signature"); + assert_eq!(attributes["openshell.credential.sigv4.outcome"], "denied"); + assert_eq!(attributes["openshell.request.delivery_started"], "false"); + assert!(!attributes.contains_key("openshell.credential.static_placeholder.outcome")); + let serialized = format!("{attributes:?}"); + for private_value in [ACCESS_KEY, SECRET_KEY, SESSION_TOKEN, REGION, SERVICE] { + assert!( + !serialized.contains(private_value), + "leaked {private_value}" + ); + } } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index eb34e325b9..4f22f44ba5 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -1418,6 +1418,7 @@ async fn transformed_jsonrpc_body_selects_final_operation_owner() { &info, "request-final-body", None, + None, ), ) .await diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 299013dd1c..10d47e642b 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -21,7 +21,9 @@ use http_response::{ }; use crate::l7::EndpointObserver; -use crate::l7::correlation::RequestCorrelation; +use crate::l7::correlation::{ + CredentialMechanism, CredentialOutcome, CredentialSurface, RequestCorrelation, +}; use crate::l7::http::{parse_chunk_size_line, validate_chunk_trailer_line}; use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; @@ -35,7 +37,7 @@ use openshell_core::proto::{ }; use openshell_core::secrets::{ SecretResolver, contains_reserved_credential_marker, contains_reserved_credential_marker_bytes, - rewrite_http_header_block, + header_value_contains_reserved_credential_marker, rewrite_http_header_block, }; use openshell_ocsf::ctx::ctx as ocsf_ctx; use sha1::{Digest, Sha1}; @@ -902,13 +904,79 @@ impl<'a> CredentialGenerationGuard<'a> { } } -fn ensure_credential_generation_current(options: RelayRequestOptions<'_>) -> Result<()> { - if let Some(guard) = options.credential_generation { - guard.ensure_current()?; +fn ensure_credential_generation_current( + options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, +) -> Result<()> { + if let Some(guard) = options.credential_generation + && let Err(error) = guard.ensure_current() + { + if let Some(correlation) = correlation { + correlation.complete_credential(CredentialOutcome::Stale); + } + return Err(error); } Ok(()) } +pub(crate) fn observe_static_credential_surfaces( + raw_header: &[u8], + correlation: Option<&RequestCorrelation>, +) -> bool { + let Some(correlation) = correlation else { + return false; + }; + let Some(header_end) = raw_header + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) + else { + return false; + }; + let Ok(header_text) = std::str::from_utf8(&raw_header[..header_end]) else { + return false; + }; + let mut lines = header_text.split("\r\n"); + let mut selected = false; + + if let Some(target) = lines + .next() + .and_then(|request_line| request_line.split_whitespace().nth(1)) + { + let (path, query) = target + .split_once('?') + .map_or((target, None), |(path, query)| (path, Some(query))); + if contains_reserved_credential_marker(path) { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Path, + ); + selected = true; + } + if query.is_some_and(contains_reserved_credential_marker) { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Query, + ); + selected = true; + } + } + + for line in lines.take_while(|line| !line.is_empty()) { + let Some((_, value)) = line.split_once(':') else { + continue; + }; + if header_value_contains_reserved_credential_marker(value) { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Header, + ); + selected = true; + } + } + selected +} + pub(crate) async fn relay_http_request_with_options_guarded( req: &L7Request, client: &mut C, @@ -962,9 +1030,13 @@ where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { - let mut observed_upstream = ObservedUpstream { upstream, observer }; + let mut observed_upstream = ObservedUpstream { + upstream, + observer, + correlation, + }; let upstream = &mut observed_upstream; - ensure_credential_generation_current(options)?; + ensure_credential_generation_current(options, correlation)?; let header_end = req .raw_header .windows(4) @@ -997,6 +1069,7 @@ where websocket_request.is_some(), )?; let header_bytes = strip_connection_nominated_headers(&header_bytes, is_websocket_request)?; + let static_credential_selected = observe_static_credential_surfaces(&header_bytes, correlation); let websocket_response = websocket_request .as_ref() @@ -1008,6 +1081,12 @@ where let rewrite_result = rewrite_http_header_block(&header_bytes, options.resolver).map_err(miette::Report::new)?; + if static_credential_selected && let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Prepared, + ); + } if let Some(validation) = options.mcp_request_validation { // Header credential resolution and hop-by-hop cleanup must finish @@ -1043,9 +1122,19 @@ where // Apply SigV4 signing if configured. if options.credential_signing.is_sigv4() { + if let Some(correlation) = correlation { + correlation + .observe_credential(CredentialMechanism::SigV4, CredentialSurface::Signature); + } // Defense-in-depth: credential_signing and request_body_credential_rewrite // are mutually exclusive (validated at policy load time). if options.request_body_credential_rewrite { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } return Err(miette!( "credential_signing and request_body_credential_rewrite are \ mutually exclusive on the same endpoint" @@ -1073,6 +1162,7 @@ where .resolve_current_env_key_checked("AWS_SESSION_TOKEN", "sigv4") .map_err(miette::Report::new)?; + #[allow(clippy::single_match_else)] match (access_key, secret_key) { (Some(access_key), Some(secret_key)) => { // Use explicit signing_region from policy if set, @@ -1081,6 +1171,12 @@ where match crate::sigv4::extract_aws_region(options.host) { Some(r) => r, None => { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } return Err(miette!( "SigV4 signing: cannot extract AWS region from \ hostname '{host}'; set signing_region in the \ @@ -1094,6 +1190,12 @@ where }; let service = &options.signing_service; if service.is_empty() { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } return Err(miette!( "SigV4 signing configured but signing_service not set in policy" )); @@ -1104,7 +1206,20 @@ where crate::l7::CredentialSigning::SigV4NoBody => { SigV4PayloadMode::UnsignedPayload } - crate::l7::CredentialSigning::SigV4 => detect_payload_mode(header_str)?, + crate::l7::CredentialSigning::SigV4 => { + match detect_payload_mode(header_str) { + Ok(mode) => mode, + Err(error) => { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Denied, + ); + } + return Err(error); + } + } + } crate::l7::CredentialSigning::None => unreachable!(), }; @@ -1125,6 +1240,12 @@ where // detect_payload_mode's catch-all. } BodyLength::Chunked => { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Denied, + ); + } return Err(miette!( "SigV4 body signing requires Content-Length; \ chunked transfer encoding is not supported in this mode" @@ -1142,6 +1263,12 @@ where full_request.extend_from_slice(overflow); if let BodyLength::ContentLength(body_len) = body_length { if body_len > MAX_SIGV4_BODY_BYTES as u64 { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Denied, + ); + } return Err(miette!( "SigV4 body signing buffers at most {MAX_SIGV4_BODY_BYTES} bytes" )); @@ -1162,7 +1289,7 @@ where guard.ensure_current()?; } - let signed = crate::sigv4::apply_sigv4_to_request( + let signed = match crate::sigv4::apply_sigv4_to_request( &full_request, options.host, ®ion, @@ -1170,8 +1297,25 @@ where access_key, secret_key, session_token, - )?; - ensure_credential_generation_current(options)?; + ) { + Ok(signed) => signed, + Err(error) => { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } + return Err(error); + } + }; + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Prepared, + ); + } + ensure_credential_generation_current(options, correlation)?; upstream.write_all(&signed).await.into_diagnostic()?; } else { // Sign headers only, stream body through. @@ -1181,7 +1325,7 @@ where } _ => SignableBody::UnsignedPayload, }; - let signed_headers = crate::sigv4::apply_sigv4_headers_only_with_body( + let signed_headers = match crate::sigv4::apply_sigv4_headers_only_with_body( &rewrite_result.rewritten, options.host, ®ion, @@ -1190,8 +1334,25 @@ where secret_key, session_token, signable_body, - )?; - ensure_credential_generation_current(options)?; + ) { + Ok(signed) => signed, + Err(error) => { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } + return Err(error); + } + }; + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Prepared, + ); + } + ensure_credential_generation_current(options, correlation)?; upstream .write_all(&signed_headers) .await @@ -1201,7 +1362,8 @@ where match req.body_length { BodyLength::ContentLength(len) => { if !overflow.is_empty() { - write_body_bytes(upstream, overflow, options).await?; + write_body_bytes(upstream, overflow, options, correlation) + .await?; } let overflow_len = overflow.len() as u64; let remaining = len.saturating_sub(overflow_len); @@ -1221,7 +1383,8 @@ where } BodyLength::None => { if !overflow.is_empty() { - write_body_bytes(upstream, overflow, options).await?; + write_body_bytes(upstream, overflow, options, correlation) + .await?; } } } @@ -1249,12 +1412,24 @@ where crate::telemetry::egress_emit!(event); } _ => { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } return Err(miette::Report::new(CredentialUnavailableError::new( "SigV4 signing configured but AWS credentials not found in provider", ))); } } } else { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::SigV4, + CredentialOutcome::Unavailable, + ); + } return Err(miette::Report::new(CredentialUnavailableError::new( "SigV4 signing configured but no secret resolver available", ))); @@ -1266,11 +1441,11 @@ where &rewrite_result.rewritten, header_str, &req.raw_header[header_end..], - options.resolver, - options.generation_guard, + options, + correlation, ) .await?; - ensure_credential_generation_current(options)?; + ensure_credential_generation_current(options, correlation)?; upstream.write_all(&body.headers).await.into_diagnostic()?; if !body.body.is_empty() { upstream.write_all(&body.body).await.into_diagnostic()?; @@ -1283,16 +1458,27 @@ where &rewrite_result.rewritten, &req.raw_header[header_end..], options, + correlation, ) .await { if let Some(reason) = error.downcast_ref::() { + if let Some(correlation) = correlation { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Body, + ); + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Denied, + ); + } emit_uninspected_body_credential_denial(req, &options, *reason); } return Err(error); } } else { - ensure_credential_generation_current(options)?; + ensure_credential_generation_current(options, correlation)?; upstream .write_all(&rewrite_result.rewritten) .await @@ -1302,7 +1488,7 @@ where match req.body_length { BodyLength::ContentLength(len) => { if !overflow.is_empty() { - write_body_bytes(upstream, overflow, options).await?; + write_body_bytes(upstream, overflow, options, correlation).await?; } let overflow_len = overflow.len() as u64; let remaining = len.saturating_sub(overflow_len); @@ -1315,7 +1501,7 @@ where } BodyLength::None => { if !overflow.is_empty() { - write_body_bytes(upstream, overflow, options).await?; + write_body_bytes(upstream, overflow, options, correlation).await?; } } } @@ -1344,8 +1530,11 @@ use openshell_core::secrets::body::{ BodyCredentialError, BodyPlaceholderGuard as ReservedMarkerStreamGuard, }; -fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<()> { - ensure_credential_generation_current(options)?; +fn ensure_body_generation_current( + options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, +) -> Result<()> { + ensure_credential_generation_current(options, correlation)?; if let Some(guard) = options.generation_guard { guard.ensure_current()?; } @@ -1359,6 +1548,7 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() struct ObservedUpstream<'a, U> { upstream: &'a mut U, observer: Option<&'a EndpointObserver>, + correlation: Option<&'a RequestCorrelation>, } impl AsyncRead for ObservedUpstream<'_, U> { @@ -1386,10 +1576,18 @@ impl AsyncWrite for ObservedUpstream<'_, U> { ) -> std::task::Poll> { let this = self.get_mut(); let result = std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf); - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer - { - observer.observe(EndpointResult::TransportFailed); + match &result { + std::task::Poll::Ready(Ok(written)) if *written > 0 => { + if let Some(correlation) = this.correlation { + correlation.record_request_delivery_started(); + } + } + std::task::Poll::Ready(Err(_)) => { + if let Some(observer) = this.observer { + observer.observe(EndpointResult::TransportFailed); + } + } + _ => {} } result } @@ -1421,8 +1619,9 @@ async fn write_body_bytes( writer: &mut W, bytes: &[u8], options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { - ensure_body_generation_current(options)?; + ensure_body_generation_current(options, correlation)?; writer.write_all(bytes).await.into_diagnostic() } @@ -1430,6 +1629,7 @@ async fn write_guarded_chunk( writer: &mut W, payload: &[u8], options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { if payload.is_empty() { return Ok(()); @@ -1438,10 +1638,11 @@ async fn write_guarded_chunk( writer, format!("{:X}\r\n", payload.len()).as_bytes(), options, + correlation, ) .await?; - write_body_bytes(writer, payload, options).await?; - write_body_bytes(writer, b"\r\n", options).await + write_body_bytes(writer, payload, options, correlation).await?; + write_body_bytes(writer, b"\r\n", options, correlation).await } async fn relay_request_body_with_marker_guard( @@ -1451,19 +1652,20 @@ async fn relay_request_body_with_marker_guard( headers: &[u8], already_read: &[u8], options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where C: AsyncRead + Unpin, U: AsyncWrite + Unpin, { - ensure_body_generation_current(options)?; + ensure_body_generation_current(options, correlation)?; upstream.write_all(headers).await.into_diagnostic()?; match req.body_length { BodyLength::None => { let mut scanner = ReservedMarkerStreamGuard::new(options.body_classifier); let safe = scanner.push(already_read)?; - write_body_bytes(upstream, &safe, options).await?; - write_body_bytes(upstream, &scanner.finish()?, options).await?; + write_body_bytes(upstream, &safe, options, correlation).await?; + write_body_bytes(upstream, &scanner.finish()?, options, correlation).await?; } BodyLength::ContentLength(len) => { let initial_len = usize::try_from(len) @@ -1471,7 +1673,7 @@ where .min(already_read.len()); let mut scanner = ReservedMarkerStreamGuard::new(options.body_classifier); let safe = scanner.push(&already_read[..initial_len])?; - write_body_bytes(upstream, &safe, options).await?; + write_body_bytes(upstream, &safe, options, correlation).await?; let mut remaining = len.saturating_sub(initial_len as u64); let mut buf = vec![0u8; RELAY_BUF_SIZE]; while remaining > 0 { @@ -1484,15 +1686,16 @@ where "Connection closed with {remaining} body bytes remaining" )); } - ensure_body_generation_current(options)?; + ensure_body_generation_current(options, correlation)?; let safe = scanner.push(&buf[..n])?; - write_body_bytes(upstream, &safe, options).await?; + write_body_bytes(upstream, &safe, options, correlation).await?; remaining -= n as u64; } - write_body_bytes(upstream, &scanner.finish()?, options).await?; + write_body_bytes(upstream, &scanner.finish()?, options, correlation).await?; } BodyLength::Chunked => { - relay_chunked_with_marker_guard(client, upstream, already_read, options).await?; + relay_chunked_with_marker_guard(client, upstream, already_read, options, correlation) + .await?; } } Ok(()) @@ -1503,6 +1706,7 @@ async fn relay_chunked_with_marker_guard( upstream: &mut U, already_read: &[u8], options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where C: AsyncRead + Unpin, @@ -1523,8 +1727,8 @@ where let chunk_size = parse_chunk_size_line(&size_line)?; if chunk_size == 0 { - write_guarded_chunk(upstream, &scanner.finish()?, options).await?; - write_body_bytes(upstream, b"0\r\n", options).await?; + write_guarded_chunk(upstream, &scanner.finish()?, options, correlation).await?; + write_body_bytes(upstream, b"0\r\n", options, correlation).await?; loop { let trailer = read_chunked_line(client, already_read, &mut read_state, generation_guard) @@ -1535,15 +1739,15 @@ where return Err(BodyCredentialError::Trailer.into()); } if trailer.is_empty() { - write_body_bytes(upstream, b"\r\n", options).await?; + write_body_bytes(upstream, b"\r\n", options, correlation).await?; return Ok(()); } if !http_field_line_has_name( &trailer, openshell_core::token_grant_failure::REQUEST_ID_HEADER, ) { - write_body_bytes(upstream, &trailer, options).await?; - write_body_bytes(upstream, b"\r\n", options).await?; + write_body_bytes(upstream, &trailer, options, correlation).await?; + write_body_bytes(upstream, b"\r\n", options, correlation).await?; } } } @@ -1562,7 +1766,7 @@ where ) .await .map_err(CollectChunkedError::into_report)?; - write_guarded_chunk(upstream, &scanner.push(&block)?, options).await?; + write_guarded_chunk(upstream, &scanner.push(&block)?, options, correlation).await?; remaining -= block_len; } @@ -1832,12 +2036,24 @@ async fn collect_and_rewrite_request_body( rewritten_headers: &[u8], original_header_str: &str, already_read: &[u8], - resolver: Option<&SecretResolver>, - generation_guard: Option<&PolicyGenerationGuard>, + options: RelayRequestOptions<'_>, + correlation: Option<&RequestCorrelation>, ) -> Result { + let resolver = options.resolver; + let generation_guard = options.generation_guard; match req.body_length { BodyLength::None => { if body_bytes_contain_reserved_marker(already_read) { + if let Some(correlation) = correlation { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Body, + ); + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Denied, + ); + } return Err(miette!( "request body credential rewrite cannot resolve placeholders without explicit body framing" )); @@ -1874,8 +2090,13 @@ async fn collect_and_rewrite_request_body( body.extend_from_slice(&buf[..n]); remaining -= n; } - let (headers, body) = - rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; + let (headers, body) = rewrite_buffered_body( + rewritten_headers, + original_header_str, + body, + resolver, + correlation, + )?; Ok(PreparedRequestBody { headers, body }) } BodyLength::Chunked => { @@ -1887,8 +2108,13 @@ async fn collect_and_rewrite_request_body( ) .await .map_err(CollectChunkedError::into_report)?; - let (mut headers, body) = - rewrite_buffered_body(rewritten_headers, original_header_str, body, resolver)?; + let (mut headers, body) = rewrite_buffered_body( + rewritten_headers, + original_header_str, + body, + resolver, + correlation, + )?; headers = set_content_length(&headers, body.len())?; headers = strip_header(&headers, "transfer-encoding")?; headers = strip_header(&headers, "trailer")?; @@ -1902,14 +2128,28 @@ fn rewrite_buffered_body( original_header_str: &str, body: Vec, resolver: Option<&SecretResolver>, + correlation: Option<&RequestCorrelation>, ) -> Result<(Vec, Vec)> { if body.is_empty() { return Ok((headers.to_vec(), body)); } + let static_credential_selected = body_bytes_contain_reserved_marker(&body); + if static_credential_selected && let Some(correlation) = correlation { + correlation.observe_credential( + CredentialMechanism::StaticPlaceholder, + CredentialSurface::Body, + ); + } let content_type = content_type(original_header_str); if !is_rewritable_content_type(content_type.as_deref()) { - if body_bytes_contain_reserved_marker(&body) { + if static_credential_selected { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Denied, + ); + } return Err(miette!( "request body credential rewrite found placeholders in an unsupported content type" )); @@ -1917,33 +2157,82 @@ fn rewrite_buffered_body( return Ok((headers.to_vec(), body)); } - let mut text = String::from_utf8(body) - .map_err(|_| miette!("request body credential rewrite requires UTF-8 text bodies"))?; + let Ok(mut text) = String::from_utf8(body) else { + if static_credential_selected && let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Denied, + ); + } + return Err(miette!( + "request body credential rewrite requires UTF-8 text bodies" + )); + }; if !contains_reserved_credential_marker(&text) { return Ok((headers.to_vec(), text.into_bytes())); } let Some(resolver) = resolver else { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Unavailable, + ); + } return Err(miette!( "request body credential rewrite found placeholders but no resolver is available" )); }; - let replacements = if content_type.as_deref() == Some("application/x-www-form-urlencoded") { - let (rewritten, replacements) = rewrite_form_urlencoded_body(&text, resolver)?; - text = rewritten; - replacements - } else { - resolver - .rewrite_text_placeholders(&mut text, "request_body") - .map_err(miette::Report::new)? + let rewrite_result: Result = + if content_type.as_deref() == Some("application/x-www-form-urlencoded") { + rewrite_form_urlencoded_body(&text, resolver).map(|(rewritten, replacements)| { + text = rewritten; + replacements + }) + } else { + resolver + .rewrite_text_placeholders(&mut text, "request_body") + .map_err(miette::Report::new) + }; + let replacements = match rewrite_result { + Ok(replacements) => replacements, + Err(error) => { + if let Some(correlation) = correlation { + let outcome = if error + .downcast_ref::() + .is_some_and( + openshell_core::secrets::UnresolvedPlaceholderError::is_endpoint_mismatch, + ) { + CredentialOutcome::Denied + } else { + CredentialOutcome::Unavailable + }; + correlation + .complete_credential_mechanism(CredentialMechanism::StaticPlaceholder, outcome); + } + return Err(error); + } }; if replacements == 0 || contains_reserved_credential_marker(&text) { + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Unavailable, + ); + } return Err(miette!( "request body credential rewrite left unresolved credential placeholders" )); } + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Prepared, + ); + } + let body = text.into_bytes(); let headers = set_content_length(headers, body.len())?; Ok((headers, body)) @@ -4033,6 +4322,40 @@ mod tests { captured_preflight_headers: Option>>>, } + #[derive(Clone, Copy)] + enum TestWriteResult { + Success, + Zero, + Failure, + } + + struct TestWriter(TestWriteResult); + + impl AsyncWrite for TestWriter { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match self.0 { + TestWriteResult::Success => Poll::Ready(Ok(buf.len())), + TestWriteResult::Zero => Poll::Ready(Ok(0)), + TestWriteResult::Failure => Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "test write failure", + ))), + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + #[derive(Clone)] struct ResponseBodyGate { entered: Arc, @@ -5284,6 +5607,7 @@ mod tests { &mut writer, &[], RelayRequestOptions::default(), + None, ) .await .expect("guarded chunked body should relay"); @@ -5753,7 +6077,7 @@ mod tests { headers, std::str::from_utf8(headers).expect("headers"), &wire, - None, + RelayRequestOptions::default(), None, ) .await; @@ -5785,7 +6109,7 @@ mod tests { headers, std::str::from_utf8(headers).expect("headers"), b"1\r\nx\r\n0\r\nDigest: sha-256=:abc123:\r\n\r\n", - None, + RelayRequestOptions::default(), None, ) .await; @@ -6753,6 +7077,86 @@ mod tests { assert!(receiver.try_recv().is_err(), "one result per exchange"); } + fn capture_request_delivery_started(result: TestWriteResult) -> String { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "request-delivery-correlation-test", + )); + let request = L7Request { + action: "POST".into(), + target: "/v1/action".into(), + query_params: HashMap::new(), + raw_header: + b"POST /v1/action HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 1\r\n\r\n" + .to_vec(), + body_length: BodyLength::ContentLength(1), + }; + let context = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "delivery-test".into(), + ..Default::default() + }; + + tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + let correlation = RequestCorrelation::with_request_id( + &request, + &context, + "00000000-0000-4000-8000-000000000013".into(), + ); + let mut writer = TestWriter(result); + let mut upstream = ObservedUpstream { + upstream: &mut writer, + observer: None, + correlation: Some(&correlation), + }; + let mut context = Context::from_waker(std::task::Waker::noop()); + let _ = Pin::new(&mut upstream).poll_write(&mut context, b"x"); + correlation.record_final_http_outcome("allowed"); + }); + provider.force_flush().expect("flush spans"); + + let attributes = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.http.request") + .expect("HTTP enforcement span") + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + attributes["openshell.request.delivery_started"].clone() + } + + #[test] + fn request_delivery_starts_only_after_a_nonzero_upstream_write() { + assert_eq!( + capture_request_delivery_started(TestWriteResult::Success), + "true" + ); + assert_eq!( + capture_request_delivery_started(TestWriteResult::Zero), + "false" + ); + assert_eq!( + capture_request_delivery_started(TestWriteResult::Failure), + "false" + ); + } + #[tokio::test] async fn endpoint_observation_preserves_response_result_after_body_read_error() { struct FailedReader; @@ -6785,6 +7189,7 @@ mod tests { let mut upstream = ObservedUpstream { upstream: &mut response, observer: Some(&observer), + correlation: None, }; let mut client = Vec::new(); relay_response( @@ -10331,6 +10736,158 @@ mod tests { (child_env, resolver) } + #[test] + fn static_credential_surface_correlation_is_bounded_and_omits_unrelated_requests() { + use tracing_subscriber::layer::SubscriberExt as _; + + const MARKER: &str = "openshell:resolve:env:SENTINEL_KEY"; + let basic = base64::engine::general_purpose::STANDARD.encode(format!("user:{MARKER}")); + let selected = L7Request { + action: "GET".into(), + target: format!("/{MARKER}?token={MARKER}"), + query_params: HashMap::new(), + raw_header: format!( + "GET /{MARKER}?token={MARKER} HTTP/1.1\r\nHost: api.example.test\r\nX-Api-Key: {MARKER}\r\nAuthorization: Basic {basic}\r\n\r\n" + ) + .into_bytes(), + body_length: BodyLength::None, + }; + let unrelated = L7Request { + action: "GET".into(), + target: "/health".into(), + query_params: HashMap::new(), + raw_header: b"GET /health HTTP/1.1\r\nHost: api.example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + let ctx = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "credential-test".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "static-credential-correlation-test", + )); + + tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + let correlation = RequestCorrelation::with_request_id( + &selected, + &ctx, + "00000000-0000-4000-8000-000000000011".into(), + ); + assert!(observe_static_credential_surfaces( + &selected.raw_header, + Some(&correlation) + )); + correlation.complete_credential_mechanism( + CredentialMechanism::StaticPlaceholder, + CredentialOutcome::Prepared, + ); + correlation.record_final_http_outcome("allowed"); + + let correlation = RequestCorrelation::with_request_id( + &unrelated, + &ctx, + "00000000-0000-4000-8000-000000000012".into(), + ); + assert!(!observe_static_credential_surfaces( + &unrelated.raw_header, + Some(&correlation) + )); + correlation.record_final_http_outcome("allowed"); + }); + provider.force_flush().expect("flush spans"); + + let spans = exporter.get_finished_spans().expect("finished spans"); + let attributes_for = |request_id: &str| { + crate::l7::correlation::test_request_attributes(&spans, Some(request_id)) + }; + let selected = attributes_for("00000000-0000-4000-8000-000000000011"); + assert_eq!(selected["openshell.credential.outcome"], "prepared"); + assert_eq!( + selected["openshell.credential.mechanisms"], + "static_placeholder" + ); + assert_eq!( + selected["openshell.credential.surfaces"], + "header,path,query" + ); + assert!(!format!("{selected:?}").contains("SENTINEL_KEY")); + + let unrelated = attributes_for("00000000-0000-4000-8000-000000000012"); + assert!(!unrelated.contains_key("openshell.credential.outcome")); + assert!(!unrelated.contains_key("openshell.credential.mechanisms")); + assert!(!unrelated.contains_key("openshell.credential.surfaces")); + } + + #[test] + fn invalid_utf8_body_with_placeholder_is_correlated_as_denied() { + use tracing_subscriber::layer::SubscriberExt as _; + + let request = L7Request { + action: "POST".into(), + target: "/v1/action".into(), + query_params: HashMap::new(), + raw_header: b"POST /v1/action HTTP/1.1\r\nHost: api.example.test\r\nContent-Type: application/json\r\nContent-Length: 45\r\n\r\n" + .to_vec(), + body_length: BodyLength::ContentLength(45), + }; + let context = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "credential-test".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "body-credential-correlation-test", + )); + + tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + let correlation = RequestCorrelation::with_request_id( + &request, + &context, + "00000000-0000-4000-8000-000000000014".into(), + ); + let mut body = b"{\"key\":\"openshell:resolve:env:SENTINEL_KEY\"".to_vec(); + body.push(0xff); + let error = rewrite_buffered_body( + &request.raw_header, + std::str::from_utf8(&request.raw_header).expect("valid test headers"), + body, + None, + Some(&correlation), + ) + .expect_err("invalid UTF-8 body must fail closed"); + assert!(error.to_string().contains("UTF-8")); + correlation.record_final_http_outcome("denied"); + }); + provider.force_flush().expect("flush spans"); + + let attributes = crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + None, + ); + assert_eq!(attributes["openshell.credential.outcome"], "denied"); + assert_eq!( + attributes["openshell.credential.mechanisms"], + "static_placeholder" + ); + assert_eq!(attributes["openshell.credential.surfaces"], "body"); + assert!(!format!("{attributes:?}").contains("SENTINEL_KEY")); + } + /// Helper: run a request through the relay and capture what upstream receives. async fn relay_and_capture( raw_header: Vec, @@ -10592,6 +11149,7 @@ mod tests { )), ..Default::default() }, + None, ) .await .unwrap(); @@ -10637,7 +11195,7 @@ mod tests { state.revoke_static_provider_environment(2); let mut output = Vec::new(); assert!( - write_body_bytes(&mut output, b"must not forward", options) + write_body_bytes(&mut output, b"must not forward", options, None) .await .is_err() ); @@ -10728,6 +11286,99 @@ mod tests { assert!(!contains_reserved_credential_marker_bytes(&forwarded)); } + #[tokio::test] + async fn late_body_credential_denial_preserves_started_delivery_correlation() { + use tracing_subscriber::layer::SubscriberExt as _; + + const MARKER: &str = "openshell:resolve:env:LATE_SECRET_SENTINEL"; + let prefix = vec![b'a'; 256]; + let body_len = prefix.len() + MARKER.len(); + let mut raw_header = format!( + "POST /api HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: {body_len}\r\n\r\n" + ) + .into_bytes(); + raw_header.extend_from_slice(&prefix); + let request = L7Request { + action: "POST".into(), + target: "/api".into(), + query_params: HashMap::new(), + raw_header, + body_length: BodyLength::ContentLength(body_len as u64), + }; + let context = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "credential-test".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "late-body-credential-correlation-test", + )); + let correlation = tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + RequestCorrelation::with_request_id( + &request, + &context, + "00000000-0000-4000-8000-000000000015".into(), + ) + }); + + let (mut client_writer, mut proxy_client) = tokio::io::duplex(1024); + client_writer.write_all(MARKER.as_bytes()).await.unwrap(); + drop(client_writer); + let (mut proxy_upstream, mut upstream_reader) = tokio::io::duplex(4096); + + let error = relay_http_request_with_response_middleware_guarded_observed_correlated( + &request, + &mut proxy_client, + &mut proxy_upstream, + RelayRequestOptions { + deny_uninspected_credentials: true, + host: "api.example.test", + port: 443, + ..Default::default() + }, + None, + None, + Some(&correlation), + ) + .await + .expect_err("late body credential marker must fail closed"); + assert!(error.to_string().contains("rewrite is disabled")); + correlation.record_final_http_outcome("denied"); + + drop(proxy_upstream); + let mut forwarded = Vec::new(); + upstream_reader.read_to_end(&mut forwarded).await.unwrap(); + assert!( + forwarded.windows(32).any(|window| window == &prefix[..32]), + "safe request-body bytes should reach upstream before the late denial" + ); + assert!(!contains_reserved_credential_marker_bytes(&forwarded)); + assert!(!String::from_utf8_lossy(&forwarded).contains("LATE_SECRET_SENTINEL")); + + drop(correlation); + provider.force_flush().expect("flush spans"); + let attributes = crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + None, + ); + + assert_eq!(attributes["openshell.credential.outcome"], "denied"); + assert_eq!( + attributes["openshell.credential.static_placeholder.outcome"], + "denied" + ); + assert_eq!(attributes["openshell.credential.surfaces"], "body"); + assert_eq!(attributes["openshell.request.delivery_started"], "true"); + assert!(!format!("{attributes:?}").contains("LATE_SECRET_SENTINEL")); + } + #[tokio::test] async fn guarded_chunked_body_detects_encoded_placeholder() { let wire = b"8\r\nprefix-o\r\n1D\r\npenshell%3Aresolve%3Aenv%3AKEY\r\n0\r\n\r\n"; @@ -10737,6 +11388,7 @@ mod tests { &mut upstream_writer, wire, RelayRequestOptions::default(), + None, ) .await .expect_err("encoded placeholder must fail closed"); @@ -10757,6 +11409,7 @@ mod tests { &mut output, wire, RelayRequestOptions::default(), + None, ) .await .expect_err("ambiguous trailer framing must fail closed"); @@ -11198,6 +11851,209 @@ mod tests { ); } + #[tokio::test] + async fn sigv4_success_correlates_preparation_and_upstream_delivery() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + const REQUEST_ID: &str = "00000000-0000-4000-8000-000000000041"; + const ACCESS_KEY: &str = "AKIA_CORRELATION_SENTINEL"; + const SECRET_KEY: &str = "sigv4-correlation-secret-sentinel"; + const REGION: &str = "us-correlation-1"; + const SERVICE: &str = "correlation-service"; + + let (_, resolver) = SecretResolver::from_provider_env( + [ + ("AWS_ACCESS_KEY_ID".to_string(), ACCESS_KEY.to_string()), + ("AWS_SECRET_ACCESS_KEY".to_string(), SECRET_KEY.to_string()), + ] + .into_iter() + .collect(), + ); + let resolver = resolver.expect("resolver"); + let request = L7Request { + action: "GET".into(), + target: "/resource".into(), + query_params: HashMap::new(), + raw_header: + b"GET /resource HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 0\r\n\r\n" + .to_vec(), + body_length: BodyLength::ContentLength(0), + }; + let context = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "sigv4-policy".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "sigv4-credential-correlation-test", + )); + let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); + let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); + + let upstream_task = tokio::spawn(async move { + let mut forwarded = Vec::new(); + let mut byte = [0_u8; 1]; + while !forwarded.ends_with(b"\r\n\r\n") { + upstream_side.read_exact(&mut byte).await.unwrap(); + forwarded.push(byte[0]); + } + upstream_side + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .unwrap(); + upstream_side.flush().await.unwrap(); + forwarded + }); + + async { + tracing::callsite::rebuild_interest_cache(); + let correlation = + RequestCorrelation::with_request_id(&request, &context, REQUEST_ID.to_string()); + relay_http_request_with_response_middleware_guarded_observed_correlated( + &request, + &mut proxy_to_client, + &mut proxy_to_upstream, + RelayRequestOptions { + resolver: Some(&resolver), + credential_signing: crate::l7::CredentialSigning::SigV4NoBody, + signing_service: SERVICE, + signing_region: REGION, + host: "api.example.test", + port: 443, + ..Default::default() + }, + None, + None, + Some(&correlation), + ) + .await + .expect("SigV4 relay succeeds"); + correlation.record_final_http_outcome("allowed"); + } + .with_subscriber(subscriber) + .await; + let forwarded = upstream_task.await.expect("upstream task completes"); + assert!( + String::from_utf8_lossy(&forwarded) + .to_ascii_lowercase() + .contains("authorization: aws4-hmac-sha256"), + "request was signed" + ); + provider.force_flush().expect("flush spans"); + + let attributes = crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + None, + ); + assert_eq!(attributes["openshell.credential.outcome"], "prepared"); + assert_eq!(attributes["openshell.credential.mechanisms"], "sigv4"); + assert_eq!(attributes["openshell.credential.surfaces"], "signature"); + assert_eq!(attributes["openshell.credential.sigv4.outcome"], "prepared"); + assert_eq!(attributes["openshell.request.delivery_started"], "true"); + assert!(!attributes.contains_key("openshell.credential.static_placeholder.outcome")); + let serialized = format!("{attributes:?}"); + for private_value in [ACCESS_KEY, SECRET_KEY, REGION, SERVICE] { + assert!( + !serialized.contains(private_value), + "leaked {private_value}" + ); + } + } + + #[tokio::test] + async fn sigv4_missing_material_correlates_unavailable_without_delivery() { + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + const REQUEST_ID: &str = "00000000-0000-4000-8000-000000000042"; + const REGION: &str = "us-private-1"; + const SERVICE: &str = "private-service"; + let request = L7Request { + action: "GET".into(), + target: "/resource".into(), + query_params: HashMap::new(), + raw_header: b"GET /resource HTTP/1.1\r\nHost: api.example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + let context = crate::l7::relay::L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "sigv4-policy".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "sigv4-credential-correlation-test", + )); + let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); + let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); + + async { + tracing::callsite::rebuild_interest_cache(); + let correlation = + RequestCorrelation::with_request_id(&request, &context, REQUEST_ID.to_string()); + let error = relay_http_request_with_response_middleware_guarded_observed_correlated( + &request, + &mut proxy_to_client, + &mut proxy_to_upstream, + RelayRequestOptions { + credential_signing: crate::l7::CredentialSigning::SigV4NoBody, + signing_service: SERVICE, + signing_region: REGION, + host: "api.example.test", + port: 443, + ..Default::default() + }, + None, + None, + Some(&correlation), + ) + .await + .expect_err("missing SigV4 material fails closed"); + assert!(error.downcast_ref::().is_some()); + correlation.record_final_http_outcome("platform_error"); + } + .with_subscriber(subscriber) + .await; + drop(proxy_to_upstream); + let mut forwarded = Vec::new(); + upstream_side.read_to_end(&mut forwarded).await.unwrap(); + assert!(forwarded.is_empty(), "no request bytes reach upstream"); + provider.force_flush().expect("flush spans"); + + let attributes = crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + None, + ); + assert_eq!(attributes["openshell.credential.outcome"], "unavailable"); + assert_eq!(attributes["openshell.credential.mechanisms"], "sigv4"); + assert_eq!(attributes["openshell.credential.surfaces"], "signature"); + assert_eq!( + attributes["openshell.credential.sigv4.outcome"], + "unavailable" + ); + assert_eq!(attributes["openshell.request.delivery_started"], "false"); + assert!(!attributes.contains_key("openshell.credential.static_placeholder.outcome")); + let serialized = format!("{attributes:?}"); + for private_value in [REGION, SERVICE] { + assert!( + !serialized.contains(private_value), + "leaked {private_value}" + ); + } + } + #[tokio::test] async fn relay_injects_url_path_credential_telegram_style() { let (child_env, resolver) = placed_test_resolver( diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index f868575097..354b46f9fe 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -26,6 +26,9 @@ use openshell_ocsf::{ }; use tracing::warn; +use crate::l7::correlation::{ + CredentialMechanism, CredentialOutcome, CredentialSurface, RequestCorrelation, +}; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; use crate::telemetry::egress_emit; @@ -180,9 +183,19 @@ pub async fn inject_for_selectors( snapshot: &ProviderCredentialSnapshot, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result { let selected = select_scoped(snapshot, ctx, request_path(&req.target), None); - inject_or_report(req, ctx, selected, request_id, observation_context).await + inject_or_report( + req, + ctx, + selected, + false, + request_id, + observation_context, + correlation, + ) + .await } /// Inject only grants owned by endpoints that admitted the inspected request. @@ -199,8 +212,11 @@ pub(super) async fn inject_for_admitted_owners( admitted_owners: &HashSet, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result { let request_path = request_path(&req.target); + let matching_withheld = + has_matching_withheld_grant(snapshot, ctx, request_path, admitted_owners); let unowned = snapshot .dynamic_credentials .iter() @@ -220,7 +236,40 @@ pub(super) async fn inject_for_admitted_owners( } else { select_scoped(snapshot, ctx, request_path, Some(admitted_owners)) }; - inject_or_report(req, ctx, selected, request_id, observation_context).await + inject_or_report( + req, + ctx, + selected, + matching_withheld, + request_id, + observation_context, + correlation, + ) + .await +} + +/// Whether a valid endpoint-bound grant matched but none of its owners admitted +/// this request. This intentionally returns only a boolean so callers cannot +/// expose provider, credential, or endpoint configuration in telemetry. +pub(super) fn has_matching_withheld_grant( + snapshot: &ProviderCredentialSnapshot, + ctx: &L7EvalContext, + request_path: &str, + admitted_owners: &HashSet, +) -> bool { + snapshot + .dynamic_credentials + .iter() + .any(|(key, credential)| { + credential.token_grant.is_some() + && !credential.token_grant_owners.is_empty() + && !credential + .token_grant_owners + .iter() + .any(|owner| admitted_owners.contains(owner)) + && dynamic_credential_key_match_score(key, &ctx.host, ctx.port, request_path) + .is_some() + }) } fn select_scoped( @@ -249,13 +298,21 @@ async fn inject_or_report( req: L7Request, ctx: &L7EvalContext, selected: Result, TokenGrantInjectionError>, + matching_withheld: bool, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&RequestCorrelation>, ) -> Result { if selected.as_ref().is_ok_and(Vec::is_empty) { + if matching_withheld { + observe_token_grant(correlation, CredentialOutcome::Withheld); + } // Most requests need no grant; keep that path free of allocation. return Ok(req); } + if let Some(correlation) = correlation { + correlation.observe_credential(CredentialMechanism::TokenGrant, CredentialSurface::Header); + } let method = req.action.clone(); let request_path = request_path(&req.target).to_string(); let observation_context = observation_context @@ -276,11 +333,33 @@ async fn inject_or_report( Vec::new() }; let result = match selected { - Ok(credentials) => inject_selected(req, ctx, credentials, request_id).await, + Ok(credentials) => { + let injection = inject_selected(req, ctx, credentials, request_id, correlation); + if let Some(correlation) = correlation { + correlation.within_credential(injection).await + } else { + injection.await + } + } Err(error) => Err(error), }; if let Err(error) = &result { - report_refused_grant(ctx, &method, &request_path, request_id, error); + report_refused_grant(ctx, &method, &request_path, request_id, error, correlation); + } + if let Some(correlation) = correlation { + correlation.complete_credential_mechanism( + CredentialMechanism::TokenGrant, + match &result { + Ok(_) => CredentialOutcome::Prepared, + Err(error) + if error.failure().kind() + == TokenGrantFailureKind::WorkloadIdentityRejected => + { + CredentialOutcome::Denied + } + Err(_) => CredentialOutcome::Unavailable, + }, + ); } if let Some((sender, context)) = ctx .endpoint_observation_tx @@ -311,6 +390,13 @@ async fn inject_or_report( result } +fn observe_token_grant(correlation: Option<&RequestCorrelation>, outcome: CredentialOutcome) { + if let Some(correlation) = correlation { + correlation.observe_credential(CredentialMechanism::TokenGrant, CredentialSurface::Header); + correlation.complete_credential_mechanism(CredentialMechanism::TokenGrant, outcome); + } +} + /// Record a refused grant. /// /// The OCSF event's shorthand form can reach the sandboxed agent through @@ -323,6 +409,7 @@ fn report_refused_grant( request_path: &str, request_id: &str, error: &TokenGrantInjectionError, + correlation: Option<&RequestCorrelation>, ) { let credential = error .credential @@ -341,22 +428,25 @@ fn report_refused_grant( .map_or_else(|| "none".to_string(), |status| status.to_string()), "Token grant failed" ); - egress_emit!( - HttpActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new(method, ctx.ocsf_url(request_path))) - .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) - .message(format!( - "Token grant failed for {credential} to {}:{}", - ctx.host, ctx.port - )) - .unmapped("request_id", request_id) - .build() - ); + let event = HttpActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new(method, ctx.ocsf_url(request_path))) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message(format!( + "Token grant failed for {credential} to {}:{}", + ctx.host, ctx.port + )) + .unmapped("request_id", request_id) + .build(); + if let Some(correlation) = correlation { + correlation.emit(event); + } else { + egress_emit!(event); + } } /// Acquires every selected grant and rewrites the request headers. @@ -367,6 +457,7 @@ async fn inject_selected( ctx: &L7EvalContext, credentials: Vec<(String, ProviderProfileCredential)>, request_id: &str, + correlation: Option<&RequestCorrelation>, ) -> Result { let resolver = ctx .token_grant_resolver @@ -399,6 +490,9 @@ async fn inject_selected( inject_header(&raw_header, &name, &value).map_err(TokenGrantInjectionError::new)?; } for (provider_key, _) in credentials { + if correlation.is_some() { + continue; + } egress_emit!( HttpActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Other) @@ -1078,6 +1172,7 @@ pub mod test_support { &snapshot, request_id, observation_context.as_ref(), + None, ) .await } @@ -1092,8 +1187,11 @@ mod tests { EndpointConfigVersion, EndpointObservationSender, EndpointStatusReceiver, EndpointStatusTracker, endpoint_status_channel, }; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::layer::SubscriberExt as _; const REQUEST_ID: &str = "6a1f0e2d-3c4b-4a59-8e7f-0123456789ab"; + const CORRELATION_ID: &str = "00000000-0000-4000-8000-000000000004"; fn credential(auth_style: &str, header_name: &str) -> ProviderProfileCredential { ProviderProfileCredential { @@ -1451,15 +1549,232 @@ mod tests { } fn grant_request() -> L7Request { + grant_request_for("/v1/projects") + } + + fn grant_request_for(target: &str) -> L7Request { L7Request { action: "GET".to_string(), - target: "/v1/projects".to_string(), + target: target.to_string(), query_params: HashMap::new(), - raw_header: b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n".to_vec(), + raw_header: format!("GET {target} HTTP/1.1\r\nHost: api.example.com\r\n\r\n") + .into_bytes(), body_length: BodyLength::None, } } + enum GrantAdmission<'a> { + Selectors, + Owners(&'a HashSet), + } + + async fn capture_correlated_grant( + fixture: &TokenGrantTestFixture, + target: &str, + admission: GrantAdmission<'_>, + ) -> ( + Result, + BTreeMap, + Vec, + ) { + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer( + &provider, + "token-grant-correlation-test", + )) + .with(ocsf.clone()); + let mut ctx = grant_context(fixture); + let snapshot = pin_live_credentials(&mut ctx).expect("provider credentials"); + let request = grant_request_for(target); + + let operation = async { + tracing::callsite::rebuild_interest_cache(); + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, CORRELATION_ID.into()); + let result = match admission { + GrantAdmission::Selectors => { + inject_for_selectors( + request, + &ctx, + &snapshot, + correlation.request_id(), + None, + Some(&correlation), + ) + .await + } + GrantAdmission::Owners(owners) => { + inject_for_admitted_owners( + request, + &ctx, + &snapshot, + owners, + correlation.request_id(), + None, + Some(&correlation), + ) + .await + } + }; + correlation.record_final_http_outcome(if result.is_ok() { + "allowed" + } else { + "platform_error" + }); + drop(correlation); + result + } + .with_subscriber(subscriber) + .await; + provider.force_flush().expect("flush spans"); + + let attributes = crate::l7::correlation::test_request_attributes( + &exporter.get_finished_spans().expect("finished spans"), + Some(CORRELATION_ID), + ); + let events = ocsf + .json_events() + .into_iter() + .filter(|event| { + event["class_uid"] == 4002 && event["unmapped"].get("credential_outcome").is_some() + }) + .collect(); + (operation, attributes, events) + } + + #[tokio::test] + async fn correlation_records_prepared_token_grant_without_configuration_details() { + const SENSITIVE_KEY: &str = + "api.example.com\t443\t/v1/**\tSENTINEL_PROVIDER:SENTINEL_CREDENTIAL"; + let fixture = TokenGrantTestFixture::success(SENSITIVE_KEY, "grant-token"); + + let (result, attributes, events) = + capture_correlated_grant(&fixture, "/v1/projects", GrantAdmission::Selectors).await; + + assert!(result.is_ok()); + assert_eq!(attributes["openshell.credential.outcome"], "prepared"); + assert_eq!(attributes["openshell.credential.mechanisms"], "token_grant"); + assert_eq!(attributes["openshell.credential.surfaces"], "header"); + assert_eq!(fixture.requested_ids(), [CORRELATION_ID]); + assert!( + events.is_empty(), + "prepared credentials need no extra OCSF record" + ); + let event = format!("{attributes:?}"); + for sensitive in [ + "SENTINEL_PROVIDER", + "SENTINEL_CREDENTIAL", + "Authorization", + "api://example", + "auth.example.com", + "grant-token", + ] { + assert!(!event.contains(sensitive), "aggregate leaked {sensitive}"); + } + } + + #[tokio::test] + async fn correlation_records_unavailable_token_grant() { + let fixture = TokenGrantTestFixture::failure(KEY, refusal()); + + let (result, attributes, events) = + capture_correlated_grant(&fixture, "/v1/projects", GrantAdmission::Selectors).await; + + assert!(result.is_err()); + assert_eq!(attributes["openshell.credential.outcome"], "unavailable"); + assert_eq!(attributes["openshell.credential.mechanisms"], "token_grant"); + assert_eq!(attributes["openshell.credential.surfaces"], "header"); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["unmapped"]["credential_outcome"], "unavailable"); + } + + #[tokio::test] + async fn correlation_records_workload_identity_rejection_as_denied() { + let fixture = TokenGrantTestFixture::failure( + KEY, + TokenGrantFailure::new( + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantStage::WorkloadIdentity, + ), + ); + + let (result, attributes, _) = + capture_correlated_grant(&fixture, "/v1/projects", GrantAdmission::Selectors).await; + + assert!(result.is_err()); + assert_eq!(attributes["openshell.credential.outcome"], "denied"); + } + + #[tokio::test] + async fn correlation_records_matching_owner_filtered_grant_as_withheld() { + let fixture = TokenGrantTestFixture::success(KEY, "grant-token"); + let admitted_owners = HashSet::from(["different-owner".to_string()]); + + let (result, attributes, events) = capture_correlated_grant( + &fixture, + "/v1/projects", + GrantAdmission::Owners(&admitted_owners), + ) + .await; + + assert!(result.is_ok()); + fixture.assert_no_requests(); + assert_eq!(attributes["openshell.credential.outcome"], "withheld"); + assert_eq!(attributes["openshell.credential.mechanisms"], "token_grant"); + assert_eq!(attributes["openshell.credential.surfaces"], "header"); + assert_eq!(events.len(), 1); + } + + #[tokio::test] + async fn correlation_records_ownerless_matching_grant_as_unavailable() { + let fixture = TokenGrantTestFixture::success(KEY, "grant-token"); + fixture + .dynamic_credentials() + .write() + .unwrap() + .get_mut(KEY) + .unwrap() + .token_grant_owners + .clear(); + let admitted_owners = HashSet::new(); + + let (result, attributes, _) = capture_correlated_grant( + &fixture, + "/v1/projects", + GrantAdmission::Owners(&admitted_owners), + ) + .await; + + assert!(result.is_err()); + fixture.assert_no_requests(); + assert_eq!(attributes["openshell.credential.outcome"], "unavailable"); + } + + #[tokio::test] + async fn correlation_omits_unrelated_grant() { + let fixture = TokenGrantTestFixture::success(KEY, "grant-token"); + let admitted_owners = HashSet::new(); + + let (result, attributes, events) = capture_correlated_grant( + &fixture, + "/unrelated", + GrantAdmission::Owners(&admitted_owners), + ) + .await; + + assert!(result.is_ok()); + fixture.assert_no_requests(); + assert!(!attributes.contains_key("openshell.credential.outcome")); + assert!(!attributes.contains_key("openshell.credential.mechanisms")); + assert!(!attributes.contains_key("openshell.credential.surfaces")); + assert!(events.is_empty()); + } + async fn installed_status( revision: u64, ) -> ( @@ -1521,7 +1836,15 @@ mod tests { authority: &EndpointObservationContext, ) -> Result { let snapshot = pin_live_credentials(ctx).expect("live provider credentials"); - inject_for_selectors(grant_request(), ctx, &snapshot, REQUEST_ID, Some(authority)).await + inject_for_selectors( + grant_request(), + ctx, + &snapshot, + REQUEST_ID, + Some(authority), + None, + ) + .await } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs index b4c3253672..e482a524f4 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs @@ -333,6 +333,7 @@ async fn admitted_owners_select_grants_independently_per_header() { &owners, REQUEST_ID, None, + None, ) .await .unwrap(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index fa47354e14..4e067b8e2d 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5212,6 +5212,7 @@ fn canonicalize_forward_host_header(raw: &[u8], authority: &str) -> Result, ) -> Result, secrets::UnresolvedPlaceholderError> { + rewrite_forward_request_correlated(raw, used, path, canonical_authority, secret_resolver, None) + .map(|(bytes, _)| bytes) +} + +fn rewrite_forward_request_correlated( + raw: &[u8], + used: usize, + path: &str, + canonical_authority: &str, + secret_resolver: Option<&SecretResolver>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, +) -> Result<(Vec, bool), secrets::UnresolvedPlaceholderError> { let header_end = raw[..used] .windows(4) .position(|w| w == b"\r\n\r\n") .map_or(used, |p| p + 4); let websocket_upgrade = crate::l7::rest::request_is_websocket_upgrade(&raw[..header_end]); - let upstream_path = match secret_resolver { - Some(resolver) => secrets::rewrite_target_for_eval(path, resolver)?.resolved, - None => path.to_string(), - }; - let header_str = String::from_utf8_lossy(&raw[..header_end]); let lines = header_str.split("\r\n").collect::>(); let connection_nominated: std::collections::HashSet = lines @@ -5240,6 +5248,54 @@ fn rewrite_forward_request( .map(|token| token.trim().to_ascii_lowercase()) .filter(|token| !token.is_empty()) .collect(); + let header_is_stripped = |field_name: &str| { + field_name == "host" + || matches!( + field_name, + "proxy-connection" | "proxy-authorization" | "proxy-authenticate" + ) + || connection_nominated.contains(field_name) + || matches!(field_name, "connection" | "upgrade") + }; + let mut static_credential_selected = false; + if let Some(correlation) = correlation { + let (path_component, query) = path + .split_once('?') + .map_or((path, None), |(path, query)| (path, Some(query))); + if secrets::contains_reserved_credential_marker(path_component) { + correlation.observe_credential( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + crate::l7::correlation::CredentialSurface::Path, + ); + static_credential_selected = true; + } + if query.is_some_and(secrets::contains_reserved_credential_marker) { + correlation.observe_credential( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + crate::l7::correlation::CredentialSurface::Query, + ); + static_credential_selected = true; + } + for line in lines.iter().skip(1).take_while(|line| !line.is_empty()) { + let (field_name, field_value) = line + .split_once(':') + .expect("forward request passed strict ingress header validation"); + let field_name = field_name.to_ascii_lowercase(); + if !header_is_stripped(&field_name) + && secrets::header_value_contains_reserved_credential_marker(field_value) + { + correlation.observe_credential( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + crate::l7::correlation::CredentialSurface::Header, + ); + static_credential_selected = true; + } + } + } + let upstream_path = match secret_resolver { + Some(resolver) => secrets::rewrite_target_for_eval(path, resolver)?.resolved, + None => path.to_string(), + }; // Rebuild headers, stripping hop-by-hop and adding proxy headers let mut output = Vec::with_capacity(header_end + 128); @@ -5271,26 +5327,7 @@ fn rewrite_forward_request( .expect("forward request passed strict ingress header validation"); let field_name = field_name.to_ascii_lowercase(); - // RFC 9112 section 3.2.2 requires proxies to replace every received - // Host field with one generated from the absolute request-target. - if field_name == "host" { - continue; - } - - // Strip proxy hop-by-hop headers - if matches!( - field_name.as_str(), - "proxy-connection" | "proxy-authorization" | "proxy-authenticate" - ) { - continue; - } - - if connection_nominated.contains(&field_name) { - continue; - } - - // Reconstruct hop-by-hop upgrade fields after processing the originals. - if field_name == "connection" || field_name == "upgrade" { + if header_is_stripped(&field_name) { continue; } @@ -5346,7 +5383,7 @@ fn rewrite_forward_request( return Err(secrets::UnresolvedPlaceholderError::unavailable("header")); } - Ok(output) + Ok((output, static_credential_selected)) } fn initial_forward_body_prefix_len(header_str: &str, overflow: &[u8]) -> usize { @@ -5503,6 +5540,7 @@ where .await } +#[allow(clippy::too_many_arguments)] async fn inject_token_grant_for_forward_request( method: &str, upstream_target: &str, @@ -5511,6 +5549,7 @@ async fn inject_token_grant_for_forward_request( inspection: Option>, request_id: &str, observation_context: Option<&EndpointObservationContext>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result> { let (path, _) = crate::l7::rest::parse_target_query(upstream_target)?; let request = crate::l7::rest::request_from_buffered_http( @@ -5528,6 +5567,7 @@ async fn inject_token_grant_for_forward_request( inspection.request_info, request_id, observation_context, + correlation, ) .await?; *l7_ctx = prepared_ctx; @@ -5545,6 +5585,7 @@ async fn inject_token_grant_for_forward_request( &snapshot, request_id, observation_context, + correlation, ) .await? .raw_header) @@ -6795,6 +6836,7 @@ async fn handle_forward_proxy( }), &request_id, endpoint_observation_context.as_ref(), + Some(&correlation), ) .await { @@ -6877,6 +6919,7 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } + correlation.complete_credential(crate::l7::correlation::CredentialOutcome::Stale); correlation.record_enforcement_action("block"); correlation.record_response_status(500); correlation.record_final_http_outcome("platform_error"); @@ -6896,14 +6939,23 @@ async fn handle_forward_proxy( } // 9. Rewrite request and forward to upstream - let rewritten = match rewrite_forward_request( + let rewritten = match rewrite_forward_request_correlated( &forward_request_bytes, forward_request_bytes.len(), &upstream_target, &canonical_authority, secret_resolver.as_deref(), + Some(&correlation), ) { - Ok(bytes) => bytes, + Ok((bytes, static_credential_selected)) => { + if static_credential_selected { + correlation.complete_credential_mechanism( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + crate::l7::correlation::CredentialOutcome::Prepared, + ); + } + bytes + } Err(e) => { warn!( dst_host = %host_lc, @@ -6916,6 +6968,14 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } + correlation.complete_credential_mechanism( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + if e.is_endpoint_mismatch() || e.is_placement_mismatch() { + crate::l7::correlation::CredentialOutcome::Denied + } else { + crate::l7::correlation::CredentialOutcome::Unavailable + }, + ); if e.is_placement_mismatch() { crate::l7::relay::reject_credential_resolution( client, @@ -7217,6 +7277,26 @@ async fn handle_forward_proxy( .await?; return Ok(()); } + if let Some(error) = + report.downcast_ref::() + { + if error.generation_is_stale() { + correlation + .complete_credential(crate::l7::correlation::CredentialOutcome::Stale); + } else { + correlation.complete_credential_mechanism( + crate::l7::correlation::CredentialMechanism::SigV4, + crate::l7::correlation::CredentialOutcome::Unavailable, + ); + } + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage(if error.generation_is_stale() { + "credential_generation" + } else { + "credential_availability" + }); + } Err(report) } outcome => outcome, @@ -12182,6 +12262,7 @@ network_policies: None, FORWARD_REQUEST_ID, None, + None, ) .await } @@ -14723,6 +14804,151 @@ network_policies: assert!(result_str.contains("X-Visible: yes\r\n")); } + #[test] + fn forward_credential_correlation_ignores_stripped_headers() { + use tracing_subscriber::layer::SubscriberExt as _; + + const MARKER: &str = "openshell:resolve:env:SENTINEL_KEY"; + let stripped_raw = format!( + "GET http://host/p HTTP/1.1\r\nHost: {MARKER}\r\nProxy-Authorization: Bearer {MARKER}\r\nConnection: x-drop\r\nX-Drop: {MARKER}\r\n\r\n" + ) + .into_bytes(); + let selected_raw = format!( + "GET http://host/p HTTP/1.1\r\nHost: host\r\nAuthorization: Bearer {MARKER}\r\n\r\n" + ) + .into_bytes(); + let unresolved_marker = "openshell:resolve:env:MISSING_KEY"; + let combined_raw = format!( + "GET http://host/{unresolved_marker} HTTP/1.1\r\nHost: host\r\nX-Keep: {unresolved_marker}\r\n\r\n" + ) + .into_bytes(); + let request = |raw_header: Vec| crate::l7::provider::L7Request { + action: "GET".into(), + target: "/p".into(), + query_params: std::collections::HashMap::new(), + raw_header, + body_length: crate::l7::provider::BodyLength::None, + }; + let ctx = crate::l7::relay::L7EvalContext { + host: "host".into(), + port: 80, + policy_name: "credential-test".into(), + ..Default::default() + }; + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "forward-credential-correlation-test", + )); + let (_, resolver) = SecretResolver::from_provider_env( + [("SENTINEL_KEY".to_string(), "resolved-value".to_string())] + .into_iter() + .collect(), + ); + let resolver = resolver.expect("credential resolver"); + + tracing::subscriber::with_default(subscriber, || { + tracing::callsite::rebuild_interest_cache(); + let stripped = request(stripped_raw); + let correlation = crate::l7::correlation::RequestCorrelation::with_request_id( + &stripped, + &ctx, + "00000000-0000-4000-8000-000000000021".into(), + ); + let (_, selected) = rewrite_forward_request_correlated( + &stripped.raw_header, + stripped.raw_header.len(), + "/p", + "host", + None, + Some(&correlation), + ) + .expect("stripped placeholders never reach credential rewriting"); + assert!(!selected); + correlation.record_final_http_outcome("allowed"); + + let selected = request(selected_raw); + let correlation = crate::l7::correlation::RequestCorrelation::with_request_id( + &selected, + &ctx, + "00000000-0000-4000-8000-000000000022".into(), + ); + let (rewritten, selected) = rewrite_forward_request_correlated( + &selected.raw_header, + selected.raw_header.len(), + "/p", + "host", + Some(&resolver), + Some(&correlation), + ) + .expect("eligible placeholder resolves"); + assert!(selected); + assert!( + String::from_utf8_lossy(&rewritten) + .contains("Authorization: Bearer resolved-value") + ); + correlation.complete_credential_mechanism( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + crate::l7::correlation::CredentialOutcome::Prepared, + ); + correlation.record_final_http_outcome("allowed"); + + let combined = request(combined_raw); + let correlation = crate::l7::correlation::RequestCorrelation::with_request_id( + &combined, + &ctx, + "00000000-0000-4000-8000-000000000023".into(), + ); + rewrite_forward_request_correlated( + &combined.raw_header, + combined.raw_header.len(), + &format!("/{unresolved_marker}"), + "host", + Some(&resolver), + Some(&correlation), + ) + .expect_err("unresolved path must fail after all surfaces are observed"); + correlation.complete_credential_mechanism( + crate::l7::correlation::CredentialMechanism::StaticPlaceholder, + crate::l7::correlation::CredentialOutcome::Unavailable, + ); + correlation.record_final_http_outcome("platform_error"); + }); + provider.force_flush().expect("flush spans"); + + let spans = exporter.get_finished_spans().expect("finished spans"); + let attributes_for = |request_id: &str| { + crate::l7::correlation::test_request_attributes(&spans, Some(request_id)) + }; + let stripped = attributes_for("00000000-0000-4000-8000-000000000021"); + assert!( + stripped + .keys() + .all(|key| !key.starts_with("openshell.credential.")) + ); + let selected = attributes_for("00000000-0000-4000-8000-000000000022"); + assert_eq!(selected["openshell.credential.outcome"], "prepared"); + assert_eq!( + selected["openshell.credential.mechanisms"], + "static_placeholder" + ); + assert_eq!(selected["openshell.credential.surfaces"], "header"); + assert!(!format!("{selected:?}").contains("SENTINEL_KEY")); + assert!(!format!("{selected:?}").contains("resolved-value")); + + let combined = attributes_for("00000000-0000-4000-8000-000000000023"); + assert_eq!(combined["openshell.credential.outcome"], "unavailable"); + assert_eq!( + combined["openshell.credential.mechanisms"], + "static_placeholder" + ); + assert_eq!(combined["openshell.credential.surfaces"], "header,path"); + assert!(!format!("{combined:?}").contains("MISSING_KEY")); + } + #[test] fn test_rewrite_preserves_body_overflow() { let raw = b"POST http://host/api HTTP/1.1\r\nHost: host\r\nContent-Length: 13\r\n\r\n{\"key\":\"val\"}"; diff --git a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs index 0d26319749..b1dbeb00d4 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs @@ -80,6 +80,7 @@ async fn forward_inspection_selects_only_the_owner_admitting_the_request() { }), "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", None, + None, ) .await .expect("prepare admitted forward request"); @@ -130,6 +131,7 @@ async fn forward_inspection_retains_installation_through_guarded_write() { }), "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", None, + None, ) .await .expect("prepare authenticated request"); diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index 1b89c6666d..8dd7cf7fad 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -87,6 +87,10 @@ handle, not authorization or proof of delivery. Match it to trusted OpenShell output for the same sandbox. L4 and uninspected passthrough traffic do not provide this receipt guarantee. +When credential handling runs, an `openshell.credential.prepare` child span records +the mechanism, placement and outcome without credential values. The primary +OCSF record includes notable withholding, denial, unavailability or stale state. + The primary HTTP Activity record describes the final response headers accepted by the client connection. Response-body completion is recorded separately in the trace. A