diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 1c3b0f8f..46e20fe8 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -11,7 +11,8 @@ use walshadow::ch_emitter::EmitterStats; use walshadow::config::ConfigResolver; use walshadow::metrics::WorkerState; use walshadow::metrics::{ - DbSeries, MetricsRegistry, MetricsSnapshot, RateEstimator, source_server_wal_end, + DbSeries, HistogramSnapshot, MetricsRegistry, MetricsSnapshot, RateEstimator, + source_server_wal_end, }; use walshadow::pipeline::row_ledger::TableRowCounts; use walshadow::pos::{ @@ -162,6 +163,15 @@ impl DbMetricSources { let bridge_op_seconds = |pick: fn(&BridgeStats) -> &[AtomicU64; OP_COUNT]| { bridge_ops(pick).map(|nanos| nanos as f64 / 1e9) }; + let bridge_latency = || -> [HistogramSnapshot; OP_COUNT] { + std::array::from_fn(|i| { + let mut folded = HistogramSnapshot::default(); + for s in self.bridge.iter().flatten() { + folded.merge(s.request_latency[i].snapshot(1e9)); + } + folded + }) + }; let log_stats = self.desc_log.as_ref().map(|log| log.stats_handle()); let log = |pick: fn(&DescLogStats) -> &AtomicU64| -> u64 { log_stats.as_ref().map_or(0, |s| ld(pick(s))) @@ -183,6 +193,7 @@ impl DbMetricSources { bridge_requests_by_op: bridge_ops(|s| &s.requests), bridge_errors_by_op: bridge_ops(|s| &s.errors), bridge_request_seconds_by_op: bridge_op_seconds(|s| &s.request_nanos), + request_duration_seconds: bridge_latency(), bridge_lock_wait_seconds_by_op: bridge_op_seconds(|s| &s.lock_wait_nanos), bridge_service_seconds_by_op: bridge_op_seconds(|s| &s.service_nanos), bridge_request_bytes_by_op: bridge_ops(|s| &s.request_bytes), @@ -564,6 +575,9 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M toast_value_fetch_seconds: emitter_seconds(|s| &s.toast_value_fetch_nanos), toast_values_read_ahead_total: emitter(|s| &s.toast_values_read_ahead), toast_value_wait_seconds: emitter_seconds(|s| &s.toast_value_wait_nanos), + toast_values_per_fetch: v.emitter.map_or_else(HistogramSnapshot::default, |s| { + s.toast_values_per_fetch.snapshot(1.0) + }), toast_image_rows_mirrored_total: emitter(|s| &s.toast_image_rows_mirrored), toast_values_filled_superseded_total: emitter(|s| &s.toast_values_filled_superseded), toast_values_filled_mismatch_total: emitter(|s| &s.toast_values_filled_mismatch), @@ -1055,6 +1069,7 @@ mod tests { let bump = |s: &BridgeStats, n: u64| { s.up.store(1, Ordering::Relaxed); s.requests[encode].fetch_add(n, Ordering::Relaxed); + s.request_latency[encode].observe(30_000); s.native_bytes.fetch_add(n, Ordering::Relaxed); }; let (live, boot) = ( @@ -1076,6 +1091,10 @@ mod tests { let handed_over = sources([Some(live.clone()), Some(boot.clone())]).series(); assert_eq!(handed_over.database, "app"); assert_eq!(handed_over.bridge_native_bytes_total, 7); + assert_eq!( + handed_over.request_duration_seconds[encode].count, 2, + "both bridges' latency samples fold into one histogram" + ); assert_eq!( handed_over.bridge_requests_by_op[encode], 7, "both bridges' requests land on the database's series" diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 00f5a230..7f473430 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -2639,6 +2639,10 @@ fn oracle_cell(v: Option<&ColumnValue>, source_type_oid: u32) -> Result; + crate::atomic_stats! { /// CH emitter counters. `fetch_add(_, Relaxed)`; status loop reads /// via `.load(Relaxed)`. @@ -2680,6 +2684,8 @@ crate::atomic_stats! { pub toast_values_read_ahead, /// Planning blocked on reads started ahead of it pub toast_value_wait_nanos, + /// Values requested per store fetch + pub toast_values_per_fetch: ValuesPerFetch, /// Toasted values NULL/default-filled because no store could rebuild /// them (disabled mode). Surfaced, never silent pub toast_values_filled_default, diff --git a/src/ops/bridge.rs b/src/ops/bridge.rs index f22978e5..216487f3 100644 --- a/src/ops/bridge.rs +++ b/src/ops/bridge.rs @@ -20,6 +20,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::UnixStream; use tokio::sync::Mutex; +use crate::ops::metrics::Histogram; use crate::toast::FetchedValue; /// Frame and op layouts. Must equal `WS_PROTO_VERSION` in `pgext/walshadow.h` @@ -258,6 +259,11 @@ pub struct ScanResult { pub rows: Vec>>, } +/// Bridge request latency in nanoseconds: 25us doubling to 102.4ms, which +/// spans a local socket round trip through a worker stalled on slow storage. +/// Slower requests land in `+Inf` +pub type RequestLatency = Histogram<25_000, 13>; + crate::atomic_stats! { pub struct BridgeStats { /// 1 while the last transport attempt succeeded @@ -276,6 +282,8 @@ crate::atomic_stats! { pub requests: [AtomicU64; OP_COUNT], pub errors: [AtomicU64; OP_COUNT], pub request_nanos: [AtomicU64; OP_COUNT], + /// Distribution behind `request_nanos` + pub request_latency: [RequestLatency; OP_COUNT], /// Queued behind another caller's request on the one socket. Against /// `service_nanos` this is what says whether the worker is the /// limiter or the funnel in front of it is @@ -751,7 +759,9 @@ impl Bridge { fn record(stats: &BridgeStats, op: Op, started: Instant, res: &Result) { let slot = op.slot(); stats.requests[slot].fetch_add(1, Ordering::Relaxed); - stats.request_nanos[slot].fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + let nanos = started.elapsed().as_nanos() as u64; + stats.request_nanos[slot].fetch_add(nanos, Ordering::Relaxed); + stats.request_latency[slot].observe(nanos); if res.is_err() { stats.errors[slot].fetch_add(1, Ordering::Relaxed); } diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 037769fc..8ddedc17 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -17,6 +17,7 @@ use std::fmt; use std::io; use std::net::SocketAddr; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use prometheus_client::collector::Collector; @@ -173,6 +174,9 @@ macro_rules! encode_metric { ($enc:expr, gauge, $name:expr, $help:expr, $value:expr) => { gauge($enc, $name, $help, $value)? }; + ($enc:expr, histogram, $name:expr, $help:expr, $value:expr) => { + histogram($enc, $name, $help, &$value)? + }; ($enc:expr, counter, $name:expr, $help:expr, $value:expr, $key:literal, $labels:expr) => { counter_series($enc, $name, $help, $key, $labels.into_iter().zip($value))? }; @@ -446,6 +450,8 @@ snapshot! { "Values read from the store ahead of planning, off the serial path. The fetch counters include these reads.", counter toast_value_wait_seconds: f64 = "Cumulative wall-clock planning spent waiting on values read ahead of it, apart from the store time those reads count in toast_value_fetch_seconds.", + histogram toast_values_per_fetch: HistogramSnapshot = + "Values requested per chunk-store fetch round trip; the distribution behind the batching factor toast_values_fetched_total / toast_value_fetch_batches_total.", counter toast_image_rows_mirrored_total: u64 = "Chunk rows mirrored from restored page images during a backup window; non-zero means the backup copied TOAST pages mid-write.", counter toast_values_filled_superseded_total: u64 = @@ -669,6 +675,9 @@ db_snapshot! { /// but not yet complete. Only `ctl status` reads it; the scrape /// renders the `mode=` split beside it config_backfills_pending: u64, + /// Bridge round-trip wall time per request and op, lock wait + /// included, rendered `walshadow_request_duration_seconds{target="bridge",op}` + request_duration_seconds: [HistogramSnapshot; OP_COUNT], } gauge bridge_up: u64 = @@ -952,6 +961,110 @@ pub(crate) fn gauge_series<'a, V: EncodeGaugeValue>( Ok(()) } +/// Fixed-bucket histogram observed lock-free off the hot path: one relaxed +/// `fetch_add` per bucket and sum. `N` finite upper bounds starting at +/// `FIRST` and doubling, in the owner's integer unit (nanoseconds, values), +/// plus an implicit `+Inf` bucket. A scrape racing an observation can see the +/// count one ahead of the sum +#[derive(Debug)] +pub struct Histogram { + /// Per-bucket, not cumulative; the encoder accumulates + counts: [AtomicU64; N], + overflow: AtomicU64, + sum: AtomicU64, +} + +impl Default for Histogram { + fn default() -> Self { + Self { + counts: std::array::from_fn(|_| AtomicU64::new(0)), + overflow: AtomicU64::new(0), + sum: AtomicU64::new(0), + } + } +} + +impl Histogram { + fn bound(i: usize) -> u64 { + FIRST << i + } + + /// A sample lands in the first bucket with `v <= bound` + pub fn observe(&self, v: u64) { + let i = (0..N).find(|i| v <= Self::bound(*i)); + let bucket = i.map_or(&self.overflow, |i| &self.counts[i]); + bucket.fetch_add(1, Ordering::Relaxed); + self.sum.fetch_add(v, Ordering::Relaxed); + } + + /// Bounds and sum divided by `scale` into the exported unit + pub fn snapshot(&self, scale: f64) -> HistogramSnapshot { + let mut buckets: Vec<(f64, u64)> = self + .counts + .iter() + .enumerate() + .map(|(i, c)| (Self::bound(i) as f64 / scale, c.load(Ordering::Relaxed))) + .collect(); + buckets.push((f64::INFINITY, self.overflow.load(Ordering::Relaxed))); + HistogramSnapshot { + count: buckets.iter().map(|(_, c)| c).sum(), + sum: self.sum.load(Ordering::Relaxed) as f64 / scale, + buckets, + } + } +} + +/// A [`Histogram`] read at scrape time: per-bucket counts under finite upper +/// bounds, closed by an `f64::INFINITY` bucket that renders as `+Inf`. Empty until +/// something publishes one +#[derive(Debug, Default, Clone, PartialEq)] +pub struct HistogramSnapshot { + pub buckets: Vec<(f64, u64)>, + pub sum: f64, + pub count: u64, +} + +impl HistogramSnapshot { + /// Buckets as rendered: a snapshot nobody published still closes with + /// `+Inf`, which every histogram sample needs + fn encoded_buckets(&self) -> &[(f64, u64)] { + if self.buckets.is_empty() { + &[(f64::INFINITY, 0)] + } else { + &self.buckets + } + } + + /// Fold another histogram over the same bounds in, as one database's + /// bootstrap and live bridges share a series + pub fn merge(&mut self, other: HistogramSnapshot) { + if self.buckets.is_empty() { + *self = other; + return; + } + debug_assert_eq!(self.buckets.len(), other.buckets.len()); + for (mine, theirs) in self.buckets.iter_mut().zip(other.buckets) { + mine.1 += theirs.1; + } + self.sum += other.sum; + self.count += other.count; + } +} + +pub(crate) fn histogram( + enc: &mut DescriptorEncoder<'_>, + name: &str, + help: &str, + value: &HistogramSnapshot, +) -> fmt::Result { + declare(enc, name, help, MetricType::Histogram)?.encode_histogram::( + value.sum, + value.count, + value.encoded_buckets(), + None, + ) +} + /// `kind=`/`op=` grid over [`HEAP_OP_LABELS`] fn counter_kind_op( enc: &mut DescriptorEncoder<'_>, @@ -1189,7 +1302,28 @@ fn encode_snapshot(snap: &MetricsSnapshot, enc: &mut DescriptorEncoder<'_>) -> f .encode_family(&[("system_id", snap.source_system_id)])? .encode_gauge(&1u64)?; - encode_db_fields(&snap.by_database, enc) + encode_db_fields(&snap.by_database, enc)?; + + let mut durations = declare( + enc, + "walshadow_request_duration_seconds", + "Round-trip wall time of requests to an external service, by target and op. bridge is the pgext bridge worker over its socket, lock wait included; it gives the quantiles the per-op seconds and request counters can only average.", + MetricType::Histogram, + )?; + for db in &snap.by_database { + let database = db.database.as_str(); + for (op, value) in OP_LABELS.iter().zip(&db.request_duration_seconds) { + durations + .encode_family(&[("database", database), ("target", "bridge"), ("op", *op)])? + .encode_histogram::( + value.sum, + value.count, + value.encoded_buckets(), + None, + )?; + } + } + Ok(()) } /// Snapshot families, stage timings, then the OpenMetrics `# EOF` marker @@ -1660,6 +1794,69 @@ mod tests { assert!(body.contains("walshadow_pump_queue_depth 0"), "{body}"); } + #[test] + fn histograms_render_cumulative_buckets_per_series() { + let h = Histogram::<10, 3>::default(); + for v in [5, 10, 11, 20, 5000] { + h.observe(v); + } + let snap = h.snapshot(1.0); + assert_eq!(snap.count, 5); + assert_eq!(snap.sum, 5046.0); + assert_eq!( + snap.buckets.iter().map(|b| b.1).collect::>(), + [2, 2, 0, 1] + ); + + let mut folded = HistogramSnapshot::default(); + folded.merge(snap.clone()); + folded.merge(snap.clone()); + assert_eq!(folded.count, 10); + assert_eq!(folded.buckets[1].1, 4); + + let mut bridge = [ + HistogramSnapshot::default(), + HistogramSnapshot::default(), + HistogramSnapshot::default(), + HistogramSnapshot::default(), + HistogramSnapshot::default(), + ]; + bridge[1] = snap.clone(); + let body = render(MetricsSnapshot { + toast_values_per_fetch: snap, + by_database: vec![DbSeries { + database: "app".into(), + request_duration_seconds: bridge, + ..DbSeries::default() + }], + ..MetricsSnapshot::default() + }); + let latency = crate::bridge::RequestLatency::default(); + latency.observe(6_400_000); + let latency = latency.snapshot(1e9); + assert_eq!(latency.buckets[0].0, 0.000025); + assert_eq!(latency.buckets[8], (0.0064, 1)); + let op = OP_LABELS[1]; + for want in [ + "# TYPE walshadow_toast_values_per_fetch histogram".to_string(), + "walshadow_toast_values_per_fetch_bucket{le=\"10.0\"} 2".to_string(), + "walshadow_toast_values_per_fetch_bucket{le=\"20.0\"} 4".to_string(), + "walshadow_toast_values_per_fetch_bucket{le=\"+Inf\"} 5".to_string(), + "walshadow_toast_values_per_fetch_count 5".to_string(), + "# TYPE walshadow_request_duration_seconds histogram".to_string(), + format!( + "walshadow_request_duration_seconds_bucket{{le=\"40.0\",database=\"app\",target=\"bridge\",op=\"{op}\"}} 4" + ), + format!( + "walshadow_request_duration_seconds_count{{database=\"app\",target=\"bridge\",op=\"{op}\"}} 5" + ), + ] { + assert!(body.contains(&want), "missing {want}\n{body}"); + } + } + + /// A database name is a PG identifier, so a quote in it must not end the + /// label and truncate the scrape #[test] fn render_escapes_a_database_label() { let body = render(MetricsSnapshot { @@ -1838,7 +2035,7 @@ mod tests { if let Some(rest) = line.strip_prefix("# TYPE ") { let (name, kind) = rest.split_once(' ').expect("TYPE keyword"); assert_eq!(name, helped, "TYPE without its own HELP: {line}"); - assert!(matches!(kind, "counter" | "gauge"), "{line}"); + assert!(matches!(kind, "counter" | "gauge" | "histogram"), "{line}"); declared = Some((name, kind)); continue; } @@ -1848,11 +2045,20 @@ mod tests { None => (head, ""), }; let (family, kind) = declared.expect("sample ahead of its declaration"); - let want = match kind { - "counter" => format!("{family}_total"), - _ => family.to_owned(), - }; - assert_eq!(name, want, "sample outside its family: {line}"); + if kind == "histogram" { + assert!( + ["_sum", "_count", "_bucket"] + .iter() + .any(|suffix| name == format!("{family}{suffix}")), + "sample outside its family: {line}" + ); + } else { + let want = match kind { + "counter" => format!("{family}_total"), + _ => family.to_owned(), + }; + assert_eq!(name, want, "sample outside its family: {line}"); + } for label in labels.split_terminator(',') { let (key, val) = label.split_once('=').expect("key=value label"); assert!(!key.is_empty(), "{line}"); diff --git a/src/toast/resolver.rs b/src/toast/resolver.rs index ec600003..e5a9277a 100644 --- a/src/toast/resolver.rs +++ b/src/toast/resolver.rs @@ -1601,6 +1601,9 @@ impl ToastResolver { self.stats .toast_value_fetch_batches .fetch_add(1, Ordering::Relaxed); + self.stats + .toast_values_per_fetch + .observe(values.len() as u64); let assembled = got .iter() .filter(|v| matches!(v, FetchedValue::Assembled(_))) @@ -2670,6 +2673,9 @@ mod tests { 1, "miss does not count as fetched" ); + // Both lookups were one-value fetches, miss included + let per_fetch = stats.toast_values_per_fetch.snapshot(1.0); + assert_eq!((per_fetch.count, per_fetch.buckets[0].1), (2, 2)); } #[tokio::test]