From 4b527190c3d9ffde895985733fcf676efa0aaec0 Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Tue, 20 Oct 2026 08:41:00 -0500 Subject: [PATCH] feat(shadow-stream): walsender connection, backfill and queue metrics --- src/bin/stream/metrics_publish.rs | 8 +- src/bin/stream/session.rs | 6 +- src/ops/metrics.rs | 64 +++++++++-- src/source/shadow_stream.rs | 179 ++++++++++++++++++++++-------- tests/bin_stream_e2e.rs | 4 +- tests/walsender_restore_miss.rs | 2 +- 6 files changed, 205 insertions(+), 58 deletions(-) diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 7371315c..99c2821e 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -28,7 +28,9 @@ pub(crate) struct ShadowMetricsView { /// yet, so the gauges keep their last value pub(crate) apply_lag: Option<(u64, f64)>, pub(crate) active_connections: u64, - pub(crate) dropped_total: u64, + pub(crate) socket_events: [u64; walshadow::shadow_stream::SocketEvent::COUNT], + pub(crate) backfill_bytes_total: u64, + pub(crate) send_queue_bytes: u64, } /// Apply lag against the source head. A zero `shadow_replay` means no walreceiver @@ -313,7 +315,9 @@ pub(crate) async fn populate_metrics( shadow_apply_lag_bytes: apply_lag_bytes, shadow_apply_lag_seconds: apply_lag_seconds, shadow_stream_active_connections: shadow_view.active_connections, - shadow_stream_dropped_connections_total: shadow_view.dropped_total, + shadow_stream_sockets_by_event: shadow_view.socket_events, + wal_bytes: [shadow_view.backfill_bytes_total], + buffer_bytes: [shadow_view.send_queue_bytes], ..registry.snapshot().await }; populate_pipeline_metrics( diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index 027d9a95..0a26e0b3 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -117,7 +117,7 @@ async fn await_walreceiver( // `accepted` separates "shadow never dialed" from "shadow dialed // and stalled in the handshake" — the latter reads as the former // without it, since only START_REPLICATION registers a connection - let (secs, accepted) = (timeout.as_secs(), agg.accepted_total); + let (secs, accepted) = (timeout.as_secs(), agg.accepted()); anyhow::ensure!( start.elapsed() < timeout, "no walreceiver streaming from walsender {walsender_addr} within \ @@ -2354,7 +2354,9 @@ async fn session_body( ShadowMetricsView { apply_lag, active_connections: shadow_agg.active_connections as u64, - dropped_total: shadow_agg.dropped_total, + socket_events: shadow_agg.socket_events, + backfill_bytes_total: shadow_agg.backfill_bytes_total, + send_queue_bytes: shadow_agg.send_queue_bytes, }, &boundary_hold_stats, metrics_dbs.iter().map(DbMetricSources::series).collect(), diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index a209ebb0..4ca09ef7 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -36,6 +36,7 @@ use crate::catalog::pending::DegradeReason; use crate::decode::heap_decoder::HEAP_OP_LABELS; use crate::ops::bridge::{OP_COUNT, OP_LABELS}; use crate::pos::{Drain, EmitterAck, FilterDispatched, Floor, Pos, ShadowReplay, SourceReceived}; +use crate::source::shadow_stream::SocketEvent; use crate::source::transition::SWITCH_FAILURE_REASONS; #[macro_export] @@ -97,6 +98,18 @@ macro_rules! label_class { }; } +label_class! { + pub enum WalFlow { + WalsenderBackfill => "walsender_backfill", + } +} + +label_class! { + pub enum Buffer { + WalsenderSendQueue => "walsender_send_queue", + } +} + label_class! { pub enum WorkerState { Busy => "busy", @@ -281,6 +294,11 @@ snapshot! { /// Failures by [`ErrorKind`], rendered /// `walshadow_errors_total{component,kind}` errors: [u64; ErrorKind::COUNT], + /// Postgres data moved, by [`WalFlow`], rendered + /// `walshadow_wal_bytes_total{flow}` + wal_bytes: [u64; WalFlow::COUNT], + /// Data held, by [`Buffer`], rendered `walshadow_buffer_bytes{buffer}` + buffer_bytes: [u64; Buffer::COUNT], /// Refusal a crossing parked on, rendered as `crossing_wedged`. The /// pump keeps publishing rather than exiting into a restart that /// re-crosses and re-fails, so this is how the daemon says it is @@ -586,9 +604,9 @@ snapshot! { "Estimated seconds shadow trails source, by source byte rate over last 30s.", gauge shadow_stream_active_connections: u64 = "Currently-attached walreceiver connections to walshadow's walsender.", - /// Cumulative connections dropped by `slow_threshold` overflow - counter shadow_stream_dropped_connections_total: u64 = - "Connections dropped by slow-client cutoff since daemon start.", + counter shadow_stream_sockets_by_event: [u64; SocketEvent::COUNT] + ["event" = SocketEvent::LABELS] = + "Walsender socket lifecycle events: accepted (entered the handshake, START_REPLICATION or not), closed_slow_client (dropped at the send queue cap), closed_by_client (clean hangup), closed_error (handshake, I/O, protocol failure or refused START_REPLICATION). accepted minus the closed events is sockets alive.", counter catalog_boundary_holds_total: u64 = "Publication holds released at catalog-mutating commit boundaries.", counter catalog_boundary_hold_failures_total: u64 = @@ -1069,6 +1087,26 @@ fn encode_snapshot(snap: &MetricsSnapshot, enc: &mut DescriptorEncoder<'_>) -> f .map(|e| (e.pair(), snap.errors[e.index()])), )?; + counter_series( + enc, + "walshadow_wal_bytes_total", + "Postgres data moved, by flow. walsender_backfill is WAL replayed from the retained wire buffer to a reconnecting shadow walreceiver.", + "flow", + WalFlow::ALL + .into_iter() + .map(|f| (f.label(), snap.wal_bytes[f.index()])), + )?; + + gauge_series( + enc, + "walshadow_buffer_bytes", + "Data held right now, by buffer. walsender_send_queue is the largest unsent walsender queue across connections with the reconnect backfill left out, read against the slow-client cap.", + "buffer", + Buffer::ALL + .into_iter() + .map(|b| (b.label(), snap.buffer_bytes[b.index()])), + )?; + counter_pairs( enc, "walshadow_worker_seconds_total", @@ -1604,7 +1642,9 @@ mod tests { shadow_apply_lag_bytes: 12345, shadow_apply_lag_seconds: 1.23456, shadow_stream_active_connections: 2, - shadow_stream_dropped_connections_total: 5, + shadow_stream_sockets_by_event: [9, 5, 3, 1], + wal_bytes: [4096], + buffer_bytes: [777], ..MetricsSnapshot::default() }; let body = render(snap); @@ -1617,9 +1657,19 @@ mod tests { assert!(body.contains("# HELP walshadow_shadow_stream_active_connections")); assert!(body.contains("# TYPE walshadow_shadow_stream_active_connections gauge")); assert!(body.contains("walshadow_shadow_stream_active_connections 2")); - assert!(body.contains("# HELP walshadow_shadow_stream_dropped_connections")); - assert!(body.contains("# TYPE walshadow_shadow_stream_dropped_connections counter")); - assert!(body.contains("walshadow_shadow_stream_dropped_connections_total 5")); + assert!(body.contains("# TYPE walshadow_shadow_stream_sockets counter")); + assert!(body.contains("walshadow_shadow_stream_sockets_total{event=\"accepted\"} 9")); + assert!( + body.contains("walshadow_shadow_stream_sockets_total{event=\"closed_slow_client\"} 5") + ); + assert!( + body.contains("walshadow_shadow_stream_sockets_total{event=\"closed_by_client\"} 3") + ); + assert!(body.contains("walshadow_shadow_stream_sockets_total{event=\"closed_error\"} 1")); + assert!(body.contains("# TYPE walshadow_wal_bytes counter")); + assert!(body.contains("walshadow_wal_bytes_total{flow=\"walsender_backfill\"} 4096")); + assert!(body.contains("# TYPE walshadow_buffer_bytes gauge")); + assert!(body.contains("walshadow_buffer_bytes{buffer=\"walsender_send_queue\"} 777")); } /// Unknown rate stays infinite on the wire. prometheus-client spells it diff --git a/src/source/shadow_stream.rs b/src/source/shadow_stream.rs index 568f6520..bfe4979c 100644 --- a/src/source/shadow_stream.rs +++ b/src/source/shadow_stream.rs @@ -141,13 +141,13 @@ pub struct AggregateLsn { pub min_apply_lsn: Option>, /// Connections past `START_REPLICATION`, the only ones carrying LSNs. pub active_connections: usize, - /// Monotonic count of sockets that entered the handshake. Above - /// `active_connections` means a client is connected but stalled before - /// `START_REPLICATION`, which reads as "nothing attached" otherwise. - pub accepted_total: u64, - /// Monotonic count of connections dropped by `slow_threshold` - /// overflow since process start. - pub dropped_total: u64, + pub socket_events: [u64; SocketEvent::COUNT], + /// Monotonic count of `START_REPLICATION` bytes replayed from the retained + /// wire buffer to reconnecting clients + pub backfill_bytes_total: u64, + /// Largest unsent send queue across connections, to read against + /// `slow_threshold` + pub send_queue_bytes: u64, /// Oldest branch an attached client is still reading: a historic /// connection is pinned to the branch its `ends_at` names, the rest to the /// served one. `None` with nothing attached. Beside the apply LSN it is @@ -156,6 +156,12 @@ pub struct AggregateLsn { pub replay_timeline: Option, } +impl AggregateLsn { + pub fn accepted(&self) -> u64 { + self.socket_events[SocketEvent::Accepted.index()] + } +} + /// Why a `START_REPLICATION` gets an `ErrorResponse` instead of a stream #[derive(Debug, Error, PartialEq, Eq)] pub enum Refused { @@ -184,6 +190,15 @@ impl Refused { } } +crate::label_class! { + pub enum SocketEvent { + Accepted => "accepted", + ClosedSlowClient => "closed_slow_client", + ClosedByClient => "closed_by_client", + ClosedError => "closed_error", + } +} + #[derive(Debug, Error)] pub enum ShadowStreamError { #[error("io: {0}")] @@ -217,10 +232,8 @@ pub struct ShadowStreamState { pub slow_threshold: usize, /// Populates `'w'`/`'k'` frame headers. pub server_wal_end: Pos, - /// Surfaced via [`AggregateLsn::dropped_total`] for the - /// `walshadow_shadow_stream_dropped_connections_total` gauge. - dropped_total: u64, - accepted_total: u64, + socket_events: [u64; SocketEvent::COUNT], + backfill_bytes_total: u64, /// Wire bytes `[wire_buf_start, server_wal_end]` not yet on disk, used to /// backfill a reconnect behind the live head (else it gets an unappliable /// gap and strands at segment boundaries). Spans up to two segments while @@ -255,8 +268,8 @@ impl ShadowStreamState { send_queues: HashMap::new(), slow_threshold, server_wal_end: Pos::new(current_lsn), - dropped_total: 0, - accepted_total: 0, + socket_events: [0; SocketEvent::COUNT], + backfill_bytes_total: 0, wire_buf: Vec::new(), wire_buf_start: current_lsn, queued: Monotone::default(), @@ -301,8 +314,17 @@ impl ShadowStreamState { min_flush_lsn: (active > 0).then_some(min_flush), min_apply_lsn: (active > 0).then_some(min_apply), active_connections: active, - accepted_total: self.accepted_total, - dropped_total: self.dropped_total, + socket_events: self.socket_events, + backfill_bytes_total: self.backfill_bytes_total, + send_queue_bytes: self + .send_queues + .iter() + .map(|(id, q)| { + let backfill = self.connections.get(id).map_or(0, |c| c.backfill_queued); + q.len().saturating_sub(backfill) + }) + .max() + .unwrap_or(0) as u64, replay_timeline: (active > 0).then_some(branch), } } @@ -311,8 +333,8 @@ impl ShadowStreamState { /// `register_connection`: per-connection state needs the `START_REPLICATION` /// LSN, and a placeholder would feed boundary holds a fabricated apply /// point. - pub fn note_accepted(&mut self) { - self.accepted_total += 1; + pub fn note_socket(&mut self, event: SocketEvent) { + self.socket_events[event.index()] += 1; } /// Refuse a `START_REPLICATION` this walsender cannot serve gap-free @@ -551,6 +573,7 @@ impl ShadowStreamState { .get_mut(&id) .expect("backfill follows registration") .backfill_queued = self.send_queues.get(&id).map_or(0, Vec::len); + self.backfill_bytes_total += backfill.len() as u64; self.advance_dispatched(id, Pos::new(server_wal_end)); } @@ -623,10 +646,8 @@ impl ShadowStreamState { } fn overflow(&mut self, id: u64) { - if let Some(c) = self.connections.get_mut(&id) - && c.phase.close() - { - self.dropped_total += 1; + if let Some(c) = self.connections.get_mut(&id) { + c.phase.close(); } self.send_queues.remove(&id); } @@ -830,16 +851,32 @@ async fn handle_connection( /// Generic over the socket transport so unix + TCP share the logic. async fn drive_connection( - mut sock: S, + sock: S, state: Arc>, flush_interval: Duration, ) -> Result<(), ShadowStreamError> +where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send, +{ + let ended = serve_connection(sock, state.clone(), flush_interval).await; + state.lock().await.note_socket(match &ended { + Ok(event) => *event, + Err(_) => SocketEvent::ClosedError, + }); + ended.map(|_| ()) +} + +async fn serve_connection( + mut sock: S, + state: Arc>, + flush_interval: Duration, +) -> Result where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send, { let identity = { let mut s = state.lock().await; - s.note_accepted(); + s.note_socket(SocketEvent::Accepted); s.identity() }; @@ -867,7 +904,7 @@ where let mut err = bytes::BytesMut::new(); server::encode_error_response(&mut err, refused.sqlstate(), &refused.to_string()); conn.write_framed(&err).await?; - return Ok(()); + return Ok(SocketEvent::ClosedError); } }; tracing::info!( @@ -881,7 +918,8 @@ where let outcome = run_connection_loop(&mut conn, state.clone(), id, flush_interval).await; state.lock().await.drop_connection(id); match outcome? { - StreamOutcome::Closed => return Ok(()), + StreamOutcome::Closed => return Ok(SocketEvent::ClosedByClient), + StreamOutcome::Slow => return Ok(SocketEvent::ClosedSlowClient), StreamOutcome::TimelineEnded(switch) => { conn.end_timeline(switch).await?; let identity = state.lock().await.identity(); @@ -893,8 +931,10 @@ where /// How one stream on a walsender connection finished. enum StreamOutcome { - /// Client gone, or the connection dropped for slowness + /// Client gone Closed, + /// Dropped for overflowing `slow_threshold` + Slow, /// Served through a historic branch's switchpoint; the client is owed the /// next-timeline result TimelineEnded(TimelineSwitch), @@ -987,7 +1027,7 @@ where return Ok(StreamOutcome::TimelineEnded(switch)); } if closing { - return Ok(StreamOutcome::Closed); + return Ok(StreamOutcome::Slow); } } } @@ -1226,15 +1266,15 @@ mod tests { #[test] fn accepted_counts_sockets_short_of_start_replication() { let mut s = fresh_state(); - s.note_accepted(); + s.note_socket(SocketEvent::Accepted); let agg = s.aggregate(); - assert_eq!(agg.accepted_total, 1); + assert_eq!(agg.accepted(), 1); assert_eq!(agg.active_connections, 0, "handshake carries no LSN"); assert_eq!(agg.min_apply_lsn, None, "holds must not see a fake apply"); s.register_connection(0x1000, 1, None) .expect("current timeline"); let agg = s.aggregate(); - assert_eq!(agg.accepted_total, 1, "monotonic, not decremented"); + assert_eq!(agg.accepted(), 1, "monotonic, not decremented"); assert_eq!(agg.active_connections, 1); } @@ -1291,7 +1331,6 @@ mod tests { assert!(!s.enqueue(id, vec![0u8; 64])); assert!(s.connections[&id].phase.is_closing()); assert!(!s.send_queues.contains_key(&id)); - assert_eq!(s.aggregate().dropped_total, 1); } #[test] @@ -1311,16 +1350,6 @@ mod tests { assert_eq!(s.timeline_ended_for(id), Some(switch)); } - #[test] - fn dropped_total_increments_once_per_connection() { - let mut s = ShadowStreamState::new(1, "x".into(), 0, 64); - let id = s.register_connection(0, 1, None).expect("current timeline"); - assert!(!s.enqueue(id, vec![0u8; 128])); - // second overflow on the closing slot must not double-count - assert!(!s.enqueue(id, vec![0u8; 128])); - assert_eq!(s.aggregate().dropped_total, 1); - } - #[tokio::test(flavor = "current_thread")] async fn sink_dispatches_one_wire_chunk_per_active_connection() { let state = Arc::new(Mutex::new(fresh_state())); @@ -1420,6 +1449,59 @@ mod tests { } } + #[tokio::test(flavor = "current_thread")] + async fn backfill_is_counted_and_starts_below_the_buffer_are_refused() { + let state = Arc::new(Mutex::new(fresh_state())); + let mut sink = ShadowStreamSink::new(state.clone()); + sink.on_wire_chunk(0x1000, b"AAAA").await.unwrap(); + sink.on_segment_retired(0x1004).await.unwrap(); + sink.on_wire_chunk(0x1004, b"CCCC").await.unwrap(); + let mut s = state.lock().await; + s.register_connection(0x1008, 1, None).unwrap(); + assert_eq!(s.aggregate().backfill_bytes_total, 0); + s.register_connection(0x1004, 1, None).unwrap(); + assert_eq!(s.aggregate().backfill_bytes_total, 4); + assert!(matches!( + s.register_connection(0x1000, 1, None), + Err(Refused::Archived { .. }) + )); + assert_eq!(s.aggregate().backfill_bytes_total, 4); + } + + #[test] + fn send_queue_bytes_leaves_out_the_reconnect_backfill() { + let mut s = ShadowStreamState::new(1, "x".into(), 0, 64); + s.dispatch_wire(0, b"AAAABBBB"); + let id = s.register_connection(4, 1, None).unwrap(); + assert!(s.aggregate().backfill_bytes_total > 0); + assert_eq!(s.aggregate().send_queue_bytes, 0); + assert!(s.enqueue(id, vec![0u8; 10])); + assert_eq!(s.aggregate().send_queue_bytes, 10); + } + + #[test] + fn send_queue_bytes_is_the_largest_queue() { + let mut s = ShadowStreamState::new(1, "x".into(), 0, 64); + let a = s.register_connection(0, 1, None).unwrap(); + let b = s.register_connection(0, 1, None).unwrap(); + assert!(s.enqueue(a, vec![0u8; 10])); + assert!(s.enqueue(b, vec![0u8; 30])); + assert_eq!(s.aggregate().send_queue_bytes, 30); + s.drain_send_queue(b); + assert_eq!(s.aggregate().send_queue_bytes, 10); + } + + #[test] + fn socket_events_count_per_kind() { + let mut s = fresh_state(); + s.note_socket(SocketEvent::ClosedSlowClient); + s.note_socket(SocketEvent::ClosedByClient); + s.note_socket(SocketEvent::ClosedByClient); + s.note_socket(SocketEvent::ClosedError); + assert_eq!(s.aggregate().socket_events, [0, 1, 2, 1]); + assert_eq!(SocketEvent::ClosedByClient.label(), "closed_by_client"); + } + /// A rebuilt `WalStream` replays its segment over retained bytes, which /// keeps `wire_buf_start` and the backfill gap-free #[tokio::test(flavor = "current_thread")] @@ -1443,9 +1525,9 @@ mod tests { s.dispatch_wire(16, b"EEEEEEEE"); s.swap_send_queue(id, &mut Vec::new()); s.dispatch_wire(24, b"FFFFFFFF"); - assert_eq!(s.aggregate().dropped_total, 0); + assert!(!s.connections[&id].phase.is_closing()); s.dispatch_wire(32, b"GGGGGGGG"); - assert_eq!(s.aggregate().dropped_total, 1); + assert!(s.connections[&id].phase.is_closing()); } #[tokio::test(flavor = "current_thread")] @@ -1481,7 +1563,6 @@ mod tests { let id = s.register_connection(0, 1, Some(switch)).unwrap(); s.dispatch_wire(0, b"AAAABBBB"); assert!(s.connections[&id].phase.is_closing()); - assert_eq!(s.aggregate().dropped_total, 1); assert_eq!(s.timeline_ended_for(id), Some(switch)); } @@ -1591,13 +1672,21 @@ mod tests { applied_past(&source.state, Pos::ZERO).await; source.state.lock().await.dispatch_wire(0, &[0; 128]); applied_past(&source.state, Pos::new(1)).await; - assert_eq!(source.state.lock().await.aggregate().dropped_total, 1); + assert_eq!(source.state.lock().await.aggregate().active_connections, 0); let mut buf = Vec::new(); assert!( feed.next_event(StandbyStatus::default(), &mut buf) .await .is_err() ); + let slow = SocketEvent::ClosedSlowClient.index(); + for _ in 0..200 { + if source.state.lock().await.aggregate().socket_events[slow] == 1 { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("slow client close was never counted"); } /// Refusals arrive inside CopyBoth with the SQLSTATE a PG walsender uses, diff --git a/tests/bin_stream_e2e.rs b/tests/bin_stream_e2e.rs index a4c89dca..b678153f 100644 --- a/tests/bin_stream_e2e.rs +++ b/tests/bin_stream_e2e.rs @@ -366,7 +366,9 @@ async fn bin_stream_replicates_segments_and_serves_metrics() { "walshadow_shadow_apply_lag_bytes", "walshadow_shadow_apply_lag_seconds", "walshadow_shadow_stream_active_connections", - "walshadow_shadow_stream_dropped_connections_total", + "walshadow_shadow_stream_sockets_total", + "walshadow_wal_bytes_total", + "walshadow_buffer_bytes", ] { assert!( body.contains(name), diff --git a/tests/walsender_restore_miss.rs b/tests/walsender_restore_miss.rs index b259d258..294cdf10 100644 --- a/tests/walsender_restore_miss.rs +++ b/tests/walsender_restore_miss.rs @@ -101,7 +101,7 @@ async fn restore_miss_below_the_backfill_never_panics_shadow() { let mut tail = Some(tail); loop { let agg = state.lock().await.aggregate(); - if agg.accepted_total > 0 && !sealed.exists() { + if agg.accepted() > 0 && !sealed.exists() { std::fs::copy(src_wal.join(segment_name(missed)), &sealed).unwrap(); } if agg.active_connections > 0