From c39e676d5baba7f3b457f869152de3a78187c12c Mon Sep 17 00:00:00 2001 From: Kaushik Iska Date: Tue, 20 Oct 2026 09:16:51 -0500 Subject: [PATCH] feat(metrics): export commit-time lag gauges Carry the commit record's xact_time with the emitter ack frontier and export it as walshadow_emitter_ack_commit_timestamp_seconds, plus the newest drained commit as walshadow_source_last_commit_timestamp_seconds. Both are Unix seconds, NaN until the first commit, so replication lag is a plain subtraction with no PromQL rate window. --- docs/operations.md | 15 +++++ src/backfill/wal_replay.rs | 9 ++- src/bin/stream/metrics_publish.rs | 4 ++ src/bin/stream/session.rs | 3 + src/emit/pipeline/ack.rs | 94 ++++++++++++++++++++++++++----- src/emit/pipeline/bootstrap.rs | 4 +- src/emit/pipeline/reorder.rs | 6 +- src/filter/engine.rs | 25 ++++++++ src/ops/metrics.rs | 57 +++++++++++++++++++ tests/emitter_budget_flush.rs | 2 +- tests/emitter_native_types.rs | 2 +- tests/emitter_tls.rs | 2 +- 12 files changed, 199 insertions(+), 24 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 47456290..3061a52a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -59,6 +59,21 @@ primary LSN scraped elsewhere shows a sawtooth at high WAL rates. A gap in `source_head` means walshadow cannot see the source head, not that it has caught up. +Lag in time rather than bytes: + +| `walshadow_commit_timestamp_seconds{at}` | Meaning | +|---|---| +| `at="source"` | commit time of the newest source transaction the pump has read; absent until the first commit | +| `at="ack"` | commit time of the newest transaction ClickHouse has acknowledged; advances when `position_lsn{at="ack"}` passes a commit and holds while the ack moves over idle WAL, absent until the first acked commit | + +`commit_timestamp_seconds{at="source"} - ignoring(at) commit_timestamp_seconds{at="ack"}` +is how far behind the source's own clock ClickHouse is, and stays correct while +the source is idle. `time() - commit_timestamp_seconds{at="ack"}` also counts +source idleness as lag. The two views answer different questions: bytes say how +much WAL is still to process, seconds say how stale the destination is. Bytes +lag is absent during archive recovery; seconds lag keeps working there, since +archived commits carry their timestamps. + Set `--metrics-bind` to expose OpenMetrics text metrics, which Prometheus scrapes directly. Counter series carry `_total`. Docker deployment uses port 9484 and includes optional provisioned dashboard diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index 8e863adb..0886091e 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -624,9 +624,12 @@ impl WalReplaySink { let value_permit = detoast_heap(&mut heap, spool, &ref_maps, &self.resolver) .await .map_err(SinkError::from)?; - let (open, rows) = self - .open - .get_or_insert_with(|| (self.seqs.open(commit_lsn, Publish::Commit), 0)); + let (open, rows) = self.open.get_or_insert_with(|| { + ( + self.seqs.open(commit_lsn, Some(commit_ts), Publish::Commit), + 0, + ) + }); *rows += 1; let seq = open.seq(); if !route.drops_deletes() diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 46e20fe8..ebd8d7d7 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -277,6 +277,8 @@ pub(crate) async fn populate_metrics( shadow_replay_lsn: Pos, decoder_commit_lsn: Pos, emitter_ack_lsn: Pos, + ack_commit_ts: Option, + source_commit_ts: Option, rec_metrics: &MetricsRecordSink, pump_queue_depth: u64, queue_records_out_total: u64, @@ -302,6 +304,8 @@ pub(crate) async fn populate_metrics( shadow_replay_lsn, decoder_commit_lsn, emitter_ack_lsn, + emitter_ack_commit_timestamp_seconds: ack_commit_ts, + source_last_commit_timestamp_seconds: source_commit_ts, source_endpoint_swaps_total: source_swap.swaps, source_endpoint_swap_failures_total: source_swap.failures, source_endpoint_swap_pending: u64::from(source_swap.pending), diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index 5afcc3dd..b4eb3254 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -2323,6 +2323,7 @@ async fn session_body( // drained, not the top-of-iteration values. let emitter_ack_for_metric = emitter_ack.get(); let drain_for_metric = xact_stats.drain_lsn; + let ack_commit_ts = ack_probe.borrow().ack_commit_ts; let source_wal_end = matches!(path, SourcePath::Live).then(|| feed.last_server_wal_end()); populate_metrics( &metrics, @@ -2332,6 +2333,8 @@ async fn session_body( shadow_replay, drain_for_metric, emitter_ack_for_metric, + ack_commit_ts, + stream.filter().stats().last_commit_ts, &record_sink.metrics, record_sink.decoder_xact.in_flight(), record_sink.decoder_xact.processed(), diff --git a/src/emit/pipeline/ack.rs b/src/emit/pipeline/ack.rs index c11f5ecb..edabd7d3 100644 --- a/src/emit/pipeline/ack.rs +++ b/src/emit/pipeline/ack.rs @@ -40,6 +40,9 @@ pub enum AckEvent { Register { seq: u64, commit_lsn: u64, + /// Commit record's `xact_time` (PG epoch µs); `None` for aborts and + /// bootstrap seqs, which carry no source commit time + commit_ts: Option, publish: bool, }, Placed { @@ -100,6 +103,7 @@ impl Progress { struct Seq { commit_lsn: u64, + commit_ts: Option, publish: bool, progress: Progress, } @@ -120,6 +124,9 @@ pub struct AckSnapshot { pub frontier: u64, pub oldest_incomplete: Option, pub trailing_held: Pos, + /// `xact_time` (PG epoch µs) of the newest commit the watermark has + /// retired; `None` until the first one + pub ack_commit_ts: Option, /// Protocol faults; the first trips pipeline fatal pub wedged: u64, /// Events for retired seqs, benign: inserter batches straddle retirement @@ -147,6 +154,8 @@ pub struct AckState { registered: u64, /// Highest publishing `commit_lsn` retired from map watermark: u64, + /// Newest published commit time, advances with `watermark` + watermark_ts: Option, /// Retained across events because quiescent source cannot resend it trailing: u64, emitter_ack: Arc>, @@ -170,6 +179,7 @@ impl AckState { frontier: 0, registered: 0, watermark: 0, + watermark_ts: None, trailing: 0, emitter_ack, frontier_cell, @@ -190,8 +200,9 @@ impl AckState { AckEvent::Register { seq, commit_lsn, + commit_ts, publish, - } => self.open_seq(seq, commit_lsn, publish), + } => self.open_seq(seq, commit_lsn, commit_ts, publish), AckEvent::Placed { seq, rows } => self.step(seq, |p| p.place(rows)), AckEvent::Acked { counts } => { for (seq, n) in counts { @@ -207,7 +218,7 @@ impl AckState { } } - fn open_seq(&mut self, seq: u64, commit_lsn: u64, publish: bool) { + fn open_seq(&mut self, seq: u64, commit_lsn: u64, commit_ts: Option, publish: bool) { // Preserve live counts, never resurrect retired seqs if seq < self.frontier || self.map.contains_key(&seq) { self.wedge(seq, "seq registered twice"); @@ -217,6 +228,7 @@ impl AckState { seq, Seq { commit_lsn, + commit_ts, publish, progress: Progress::default(), }, @@ -265,10 +277,11 @@ impl AckState { if !s.progress.is_done() { break; } - let (publish, commit_lsn) = (s.publish, s.commit_lsn); + let (publish, commit_lsn, commit_ts) = (s.publish, s.commit_lsn, s.commit_ts); self.map.remove(&self.frontier); if publish { self.watermark = self.watermark.max(commit_lsn); + self.watermark_ts = self.watermark_ts.max(commit_ts); } self.frontier += 1; } @@ -292,6 +305,7 @@ impl AckState { frontier: self.frontier, oldest_incomplete: self.oldest_incomplete(), trailing_held: Pos::new(self.trailing), + ack_commit_ts: self.watermark_ts, wedged: self.wedged, late: self.late, } @@ -401,12 +415,15 @@ impl SeqAlloc { self.next } - pub fn open(&mut self, commit_lsn: u64, publish: Publish) -> OpenSeq { + /// `commit_ts` is the commit record's `xact_time`, which the ack frontier + /// carries once the seq publishes + pub fn open(&mut self, commit_lsn: u64, commit_ts: Option, publish: Publish) -> OpenSeq { let seq = self.next; self.next += 1; let ev = AckEvent::Register { seq, commit_lsn, + commit_ts, publish: publish == Publish::Commit, }; if let Some(buffered_events) = &mut self.buffered_events { @@ -418,8 +435,8 @@ impl SeqAlloc { } /// Ordering marker carrying no rows - pub fn mark(&mut self, commit_lsn: u64) { - let seq = self.open(commit_lsn, Publish::Commit); + pub fn mark(&mut self, commit_lsn: u64, commit_ts: Option) { + let seq = self.open(commit_lsn, commit_ts, Publish::Commit); self.place(seq, 0); } @@ -508,10 +525,11 @@ pub fn spawn(emitter_ack: Arc>, fatal: Fatal) -> (AckHandle #[cfg(test)] impl AckState { - fn register(&mut self, seq: u64, commit_lsn: u64, publish: bool) { + fn register(&mut self, seq: u64, commit_lsn: u64, commit_ts: Option, publish: bool) { self.apply(AckEvent::Register { seq, commit_lsn, + commit_ts, publish, }); } @@ -550,7 +568,7 @@ mod tests { let (mut s, ack) = state(); let n = 50_000u64; for seq in 0..n { - s.register(seq, (seq + 1) * 10, true); + s.register(seq, (seq + 1) * 10, None, true); s.placed(seq, 1); } assert_eq!(ack.get(), 0); @@ -565,7 +583,7 @@ mod tests { #[test] fn probe_names_what_holds_the_watermark() { let (mut s, _ack) = state(); - s.register(0, 100, true); + s.register(0, 100, None, true); let snap = s.snapshot(); assert_eq!(snap.stall_reason(), Some("rows not yet placed")); assert_eq!( @@ -589,10 +607,59 @@ mod tests { assert_eq!((snap.wedged, snap.late), (0, 0)); } + #[test] + fn commit_ts_advances_with_the_contiguous_frontier() { + let (mut s, ack) = state(); + assert_eq!(s.snapshot().ack_commit_ts, None); + s.register(0, 100, Some(1_000), true); + s.register(1, 200, Some(2_000), true); + s.placed(0, 1); + s.placed(1, 1); + // seq 1 done first: seq 0 gates both the LSN and its timestamp + s.acked(1, 1); + assert_eq!((ack.get().get(), s.snapshot().ack_commit_ts), (0, None)); + s.acked(0, 1); + assert_eq!( + (ack.get().get(), s.snapshot().ack_commit_ts), + (200, Some(2_000)) + ); + } + + #[test] + fn commit_ts_keeps_the_newest_and_ignores_trailing() { + let (mut s, ack) = state(); + s.register(0, 100, Some(2_000), true); + s.register(1, 200, Some(1_000), true); + for seq in 0..2 { + s.placed(seq, 1); + s.acked(seq, 1); + } + assert_eq!(s.snapshot().ack_commit_ts, Some(2_000)); + s.trailing(900); + assert_eq!( + (ack.get().get(), s.snapshot().ack_commit_ts), + (900, Some(2_000)) + ); + } + + #[test] + fn partial_slices_and_untimed_seqs_do_not_move_commit_ts() { + let (mut s, _ack) = state(); + s.register(0, 100, Some(1_000), true); + // non-final slice of a later commit, then an abort-style untimed seq + s.register(1, 200, Some(5_000), false); + s.register(2, 300, None, true); + for seq in 0..3 { + s.placed(seq, 1); + s.acked(seq, 1); + } + assert_eq!(s.snapshot().ack_commit_ts, Some(1_000)); + } + #[test] fn events_for_retired_seqs_are_benign() { let (mut s, _ack) = state(); - s.register(0, 100, true); + s.register(0, 100, None, true); s.placed(0, 1); s.acked(0, 1); s.acked(0, 1); @@ -604,14 +671,14 @@ mod tests { #[test] fn illegal_transitions_pin_the_seq_instead_of_being_absorbed() { let (mut s, ack) = state(); - s.register(0, 100, true); + s.register(0, 100, None, true); s.placed(0, 2); assert!(!s.fatal.is_set()); s.placed(0, 5); let first = s.fatal.message().expect("first fault trips fatal"); assert!(first.contains("seq placed twice"), "{first}"); s.acked(0, 3); - s.register(0, 100, true); + s.register(0, 100, None, true); assert_eq!(s.snapshot().wedged, 3); assert!(!s.all_done(), "a faulted seq stays incomplete"); assert_eq!(ack.get(), 0); @@ -623,7 +690,7 @@ mod tests { #[test] fn acks_ahead_of_placement_past_row_count_wedge() { let (mut s, ack) = state(); - s.register(0, 100, true); + s.register(0, 100, None, true); s.acked(0, 3); s.placed(0, 2); let msg = s.fatal.message().expect("over-ack trips fatal"); @@ -776,6 +843,7 @@ mod interleavings { Ev::Reg { seq, lsn, publish } => AckEvent::Register { seq, commit_lsn: lsn, + commit_ts: None, publish, }, Ev::Place { seq, rows } => AckEvent::Placed { seq, rows }, diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index f533ed71..88f421db 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -224,7 +224,7 @@ pub async fn drain( } slot.insert(Run { rfn, - seq: seqs.open(tuple.source_lsn, Publish::Commit), + seq: seqs.open(tuple.source_lsn, None, Publish::Commit), rows: 0, target: Target::resolve(&catalog, &routes, &skip_initial, rfn), }) @@ -632,7 +632,7 @@ async fn route_batch( } let mut tuple = row.tuple; let at = seq - .get_or_insert_with(|| seqs.open(tuple.source_lsn, Publish::Commit)) + .get_or_insert_with(|| seqs.open(tuple.source_lsn, None, Publish::Commit)) .seq(); let RowTable { rel, route } = &tables[row.table as usize]; render_ext_columns(&rel.attributes, &mut tuple.columns); diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index 3b3aa7b7..715692e1 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -906,7 +906,7 @@ impl ReorderSink { let heaps = std::mem::take(pending); let bytes = std::mem::take(pending_bytes); let permit = self.out.admit(bytes).await?.map(Arc::new); - let seq = self.out.seqs.open(commit_lsn, publish); + let seq = self.out.seqs.open(commit_lsn, Some(commit_ts), publish); self.stats.queue_jobs_out.fetch_add(1, Ordering::Relaxed); let chunk_rows = self.dest.current().decode_chunk_rows; let rows = decode::place_rows( @@ -1243,7 +1243,7 @@ impl ReorderSink { // rows=0 marker: publishes commit_lsn once every earlier partial // segment is durable. Covers empty / read-only commits and plans // whose tail is a control or truncate. - self.out.seqs.mark(commit_lsn); + self.out.seqs.mark(commit_lsn, Some(commit_ts)); } Ok(()) } @@ -1256,7 +1256,7 @@ impl ReorderSink { // ABORT PREPARED: buffered state keys off the prepared xid let xid = payload.twophase_xid.unwrap_or(xid); self.note_pending(xid, &payload.subxacts, false).await?; - let seq = self.out.seqs.open(record.source_lsn, Publish::Commit); + let seq = self.out.seqs.open(record.source_lsn, None, Publish::Commit); { let mut buf = self.buffer.lock().await; buf.abort(xid, Pos::new(record.source_lsn), &payload.subxacts) diff --git a/src/filter/engine.rs b/src/filter/engine.rs index e731b703..3084887b 100644 --- a/src/filter/engine.rs +++ b/src/filter/engine.rs @@ -48,6 +48,8 @@ pub struct FilterStats { pub kept_user: u64, pub kept_special: u64, pub kept_empty: u64, + /// Newest commit record `xact_time` (PG epoch µs) routed + pub last_commit_ts: Option, } impl FilterStats { @@ -539,6 +541,9 @@ impl Filter { return Ok(XactEnd::default()); } let payload = parse_xact_payload(info, &record.main_data, page_magic)?; + if is_commit { + self.stats.last_commit_ts = self.stats.last_commit_ts.max(Some(payload.xact_time)); + } let header_xid = record.header.xact_id; self.creating_dbs.remove(&header_xid); let (merged, members) = @@ -1283,6 +1288,26 @@ mod tests { assert!(v.boundary.is_none(), "dirty mark consumed once"); } + #[test] + fn last_commit_ts_tracks_newest_commit_only() { + let at = |op, xid, ts: i64| { + let mut r = xact_end(op, xid, &[], None); + r.main_data.to_mut()[..8].copy_from_slice(&ts.to_le_bytes()); + r + }; + let mut f = target_filter(); + assert_eq!(f.stats().last_commit_ts, None); + f.decide_record(&at(XLOG_XACT_ABORT, 6, 100), 0, 0xD116) + .unwrap(); + assert_eq!(f.stats().last_commit_ts, None); + f.decide_record(&at(XLOG_XACT_COMMIT, 7, 300), 0, 0xD116) + .unwrap(); + // Backends stamp xact_time before inserting WAL, so commits can land out of order + f.decide_record(&at(XLOG_XACT_COMMIT, 8, 200), 0, 0xD116) + .unwrap(); + assert_eq!(f.stats().last_commit_ts, Some(300)); + } + #[test] fn abort_clears_dirty_without_boundary() { let mut f = target_filter(); diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 8ddedc17..2f225df9 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -128,6 +128,13 @@ label_class! { } } +label_class! { + pub enum CommitPoint { + Source => "source", + Ack => "ack", + } +} + label_class! { pub enum WorkerState { Busy => "busy", @@ -177,6 +184,7 @@ macro_rules! encode_metric { ($enc:expr, histogram, $name:expr, $help:expr, $value:expr) => { histogram($enc, $name, $help, &$value)? }; + // Same for a commit time not seen yet: a 0 sample reads as 56 years of lag ($enc:expr, counter, $name:expr, $help:expr, $value:expr, $key:literal, $labels:expr) => { counter_series($enc, $name, $help, $key, $labels.into_iter().zip($value))? }; @@ -351,6 +359,12 @@ snapshot! { timeline_switch_lsn: u64, pause_consumed_lsn: u64, pause_received_lsn: u64, + /// `xact_time` in PG epoch µs of the newest commit ClickHouse acked + /// and the newest the pump read, rendered + /// `walshadow_commit_timestamp_seconds{at}` in Unix seconds, each + /// only once a commit has been seen + emitter_ack_commit_timestamp_seconds: Option, + source_last_commit_timestamp_seconds: Option, } gauge xact_active: u64 = "Active transactions buffered in memory or on spill.", @@ -618,6 +632,11 @@ snapshot! { "Cumulative seconds the pump parked in released catalog-boundary holds.", } +/// `TimestampTz` (microseconds since 2000-01-01) as Unix seconds +fn unix_seconds(pg_us: i64) -> f64 { + pg_us.saturating_add(walrus::pg::replication::PG_EPOCH_USEC) as f64 / 1e6 +} + #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct SkippedRel { pub database: String, @@ -1265,6 +1284,23 @@ fn encode_snapshot(snap: &MetricsSnapshot, enc: &mut DescriptorEncoder<'_>) -> f .map(|(at, lsn)| (at.label(), lsn)), )?; + let commit_clocks = [ + ( + CommitPoint::Source, + snap.source_last_commit_timestamp_seconds, + ), + (CommitPoint::Ack, snap.emitter_ack_commit_timestamp_seconds), + ]; + gauge_series( + enc, + "walshadow_commit_timestamp_seconds", + "Commit time in Unix seconds of the newest transaction at each point: source is the newest commit record the pump has read off the source, ack the newest transaction ClickHouse has acked. source minus ack is how far behind the source's clock ClickHouse is and stays correct on an idle source; time() minus ack also counts source idleness. A member is absent until its first commit.", + "at", + commit_clocks + .into_iter() + .filter_map(|(at, ts)| ts.map(|pg_us| (at.label(), unix_seconds(pg_us)))), + )?; + let mut records = declare( enc, "walshadow_filter_records_total", @@ -1949,6 +1985,27 @@ mod tests { ); } + #[test] + fn render_omits_commit_timestamps_until_the_first() { + let body = render(MetricsSnapshot { + // 2000-01-01T00:00:00Z plus 1.5 s + emitter_ack_commit_timestamp_seconds: Some(1_500_000), + ..MetricsSnapshot::default() + }); + assert!( + body.contains("# TYPE walshadow_commit_timestamp_seconds gauge"), + "{body}" + ); + assert!( + body.contains("walshadow_commit_timestamp_seconds{at=\"ack\"} 946684801.5"), + "{body}" + ); + assert!( + !body.contains("walshadow_commit_timestamp_seconds{at=\"source\"}"), + "{body}" + ); + } + #[test] fn rate_estimator_rate_across_window() { let mut e = RateEstimator::new(Duration::from_secs(30)); diff --git a/tests/emitter_budget_flush.rs b/tests/emitter_budget_flush.rs index 57d090f8..91cd5d60 100644 --- a/tests/emitter_budget_flush.rs +++ b/tests/emitter_budget_flush.rs @@ -182,7 +182,7 @@ async fn budget_trips_seal_complete_inserts() { const N: i32 = 5; let commit_lsn = 0xC0FFEE; let mut seqs = ack.seqs(0); - let seq = seqs.open(commit_lsn, Publish::Commit); + let seq = seqs.open(commit_lsn, None, Publish::Commit); let rows = (0..N) .map(|i| RoutedRow { seq: seq.seq(), diff --git a/tests/emitter_native_types.rs b/tests/emitter_native_types.rs index 1d4b0a8f..4406d6aa 100644 --- a/tests/emitter_native_types.rs +++ b/tests/emitter_native_types.rs @@ -174,7 +174,7 @@ async fn native_numeric_time_timetz_round_trip() { // Drive the single row through the tail: register the seq, route the // row, seal with FlushAll, wait for it durable, then drain. let mut seqs = ack.seqs(0); - let seq = seqs.open(tuple.commit_lsn, Publish::Commit); + let seq = seqs.open(tuple.commit_lsn, None, Publish::Commit); msg_tx .send(BatcherMsg::Rows(RowChunk { tables: vec![RowTable { diff --git a/tests/emitter_tls.rs b/tests/emitter_tls.rs index aecdb41c..36bd203d 100644 --- a/tests/emitter_tls.rs +++ b/tests/emitter_tls.rs @@ -375,7 +375,7 @@ async fn emitter_tls_round_trip() { commit_lsn: 0xABCD, }; let mut seqs = ack.seqs(0); - let seq = seqs.open(tuple.commit_lsn, Publish::Commit); + let seq = seqs.open(tuple.commit_lsn, None, Publish::Commit); msg_tx .send(BatcherMsg::Rows(RowChunk { tables: vec![RowTable {