diff --git a/bench/src/initial_load.rs b/bench/src/initial_load.rs index bc2d5dcc..e365717e 100644 --- a/bench/src/initial_load.rs +++ b/bench/src/initial_load.rs @@ -355,7 +355,7 @@ pub async fn run( bail!("daemon restarted while seeding"); } if sample - .get("walshadow_floor_lsn") + .get("walshadow_position_lsn{at=\"floor\"}") .is_some_and(|value| *value >= target as f64) && sample.get("walshadow_config_backfills_pending") == Some(&0.0) { diff --git a/docker/grafana/dashboards/walshadow.json b/docker/grafana/dashboards/walshadow.json index c3222a9a..b784fc3e 100644 --- a/docker/grafana/dashboards/walshadow.json +++ b/docker/grafana/dashboards/walshadow.json @@ -210,7 +210,7 @@ }, "targets": [ { - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)", + "expr": "clamp_min(walshadow_position_lsn{at=\"source_head\"} - ignoring(at) walshadow_position_lsn{at=\"ack\"}, 0)", "instant": true, "legendFormat": "backlog", "refId": "A" @@ -633,12 +633,12 @@ "refId": "A" }, { - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_decoder_commit_lsn, 0)", + "expr": "clamp_min(walshadow_position_lsn{at=\"received\"} - ignoring(at) walshadow_position_lsn{at=\"decoder_commit\"}, 0)", "legendFormat": "source to decoder", "refId": "B" }, { - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)", + "expr": "clamp_min(walshadow_position_lsn{at=\"source_head\"} - ignoring(at) walshadow_position_lsn{at=\"ack\"}, 0)", "legendFormat": "source to ClickHouse ack", "refId": "C" } diff --git a/docs/operations.md b/docs/operations.md index 72d08e1f..397b78ed 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -41,6 +41,24 @@ toward `source_received`. Short differences are expected while batches flush ## Monitor with Prometheus and Grafana +WAL positions are one family, `walshadow_position_lsn{at}`, all from +walshadow's own view and all written in one status tick: + +| `at` | Meaning | +|---|---| +| `source_head` | newest `server_wal_end` heard from the source socket; present only while `walshadow_source_mode{mode="streaming"}` is 1: archive catch-up, reconnects, pauses and timeline crossings leave it out, and so does a fresh connection until its first message | +| `received` | the further of the reported source head and the pump's dispatch frontier (what the pump has pushed into the decode queue), not WAL held locally | +| `filter`, `decoder_commit`, `ack` | filter dispatch, highest drained commit, and ClickHouse ack positions | +| `shadow_replay`, `floor` | shadow Postgres replay position and the durable resume floor | +| `pause_consumed`, `pause_received`, `promotion_target_replay`, `promotion_target_receive`, `timeline_switch` | present only during a pause, a promotion check, or after a timeline crossing | + +A member is absent while its position is unknown. Because every member comes +from the same tick, `position_lsn{at="source_head"} - ignoring(at) +position_lsn{at="ack"}` has no scrape skew, while subtracting the ack from a +primary LSN scraped elsewhere shows a sawtooth at high WAL rates. A gap in +`source_head` means walshadow cannot see the source head, not that it has +caught up. + Set `--metrics-bind` to expose OpenMetrics text metrics, which Prometheus scrapes directly. Counter series carry `_total`. Docker deployment uses port 9484 and includes optional provisioned dashboard diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 94778141..adb42066 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -10,7 +10,9 @@ use walshadow::boundary_hold::BoundaryHoldStats; use walshadow::ch_emitter::EmitterStats; use walshadow::config::ConfigResolver; use walshadow::metrics::WorkerState; -use walshadow::metrics::{DbSeries, MetricsRegistry, MetricsSnapshot, RateEstimator}; +use walshadow::metrics::{ + DbSeries, MetricsRegistry, MetricsSnapshot, RateEstimator, source_server_wal_end, +}; use walshadow::pipeline::row_ledger::TableRowCounts; use walshadow::pos::{ Drain, EmitterAck, FilterDispatched, Floor, Pos, ShadowReplay, SourceReceived, @@ -259,6 +261,7 @@ impl DrainResident { pub(crate) async fn populate_metrics( registry: &MetricsRegistry, source_received_lsn: Pos, + source_wal_end: Option, filter_lsn: Pos, shadow_replay_lsn: Pos, decoder_commit_lsn: Pos, @@ -283,6 +286,7 @@ pub(crate) async fn populate_metrics( .unwrap_or((prev.shadow_apply_lag_bytes, prev.shadow_apply_lag_seconds)); let base = MetricsSnapshot { source_received_lsn, + source_server_wal_end_lsn: source_server_wal_end(source_wal_end), filter_lsn, shadow_replay_lsn, decoder_commit_lsn, diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index b4722427..07e3af30 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -2343,9 +2343,13 @@ async fn session_body( // drained, not the top-of-iteration values. let emitter_ack_for_metric = emitter_ack.get(); let drain_for_metric = xact_stats.drain_lsn; + let streaming = + path.mode(paused, crossing.pending()) == walshadow::metrics::SourceMode::Streaming; + let source_wal_end = streaming.then(|| feed.last_server_wal_end()); populate_metrics( &metrics, received, + source_wal_end, Pos::new(now_dispatched), shadow_replay, drain_for_metric, diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 5bf0b84a..c4fb0bf0 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -110,6 +110,23 @@ label_class! { } } +label_class! { + pub enum Position { + SourceHead => "source_head", + Received => "received", + Filter => "filter", + DecoderCommit => "decoder_commit", + Ack => "ack", + ShadowReplay => "shadow_replay", + Floor => "floor", + PauseConsumed => "pause_consumed", + PauseReceived => "pause_received", + PromotionTargetReplay => "promotion_target_replay", + PromotionTargetReceive => "promotion_target_receive", + TimelineSwitch => "timeline_switch", + } +} + label_class! { pub enum WorkerState { Busy => "busy", @@ -156,15 +173,6 @@ macro_rules! encode_metric { ($enc:expr, gauge, $name:expr, $help:expr, $value:expr) => { gauge($enc, $name, $help, $value)? }; - // LSN 0 is InvalidXLogRecPtr: the position is not known yet, eg the - // default snapshot served from process start until the session's first - // status write. A 0 sample between real LSNs reads as a ~80 TB jump to - // deriv() and lag queries, so the sample is left out instead - ($enc:expr, lsn, $name:expr, $help:expr, $value:expr) => { - if $value.get() != 0 { - gauge($enc, $name, $help, $value)? - } - }; ($enc:expr, counter, $name:expr, $help:expr, $value:expr, $key:literal, $labels:expr) => { counter_series($enc, $name, $help, $key, $labels.into_iter().zip($value))? }; @@ -326,20 +334,21 @@ snapshot! { promotion_target_replay_lsn: u64, /// Target's `pg_last_wal_receive_lsn()`, read while paused promotion_target_receive_lsn: u64, + /// WAL positions rendered as `walshadow_position_lsn{at}`, each only + /// while known (0 is InvalidXLogRecPtr); all read in one status tick + /// so differences between them carry no scrape skew + source_received_lsn: Pos, + source_server_wal_end_lsn: Option, + filter_lsn: Pos, + shadow_replay_lsn: Pos, + decoder_commit_lsn: Pos, + emitter_ack_lsn: Pos, + floor_lsn: Pos, + timeline_switch_lsn: u64, + pause_consumed_lsn: u64, + pause_received_lsn: u64, } - lsn source_received_lsn: Pos = - "Source PG's most recent server_wal_end seen on the replication socket.", - /// Last segment-boundary LSN dispatched downstream; becomes durable once - /// segment fsync lands - lsn filter_lsn: Pos = - "LSN of the last filtered WAL byte the daemon has dispatched.", - lsn shadow_replay_lsn: Pos = - "Shadow PG's pg_last_wal_replay_lsn(), polled at status cadence.", - lsn decoder_commit_lsn: Pos = "Highest commit LSN drained out of the xact buffer.", - /// Live insert watermark, unlike manifest's resume-safe `emitter_ack` - lsn emitter_ack_lsn: Pos = "Contiguous-done watermark from the insert pipeline.", - lsn floor_lsn: Pos = "Durable resume floor: restart resumes here and pruners cut to it.", gauge xact_active: u64 = "Active transactions buffered in memory or on spill.", gauge xact_bytes_in_memory: u64 = "Bytes held in memory across all buffered xacts.", gauge spill_xacts_active: u64 = "Xacts with at least one entry currently in their spill file.", @@ -572,21 +581,9 @@ snapshot! { counter timeline_switch_failures_by_reason: [u64; SWITCH_FAILURE_REASONS.len()] ["reason" = SWITCH_FAILURE_REASONS] = "Crossings refused; the stream stays on the ancestor.", - /// Most recent fork point, diagnostic - gauge timeline_switch_lsn: u64 = "Fork LSN of the most recent timeline crossing.", counter timeline_prefix_bytes_verified_total: u64 = "Descendant bytes compared against the retained ancestor fork prefix.", counter timeline_transition_seconds_total: f64 = "Seconds spent inside timeline crossings.", - /// `WalStream::next_lsn` frozen when the pump observed `[stream] paused`, - /// which resume asks the source to serve. `0` while not paused: a value - /// left from an earlier pause cannot be compared against a promotion - /// decision either - gauge pause_consumed_lsn: u64 = - "Consumed frontier frozen when the pump observed the pause; 0 while running.", - /// Source head last heard about, frozen at the same instant. The promotion - /// target must reach it before it is promoted - gauge pause_received_lsn: u64 = - "Source head frozen when the pump observed the pause; 0 while running.", gauge shadow_served_timeline: u32 = "Timeline the shadow-facing walsender advertises.", /// Branch the shadow is replaying, which is how a shadow that followed the /// chain reads differently from one that merely survived @@ -861,6 +858,10 @@ const PLAN_FAILURE_REASONS: [&str; 11] = [ "drain", ]; +pub fn source_server_wal_end(wal_end: Option) -> Option { + wal_end.filter(|&end| end != 0) +} + /// `mode=` label order of [`DbSeries::config_backfills_pending_by_mode`] const BACKFILL_MODES: [&str; 3] = ["copy", "base_backup", "object_store"]; @@ -1117,6 +1118,40 @@ fn encode_snapshot(snap: &MetricsSnapshot, enc: &mut DescriptorEncoder<'_>) -> f .map(|(worker, state, secs)| ([*worker, state.label()], *secs)), )?; + let positions = [ + ( + Position::SourceHead, + snap.source_server_wal_end_lsn.unwrap_or(0), + ), + (Position::Received, snap.source_received_lsn.get()), + (Position::Filter, snap.filter_lsn.get()), + (Position::DecoderCommit, snap.decoder_commit_lsn.get()), + (Position::Ack, snap.emitter_ack_lsn.get()), + (Position::ShadowReplay, snap.shadow_replay_lsn.get()), + (Position::Floor, snap.floor_lsn.get()), + (Position::PauseConsumed, snap.pause_consumed_lsn), + (Position::PauseReceived, snap.pause_received_lsn), + ( + Position::PromotionTargetReplay, + snap.promotion_target_replay_lsn, + ), + ( + Position::PromotionTargetReceive, + snap.promotion_target_receive_lsn, + ), + (Position::TimelineSwitch, snap.timeline_switch_lsn), + ]; + gauge_series( + enc, + "walshadow_position_lsn", + "WAL byte positions, all published in one status tick so any two subtract without scrape skew. source_head is the newest server_wal_end heard on the source socket (present only while walshadow_source_mode is streaming: archive catch-up, reconnects, pauses and timeline crossings leave it out, and so does a fresh connection until its first message); received the source head floored at the filter dispatch frontier; filter the last dispatched filtered byte; decoder_commit the highest commit drained; ack the ClickHouse watermark; shadow_replay the shadow Postgres replay position; floor the durable resume floor. pause_consumed and pause_received are the frontiers frozen by a pause, promotion_target_replay and promotion_target_receive the target's positions read while paused, timeline_switch the fork of the last crossing. A member is absent while its position is unknown (0).", + "at", + positions + .into_iter() + .filter(|(_, lsn)| *lsn != 0) + .map(|(at, lsn)| (at.label(), lsn)), + )?; + let mut records = declare( enc, "walshadow_filter_records_total", @@ -1298,23 +1333,52 @@ mod tests { } #[test] - fn render_omits_unknown_lsn_gauges() { + fn render_omits_unknown_positions() { let body = render(MetricsSnapshot { emitter_ack_lsn: 0x47FD_FB00_0000.into(), + pause_received_lsn: 77, ..MetricsSnapshot::default() }); - assert!(body.contains("walshadow_emitter_ack_lsn 79156163379200")); - for name in [ - "walshadow_source_received_lsn", - "walshadow_filter_lsn", - "walshadow_shadow_replay_lsn", - "walshadow_decoder_commit_lsn", - "walshadow_floor_lsn", + assert!(body.contains("# TYPE walshadow_position_lsn gauge")); + assert!(body.contains("walshadow_position_lsn{at=\"ack\"} 79156163379200")); + assert!(body.contains("walshadow_position_lsn{at=\"pause_received\"} 77")); + for at in [ + "source_head", + "received", + "filter", + "shadow_replay", + "decoder_commit", + "floor", + "pause_consumed", + "promotion_target_replay", + "promotion_target_receive", + "timeline_switch", ] { - assert!(!body.contains(name), "{name} rendered at LSN 0: {body}"); + let line = format!("walshadow_position_lsn{{at=\"{at}\"}}"); + assert!(!body.contains(&line), "{at} rendered at LSN 0: {body}"); } } + #[test] + fn source_server_wal_end_is_absent_until_heard() { + for unknown in [None, Some(0)] { + assert_eq!(source_server_wal_end(unknown), None); + let body = render(MetricsSnapshot { + source_server_wal_end_lsn: source_server_wal_end(unknown), + ..MetricsSnapshot::default() + }); + assert!( + !body.contains("walshadow_position_lsn{at=\"source_head\"}"), + "{body}" + ); + } + let body = render(MetricsSnapshot { + source_server_wal_end_lsn: source_server_wal_end(Some(1_000)), + ..MetricsSnapshot::default() + }); + assert!(body.contains("walshadow_position_lsn{at=\"source_head\"} 1000")); + } + #[test] fn render_includes_help_type_lines() { let mut snap = MetricsSnapshot { @@ -1328,10 +1392,10 @@ mod tests { .insert(("Heap".into(), "to_decoder"), 17); let body = render(snap); - assert!(body.contains("# HELP walshadow_source_received_lsn")); - assert!(body.contains("# TYPE walshadow_source_received_lsn gauge")); - assert!(body.contains("walshadow_source_received_lsn 3405691582")); - assert!(body.contains("walshadow_filter_lsn 12648430")); + assert!(body.contains("# HELP walshadow_position_lsn")); + assert!(body.contains("# TYPE walshadow_position_lsn gauge")); + assert!(body.contains("walshadow_position_lsn{at=\"received\"} 3405691582")); + assert!(body.contains("walshadow_position_lsn{at=\"filter\"} 12648430")); assert!( body.contains("walshadow_filter_records_total{rmgr=\"Heap\",route=\"to_decoder\"} 17") ); @@ -1820,7 +1884,10 @@ mod tests { let resp = String::from_utf8(buf).unwrap(); assert!(resp.starts_with("HTTP/1.0 200 OK\r\n"), "{resp}"); assert!(resp.contains("Content-Type: application/openmetrics-text")); - assert!(resp.contains("walshadow_filter_lsn 57005"), "{resp}"); + assert!( + resp.contains("walshadow_position_lsn{at=\"filter\"} 57005"), + "{resp}" + ); let taken = serve(addr, MetricsRegistry::new()).await.unwrap_err(); assert!(matches!(taken, MetricsError::Bind { .. }), "{taken}"); } diff --git a/src/source/manifest.rs b/src/source/manifest.rs index a634479c..ad636ffb 100644 --- a/src/source/manifest.rs +++ b/src/source/manifest.rs @@ -42,7 +42,7 @@ //! * `drain`: highest commit-record LSN drained out of the xact buffer. //! Strictly higher than `emitter_ack`. //! * `emitter_ack`: [`ResumeSafe`], not the live ack behind -//! `walshadow_emitter_ack_lsn`. Slot-advance ceiling. +//! `walshadow_position_lsn{at="ack"}`. Slot-advance ceiling. //! * `shadow_flush`: min `flush_lsn` from inbound `'r'` standby status //! across active shadow streaming connections. On restart, resume //! position walsender hands shadow via `START_REPLICATION PHYSICAL diff --git a/tests/bin_stream_e2e.rs b/tests/bin_stream_e2e.rs index b678153f..c1b24dc3 100644 --- a/tests/bin_stream_e2e.rs +++ b/tests/bin_stream_e2e.rs @@ -357,7 +357,7 @@ async fn bin_stream_replicates_segments_and_serves_metrics() { // well-known walshadow_ counters. let body = http_get(metrics_addr, "/metrics").context("metrics scrape")?; assert!( - body.contains("walshadow_source_received_lsn") || body.contains("walshadow_uptime"), + body.contains("walshadow_position_lsn") || body.contains("walshadow_uptime"), "expected walshadow_* counter in /metrics body: {body}", ); // Shadow apply-lag surface is part of the static @@ -376,7 +376,7 @@ async fn bin_stream_replicates_segments_and_serves_metrics() { ); } // Absent until the session's first status write publishes a position - let ack_at_boot = metric_u64(&body, "walshadow_emitter_ack_lsn").unwrap_or(0); + let ack_at_boot = metric_u64(&body, "walshadow_position_lsn{at=\"ack\"}").unwrap_or(0); // 6. Drive workload. The daemon filters user-heap WAL records // (rmgr=HEAP, class=User → replaced with XLOG_NOOP) — only @@ -403,8 +403,9 @@ async fn bin_stream_replicates_segments_and_serves_metrics() { loop { let body = http_get(metrics_addr, "/metrics").context("ack poll scrape")?; // LSN gauges stay absent while their position is unknown - let ack = metric_u64(&body, "walshadow_emitter_ack_lsn").unwrap_or(0); - let replay = metric_u64(&body, "walshadow_shadow_replay_lsn").unwrap_or(0); + let ack = metric_u64(&body, "walshadow_position_lsn{at=\"ack\"}").unwrap_or(0); + let replay = + metric_u64(&body, "walshadow_position_lsn{at=\"shadow_replay\"}").unwrap_or(0); if ack > ack_at_boot && replay > 0 { break; } diff --git a/tests/common/bootstrap_ch_fixture.rs b/tests/common/bootstrap_ch_fixture.rs index b5a462c1..24c31bad 100644 --- a/tests/common/bootstrap_ch_fixture.rs +++ b/tests/common/bootstrap_ch_fixture.rs @@ -571,8 +571,12 @@ pub fn parse_metric(body: &str, name: &str) -> Option { continue; } let head = line.split_once(' ').map(|(h, _)| h)?; - let stem = head.split_once('{').map(|(s, _)| s).unwrap_or(head); - if stem != name { + let key = if name.contains('{') { + head + } else { + head.split_once('{').map(|(s, _)| s).unwrap_or(head) + }; + if key != name { continue; } let value_str = line.rsplit_once(' ').map(|(_, v)| v)?; diff --git a/tests/control_plane_e2e.rs b/tests/control_plane_e2e.rs index d3d6d439..89bf3ccd 100644 --- a/tests/control_plane_e2e.rs +++ b/tests/control_plane_e2e.rs @@ -1485,12 +1485,12 @@ async fn pause_publishes_frozen_frontier_lsns() { "consumed {consumed:X} above received {received:X}", ); assert_eq!( - h.metric("walshadow_pause_consumed_lsn")?, + h.metric("walshadow_position_lsn{at=\"pause_consumed\"}")?, consumed, "metric disagrees with ctl status", ); assert_eq!( - h.metric("walshadow_pause_received_lsn")?, + h.metric("walshadow_position_lsn{at=\"pause_received\"}")?, received, "metric disagrees with ctl status", ); @@ -2165,7 +2165,7 @@ async fn restart_after_switchover_resumes_on_both_sides_of_the_fork() { .await .context("pump never crossed the fork")?; // Captured now: the counters are per-process, so a restart zeroes them - let switch_lsn = h.metric("walshadow_timeline_switch_lsn")?; + let switch_lsn = h.metric("walshadow_position_lsn{at=\"timeline_switch\"}")?; assert!(switch_lsn > 0, "crossing reported no fork LSN"); // The shadow must be past the fork too, else the first restart is not // exercising the window @@ -2210,7 +2210,7 @@ async fn restart_after_switchover_resumes_on_both_sides_of_the_fork() { // Survives the restart because it comes off the source's history chain, // not off a per-process counter assert_eq!( - h.metric("walshadow_timeline_switch_lsn")?, + h.metric("walshadow_position_lsn{at=\"timeline_switch\"}")?, switch_lsn, "fork LSN did not come back from the timeline history", ); diff --git a/tests/kill_restart.rs b/tests/kill_restart.rs index 1dac9f4f..273b092d 100644 --- a/tests/kill_restart.rs +++ b/tests/kill_restart.rs @@ -14,7 +14,7 @@ //! 6. snapshot source's pg_current_wal_lsn + row state //! 7. restart the daemon with identical flags — same spill dir, same //! walsender bind (SO_REUSEADDR), no `--ignore-cursor` -//! 8. poll source + daemon's `walshadow_source_received_lsn` until the +//! 8. poll source + daemon's `walshadow_position_lsn{at="received"}` until the //! restarted pump catches up to the snapshotted LSN //! 9. poll CH count + sum(id) + md5(string_agg(name, ',' ORDER BY //! id)) until it matches source's @@ -318,7 +318,7 @@ async fn wait_for_first_commit(metrics_addr: SocketAddr, deadline: Duration) -> } /// Poll source's `pg_current_wal_lsn` + daemon's -/// `walshadow_source_received_lsn` until the restarted pump catches up +/// `walshadow_position_lsn{at="received"}` until the restarted pump catches up /// /// `emitter_ack` carries record start LSNs while `pg_current_wal_lsn` reports /// the last record's end, so an idle source never lifts it to the target @@ -332,7 +332,8 @@ async fn wait_for_received_catchup( let target = parse_pg_lsn(&target_text).context("parse source LSN")?; while start.elapsed() < deadline { if let Ok(body) = fx::http_get(metrics_addr, "/metrics") - && let Some(received) = fx::parse_metric(&body, "walshadow_source_received_lsn") + && let Some(received) = + fx::parse_metric(&body, "walshadow_position_lsn{at=\"received\"}") && received >= target { return Ok(received); diff --git a/tests/pgbench_acceptance.rs b/tests/pgbench_acceptance.rs index 65b1c346..918b6e78 100644 --- a/tests/pgbench_acceptance.rs +++ b/tests/pgbench_acceptance.rs @@ -423,7 +423,7 @@ fn run_ddl_intermix(ports: fx::Ports, decoder_pool: usize, inserter_pool: usize, )?)?; wait_for_metric_lsn( metrics_addr, - "walshadow_source_received_lsn", + "walshadow_position_lsn{at=\"received\"}", bootstrap_lsn, Duration::from_secs(120), ) @@ -492,7 +492,7 @@ fn run_ddl_intermix(ports: fx::Ports, decoder_pool: usize, inserter_pool: usize, // 14. Force a segment seal so the daemon's pump definitely // reaches every committed row in WAL, then poll the daemon's - // `walshadow_emitter_ack_lsn` until it crosses source's + // `walshadow_position_lsn{at="ack"}` until it crosses source's // pre-switch insert position. The switch record starts there, // so it carries the ack over the target. A post-switch target // waits on bgwriter's next standby snapshot instead, up to @@ -505,7 +505,7 @@ fn run_ddl_intermix(ports: fx::Ports, decoder_pool: usize, inserter_pool: usize, walshadow::pg::parse_pg_lsn(&target_lsn_text).context("parse source LSN")?; wait_for_metric_lsn( metrics_addr, - "walshadow_emitter_ack_lsn", + "walshadow_position_lsn{at=\"ack\"}", target_lsn, Duration::from_secs(60), ) @@ -668,12 +668,12 @@ fn wait_child_with_timeout( /// Stands in for `--max-segments`, which races with bootstrap-triggered /// segment seals. /// -/// `walshadow_emitter_ack_lsn` carries record *start* LSNs (a commit's seq +/// `walshadow_position_lsn{at="ack"}` carries record *start* LSNs (a commit's seq /// publishes its commit record's start, idle advance publishes the last /// record's start), so it reaches `target` only once a record starting at or /// past `target` is flushed to CH. `pg_current_wal_lsn` reports the *end* of /// the last record, so a source that never writes again pins the ack one -/// record short of its own head; gate on `walshadow_source_received_lsn` +/// record short of its own head; gate on `walshadow_position_lsn{at="received"}` /// instead, which the pump takes from the walsender's WAL end fn wait_for_metric_lsn( metrics_addr: SocketAddr, diff --git a/tests/queueing_throughput_e2e.rs b/tests/queueing_throughput_e2e.rs index 7cba21c3..5442b142 100644 --- a/tests/queueing_throughput_e2e.rs +++ b/tests/queueing_throughput_e2e.rs @@ -166,7 +166,7 @@ fn sample(addr: SocketAddr) -> Result { Ok(Sample { at: Instant::now(), queue_records_out: get("walshadow_queue_records_out_total"), - filter_lsn: get("walshadow_filter_lsn"), + filter_lsn: get("walshadow_position_lsn{at=\"filter\"}"), emitter_rows: get("walshadow_emitter_rows_total"), inserter_batches: get("walshadow_inserter_batches_in_total"), inserter_ch_ms: ch_seconds,