Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bench/src/initial_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ pub async fn run(
bail!("daemon restarted while seeding");
}
if sample
.get("walshadow_floor_lsn")
.get("walshadow_position_lsn{at=\"floor\"}")
.is_some_and(|value| *value >= target as f64)
&& sample.get("walshadow_config_backfills_pending") == Some(&0.0)
{
Expand Down
6 changes: 3 additions & 3 deletions docker/grafana/dashboards/walshadow.json
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@
},
"targets": [
{
"expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)",
"expr": "clamp_min(walshadow_position_lsn{at=\"source_head\"} - ignoring(at) walshadow_position_lsn{at=\"ack\"}, 0)",
"instant": true,
"legendFormat": "backlog",
"refId": "A"
Expand Down Expand Up @@ -633,12 +633,12 @@
"refId": "A"
},
{
"expr": "clamp_min(walshadow_source_received_lsn - walshadow_decoder_commit_lsn, 0)",
"expr": "clamp_min(walshadow_position_lsn{at=\"received\"} - ignoring(at) walshadow_position_lsn{at=\"decoder_commit\"}, 0)",
"legendFormat": "source to decoder",
"refId": "B"
},
{
"expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)",
"expr": "clamp_min(walshadow_position_lsn{at=\"source_head\"} - ignoring(at) walshadow_position_lsn{at=\"ack\"}, 0)",
"legendFormat": "source to ClickHouse ack",
"refId": "C"
}
Expand Down
18 changes: 18 additions & 0 deletions docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,24 @@ toward `source_received`. Short differences are expected while batches flush

## Monitor with Prometheus and Grafana

WAL positions are one family, `walshadow_position_lsn{at}`, all from
walshadow's own view and all written in one status tick:

| `at` | Meaning |
|---|---|
| `source_head` | newest `server_wal_end` heard from the source socket; absent unless the pump is streaming from the source: before the first message, during archive catch-up (`walshadow_source_mode{mode="archive"}`), while reconnecting, and for the first tick after a reconnect or endpoint swap |
| `received` | the further of the reported source head and the pump's dispatch frontier (what the pump has pushed into the decode queue), not WAL held locally |
| `filter`, `decoder_commit`, `ack` | filter dispatch, highest drained commit, and ClickHouse ack positions |
| `shadow_replay`, `floor` | shadow Postgres replay position and the durable resume floor |
| `pause_consumed`, `pause_received`, `promotion_target_replay`, `promotion_target_receive`, `timeline_switch` | present only during a pause, a promotion check, or after a timeline crossing |

A member is absent while its position is unknown. Because every member comes
from the same tick, `position_lsn{at="source_head"} - ignoring(at)
position_lsn{at="ack"}` has no scrape skew, while subtracting the ack from a
primary LSN scraped elsewhere shows a sawtooth at high WAL rates. A gap in
`source_head` means walshadow cannot see the source head, not that it has
caught up.

Set `--metrics-bind` to expose OpenMetrics text metrics, which Prometheus
scrapes directly. Counter series carry `_total`. Docker deployment uses port
9484 and includes optional provisioned dashboard
Expand Down
6 changes: 5 additions & 1 deletion src/bin/stream/metrics_publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ use walshadow::boundary_hold::BoundaryHoldStats;
use walshadow::ch_emitter::EmitterStats;
use walshadow::config::ConfigResolver;
use walshadow::metrics::WorkerState;
use walshadow::metrics::{DbSeries, MetricsRegistry, MetricsSnapshot, RateEstimator};
use walshadow::metrics::{
DbSeries, MetricsRegistry, MetricsSnapshot, RateEstimator, source_server_wal_end,
};
use walshadow::pipeline::row_ledger::TableRowCounts;
use walshadow::pos::{
Drain, EmitterAck, FilterDispatched, Floor, Pos, ShadowReplay, SourceReceived,
Expand Down Expand Up @@ -259,6 +261,7 @@ impl DrainResident {
pub(crate) async fn populate_metrics(
registry: &MetricsRegistry,
source_received_lsn: Pos<SourceReceived>,
source_wal_end: Option<u64>,
filter_lsn: Pos<FilterDispatched>,
shadow_replay_lsn: Pos<ShadowReplay>,
decoder_commit_lsn: Pos<Drain>,
Expand All @@ -283,6 +286,7 @@ pub(crate) async fn populate_metrics(
.unwrap_or((prev.shadow_apply_lag_bytes, prev.shadow_apply_lag_seconds));
let base = MetricsSnapshot {
source_received_lsn,
source_server_wal_end_lsn: source_server_wal_end(source_wal_end),
filter_lsn,
shadow_replay_lsn,
decoder_commit_lsn,
Expand Down
2 changes: 2 additions & 0 deletions src/bin/stream/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2323,9 +2323,11 @@ async fn session_body(
// drained, not the top-of-iteration values.
let emitter_ack_for_metric = emitter_ack.get();
let drain_for_metric = xact_stats.drain_lsn;
let source_wal_end = matches!(path, SourcePath::Live).then(|| feed.last_server_wal_end());
populate_metrics(
&metrics,
received,
source_wal_end,
Pos::new(now_dispatched),
shadow_replay,
drain_for_metric,
Expand Down
161 changes: 114 additions & 47 deletions src/ops/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,23 @@ label_class! {
}
}

label_class! {
pub enum Position {
SourceHead => "source_head",
Received => "received",
Filter => "filter",
DecoderCommit => "decoder_commit",
Ack => "ack",
ShadowReplay => "shadow_replay",
Floor => "floor",
PauseConsumed => "pause_consumed",
PauseReceived => "pause_received",
PromotionTargetReplay => "promotion_target_replay",
PromotionTargetReceive => "promotion_target_receive",
TimelineSwitch => "timeline_switch",
}
}

label_class! {
pub enum WorkerState {
Busy => "busy",
Expand Down Expand Up @@ -156,15 +173,6 @@ macro_rules! encode_metric {
($enc:expr, gauge, $name:expr, $help:expr, $value:expr) => {
gauge($enc, $name, $help, $value)?
};
// LSN 0 is InvalidXLogRecPtr: the position is not known yet, eg the
// default snapshot served from process start until the session's first
// status write. A 0 sample between real LSNs reads as a ~80 TB jump to
// deriv() and lag queries, so the sample is left out instead
($enc:expr, lsn, $name:expr, $help:expr, $value:expr) => {
if $value.get() != 0 {
gauge($enc, $name, $help, $value)?
}
};
($enc:expr, counter, $name:expr, $help:expr, $value:expr, $key:literal, $labels:expr) => {
counter_series($enc, $name, $help, $key, $labels.into_iter().zip($value))?
};
Expand Down Expand Up @@ -326,20 +334,21 @@ snapshot! {
promotion_target_replay_lsn: u64,
/// Target's `pg_last_wal_receive_lsn()`, read while paused
promotion_target_receive_lsn: u64,
/// WAL positions rendered as `walshadow_position_lsn{at}`, each only
/// while known (0 is InvalidXLogRecPtr); all read in one status tick
/// so differences between them carry no scrape skew
source_received_lsn: Pos<SourceReceived>,
source_server_wal_end_lsn: Option<u64>,
filter_lsn: Pos<FilterDispatched>,
shadow_replay_lsn: Pos<ShadowReplay>,
decoder_commit_lsn: Pos<Drain>,
emitter_ack_lsn: Pos<EmitterAck>,
floor_lsn: Pos<Floor>,
timeline_switch_lsn: u64,
pause_consumed_lsn: u64,
pause_received_lsn: u64,
}

lsn source_received_lsn: Pos<SourceReceived> =
"Source PG's most recent server_wal_end seen on the replication socket.",
/// Last segment-boundary LSN dispatched downstream; becomes durable once
/// segment fsync lands
lsn filter_lsn: Pos<FilterDispatched> =
"LSN of the last filtered WAL byte the daemon has dispatched.",
lsn shadow_replay_lsn: Pos<ShadowReplay> =
"Shadow PG's pg_last_wal_replay_lsn(), polled at status cadence.",
lsn decoder_commit_lsn: Pos<Drain> = "Highest commit LSN drained out of the xact buffer.",
/// Live insert watermark, unlike manifest's resume-safe `emitter_ack`
lsn emitter_ack_lsn: Pos<EmitterAck> = "Contiguous-done watermark from the insert pipeline.",
lsn floor_lsn: Pos<Floor> = "Durable resume floor: restart resumes here and pruners cut to it.",
gauge xact_active: u64 = "Active transactions buffered in memory or on spill.",
gauge xact_bytes_in_memory: u64 = "Bytes held in memory across all buffered xacts.",
gauge spill_xacts_active: u64 = "Xacts with at least one entry currently in their spill file.",
Expand Down Expand Up @@ -572,21 +581,9 @@ snapshot! {
counter timeline_switch_failures_by_reason: [u64; SWITCH_FAILURE_REASONS.len()]
["reason" = SWITCH_FAILURE_REASONS] =
"Crossings refused; the stream stays on the ancestor.",
/// Most recent fork point, diagnostic
gauge timeline_switch_lsn: u64 = "Fork LSN of the most recent timeline crossing.",
counter timeline_prefix_bytes_verified_total: u64 =
"Descendant bytes compared against the retained ancestor fork prefix.",
counter timeline_transition_seconds_total: f64 = "Seconds spent inside timeline crossings.",
/// `WalStream::next_lsn` frozen when the pump observed `[stream] paused`,
/// which resume asks the source to serve. `0` while not paused: a value
/// left from an earlier pause cannot be compared against a promotion
/// decision either
gauge pause_consumed_lsn: u64 =
"Consumed frontier frozen when the pump observed the pause; 0 while running.",
/// Source head last heard about, frozen at the same instant. The promotion
/// target must reach it before it is promoted
gauge pause_received_lsn: u64 =
"Source head frozen when the pump observed the pause; 0 while running.",
gauge shadow_served_timeline: u32 = "Timeline the shadow-facing walsender advertises.",
/// Branch the shadow is replaying, which is how a shadow that followed the
/// chain reads differently from one that merely survived
Expand Down Expand Up @@ -861,6 +858,10 @@ const PLAN_FAILURE_REASONS: [&str; 11] = [
"drain",
];

pub fn source_server_wal_end(wal_end: Option<u64>) -> Option<u64> {
wal_end.filter(|&end| end != 0)
}

/// `mode=` label order of [`DbSeries::config_backfills_pending_by_mode`]
const BACKFILL_MODES: [&str; 3] = ["copy", "base_backup", "object_store"];

Expand Down Expand Up @@ -1117,6 +1118,40 @@ fn encode_snapshot(snap: &MetricsSnapshot, enc: &mut DescriptorEncoder<'_>) -> f
.map(|(worker, state, secs)| ([*worker, state.label()], *secs)),
)?;

let positions = [
(
Position::SourceHead,
snap.source_server_wal_end_lsn.unwrap_or(0),
),
(Position::Received, snap.source_received_lsn.get()),
(Position::Filter, snap.filter_lsn.get()),
(Position::DecoderCommit, snap.decoder_commit_lsn.get()),
(Position::Ack, snap.emitter_ack_lsn.get()),
(Position::ShadowReplay, snap.shadow_replay_lsn.get()),
(Position::Floor, snap.floor_lsn.get()),
(Position::PauseConsumed, snap.pause_consumed_lsn),
(Position::PauseReceived, snap.pause_received_lsn),
(
Position::PromotionTargetReplay,
snap.promotion_target_replay_lsn,
),
(
Position::PromotionTargetReceive,
snap.promotion_target_receive_lsn,
),
(Position::TimelineSwitch, snap.timeline_switch_lsn),
];
gauge_series(
enc,
"walshadow_position_lsn",
"WAL byte positions, all published in one status tick so any two subtract without scrape skew. source_head is the newest server_wal_end heard on the source socket (absent unless the pump is streaming from the source: archive catch-up, reconnects and the first tick after one leave it out); received the source head floored at the filter dispatch frontier; filter the last dispatched filtered byte; decoder_commit the highest commit drained; ack the ClickHouse watermark; shadow_replay the shadow Postgres replay position; floor the durable resume floor. pause_consumed and pause_received are the frontiers frozen by a pause, promotion_target_replay and promotion_target_receive the target's positions read while paused, timeline_switch the fork of the last crossing. A member is absent while its position is unknown (0).",
"at",
positions
.into_iter()
.filter(|(_, lsn)| *lsn != 0)
.map(|(at, lsn)| (at.label(), lsn)),
)?;

let mut records = declare(
enc,
"walshadow_filter_records_total",
Expand Down Expand Up @@ -1298,23 +1333,52 @@ mod tests {
}

#[test]
fn render_omits_unknown_lsn_gauges() {
fn render_omits_unknown_positions() {
let body = render(MetricsSnapshot {
emitter_ack_lsn: 0x47FD_FB00_0000.into(),
pause_received_lsn: 77,
..MetricsSnapshot::default()
});
assert!(body.contains("walshadow_emitter_ack_lsn 79156163379200"));
for name in [
"walshadow_source_received_lsn",
"walshadow_filter_lsn",
"walshadow_shadow_replay_lsn",
"walshadow_decoder_commit_lsn",
"walshadow_floor_lsn",
assert!(body.contains("# TYPE walshadow_position_lsn gauge"));
assert!(body.contains("walshadow_position_lsn{at=\"ack\"} 79156163379200"));
assert!(body.contains("walshadow_position_lsn{at=\"pause_received\"} 77"));
for at in [
"source_head",
"received",
"filter",
"shadow_replay",
"decoder_commit",
"floor",
"pause_consumed",
"promotion_target_replay",
"promotion_target_receive",
"timeline_switch",
] {
assert!(!body.contains(name), "{name} rendered at LSN 0: {body}");
let line = format!("walshadow_position_lsn{{at=\"{at}\"}}");
assert!(!body.contains(&line), "{at} rendered at LSN 0: {body}");
}
}

#[test]
fn source_server_wal_end_is_absent_until_heard() {
for unknown in [None, Some(0)] {
assert_eq!(source_server_wal_end(unknown), None);
let body = render(MetricsSnapshot {
source_server_wal_end_lsn: source_server_wal_end(unknown),
..MetricsSnapshot::default()
});
assert!(
!body.contains("walshadow_position_lsn{at=\"source_head\"}"),
"{body}"
);
}
let body = render(MetricsSnapshot {
source_server_wal_end_lsn: source_server_wal_end(Some(1_000)),
..MetricsSnapshot::default()
});
assert!(body.contains("walshadow_position_lsn{at=\"source_head\"} 1000"));
}

#[test]
fn render_includes_help_type_lines() {
let mut snap = MetricsSnapshot {
Expand All @@ -1328,10 +1392,10 @@ mod tests {
.insert(("Heap".into(), "to_decoder"), 17);

let body = render(snap);
assert!(body.contains("# HELP walshadow_source_received_lsn"));
assert!(body.contains("# TYPE walshadow_source_received_lsn gauge"));
assert!(body.contains("walshadow_source_received_lsn 3405691582"));
assert!(body.contains("walshadow_filter_lsn 12648430"));
assert!(body.contains("# HELP walshadow_position_lsn"));
assert!(body.contains("# TYPE walshadow_position_lsn gauge"));
assert!(body.contains("walshadow_position_lsn{at=\"received\"} 3405691582"));
assert!(body.contains("walshadow_position_lsn{at=\"filter\"} 12648430"));
assert!(
body.contains("walshadow_filter_records_total{rmgr=\"Heap\",route=\"to_decoder\"} 17")
);
Expand Down Expand Up @@ -1820,7 +1884,10 @@ mod tests {
let resp = String::from_utf8(buf).unwrap();
assert!(resp.starts_with("HTTP/1.0 200 OK\r\n"), "{resp}");
assert!(resp.contains("Content-Type: application/openmetrics-text"));
assert!(resp.contains("walshadow_filter_lsn 57005"), "{resp}");
assert!(
resp.contains("walshadow_position_lsn{at=\"filter\"} 57005"),
"{resp}"
);
let taken = serve(addr, MetricsRegistry::new()).await.unwrap_err();
assert!(matches!(taken, MetricsError::Bind { .. }), "{taken}");
}
Expand Down
2 changes: 1 addition & 1 deletion src/source/manifest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
//! * `drain`: highest commit-record LSN drained out of the xact buffer.
//! Strictly higher than `emitter_ack`.
//! * `emitter_ack`: [`ResumeSafe`], not the live ack behind
//! `walshadow_emitter_ack_lsn`. Slot-advance ceiling.
//! `walshadow_position_lsn{at="ack"}`. Slot-advance ceiling.
//! * `shadow_flush`: min `flush_lsn` from inbound `'r'` standby status
//! across active shadow streaming connections. On restart, resume
//! position walsender hands shadow via `START_REPLICATION PHYSICAL
Expand Down
9 changes: 5 additions & 4 deletions tests/bin_stream_e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ async fn bin_stream_replicates_segments_and_serves_metrics() {
// well-known walshadow_ counters.
let body = http_get(metrics_addr, "/metrics").context("metrics scrape")?;
assert!(
body.contains("walshadow_source_received_lsn") || body.contains("walshadow_uptime"),
body.contains("walshadow_position_lsn") || body.contains("walshadow_uptime"),
"expected walshadow_* counter in /metrics body: {body}",
);
// Shadow apply-lag surface is part of the static
Expand All @@ -376,7 +376,7 @@ async fn bin_stream_replicates_segments_and_serves_metrics() {
);
}
// Absent until the session's first status write publishes a position
let ack_at_boot = metric_u64(&body, "walshadow_emitter_ack_lsn").unwrap_or(0);
let ack_at_boot = metric_u64(&body, "walshadow_position_lsn{at=\"ack\"}").unwrap_or(0);

// 6. Drive workload. The daemon filters user-heap WAL records
// (rmgr=HEAP, class=User → replaced with XLOG_NOOP) — only
Expand All @@ -403,8 +403,9 @@ async fn bin_stream_replicates_segments_and_serves_metrics() {
loop {
let body = http_get(metrics_addr, "/metrics").context("ack poll scrape")?;
// LSN gauges stay absent while their position is unknown
let ack = metric_u64(&body, "walshadow_emitter_ack_lsn").unwrap_or(0);
let replay = metric_u64(&body, "walshadow_shadow_replay_lsn").unwrap_or(0);
let ack = metric_u64(&body, "walshadow_position_lsn{at=\"ack\"}").unwrap_or(0);
let replay =
metric_u64(&body, "walshadow_position_lsn{at=\"shadow_replay\"}").unwrap_or(0);
if ack > ack_at_boot && replay > 0 {
break;
}
Expand Down
8 changes: 6 additions & 2 deletions tests/common/bootstrap_ch_fixture.rs
Original file line number Diff line number Diff line change
Expand Up @@ -571,8 +571,12 @@ pub fn parse_metric(body: &str, name: &str) -> Option<u64> {
continue;
}
let head = line.split_once(' ').map(|(h, _)| h)?;
let stem = head.split_once('{').map(|(s, _)| s).unwrap_or(head);
if stem != name {
let key = if name.contains('{') {
head
} else {
head.split_once('{').map(|(s, _)| s).unwrap_or(head)
};
if key != name {
continue;
}
let value_str = line.rsplit_once(' ').map(|(_, v)| v)?;
Expand Down
Loading
Loading