Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/bin/stream/metrics_publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ pub(crate) struct ShadowMetricsView {
/// yet, so the gauges keep their last value
pub(crate) apply_lag: Option<(u64, f64)>,
pub(crate) active_connections: u64,
pub(crate) dropped_total: u64,
pub(crate) socket_events: [u64; walshadow::shadow_stream::SocketEvent::COUNT],
pub(crate) backfill_bytes_total: u64,
pub(crate) send_queue_bytes: u64,
}

/// Apply lag against the source head. A zero `shadow_replay` means no walreceiver
Expand Down Expand Up @@ -313,7 +315,9 @@ pub(crate) async fn populate_metrics(
shadow_apply_lag_bytes: apply_lag_bytes,
shadow_apply_lag_seconds: apply_lag_seconds,
shadow_stream_active_connections: shadow_view.active_connections,
shadow_stream_dropped_connections_total: shadow_view.dropped_total,
shadow_stream_sockets_by_event: shadow_view.socket_events,
wal_bytes: [shadow_view.backfill_bytes_total],
buffer_bytes: [shadow_view.send_queue_bytes],
..registry.snapshot().await
};
populate_pipeline_metrics(
Expand Down
6 changes: 4 additions & 2 deletions src/bin/stream/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ async fn await_walreceiver(
// `accepted` separates "shadow never dialed" from "shadow dialed
// and stalled in the handshake" — the latter reads as the former
// without it, since only START_REPLICATION registers a connection
let (secs, accepted) = (timeout.as_secs(), agg.accepted_total);
let (secs, accepted) = (timeout.as_secs(), agg.accepted());
anyhow::ensure!(
start.elapsed() < timeout,
"no walreceiver streaming from walsender {walsender_addr} within \
Expand Down Expand Up @@ -2354,7 +2354,9 @@ async fn session_body(
ShadowMetricsView {
apply_lag,
active_connections: shadow_agg.active_connections as u64,
dropped_total: shadow_agg.dropped_total,
socket_events: shadow_agg.socket_events,
backfill_bytes_total: shadow_agg.backfill_bytes_total,
send_queue_bytes: shadow_agg.send_queue_bytes,
},
&boundary_hold_stats,
metrics_dbs.iter().map(DbMetricSources::series).collect(),
Expand Down
64 changes: 57 additions & 7 deletions src/ops/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ use crate::catalog::pending::DegradeReason;
use crate::decode::heap_decoder::HEAP_OP_LABELS;
use crate::ops::bridge::{OP_COUNT, OP_LABELS};
use crate::pos::{Drain, EmitterAck, FilterDispatched, Floor, Pos, ShadowReplay, SourceReceived};
use crate::source::shadow_stream::SocketEvent;
use crate::source::transition::SWITCH_FAILURE_REASONS;

#[macro_export]
Expand Down Expand Up @@ -97,6 +98,18 @@ macro_rules! label_class {
};
}

label_class! {
pub enum WalFlow {
WalsenderBackfill => "walsender_backfill",
}
}

label_class! {
pub enum Buffer {
WalsenderSendQueue => "walsender_send_queue",
}
}

label_class! {
pub enum WorkerState {
Busy => "busy",
Expand Down Expand Up @@ -281,6 +294,11 @@ snapshot! {
/// Failures by [`ErrorKind`], rendered
/// `walshadow_errors_total{component,kind}`
errors: [u64; ErrorKind::COUNT],
/// Postgres data moved, by [`WalFlow`], rendered
/// `walshadow_wal_bytes_total{flow}`
wal_bytes: [u64; WalFlow::COUNT],
/// Data held, by [`Buffer`], rendered `walshadow_buffer_bytes{buffer}`
buffer_bytes: [u64; Buffer::COUNT],
/// Refusal a crossing parked on, rendered as `crossing_wedged`. The
/// pump keeps publishing rather than exiting into a restart that
/// re-crosses and re-fails, so this is how the daemon says it is
Expand Down Expand Up @@ -586,9 +604,9 @@ snapshot! {
"Estimated seconds shadow trails source, by source byte rate over last 30s.",
gauge shadow_stream_active_connections: u64 =
"Currently-attached walreceiver connections to walshadow's walsender.",
/// Cumulative connections dropped by `slow_threshold` overflow
counter shadow_stream_dropped_connections_total: u64 =
"Connections dropped by slow-client cutoff since daemon start.",
counter shadow_stream_sockets_by_event: [u64; SocketEvent::COUNT]
["event" = SocketEvent::LABELS] =
"Walsender socket lifecycle events: accepted (entered the handshake, START_REPLICATION or not), closed_slow_client (dropped at the send queue cap), closed_by_client (clean hangup), closed_error (handshake, I/O, protocol failure or refused START_REPLICATION). accepted minus the closed events is sockets alive.",
counter catalog_boundary_holds_total: u64 =
"Publication holds released at catalog-mutating commit boundaries.",
counter catalog_boundary_hold_failures_total: u64 =
Expand Down Expand Up @@ -1069,6 +1087,26 @@ fn encode_snapshot(snap: &MetricsSnapshot, enc: &mut DescriptorEncoder<'_>) -> f
.map(|e| (e.pair(), snap.errors[e.index()])),
)?;

counter_series(
enc,
"walshadow_wal_bytes_total",
"Postgres data moved, by flow. walsender_backfill is WAL replayed from the retained wire buffer to a reconnecting shadow walreceiver.",
"flow",
WalFlow::ALL
.into_iter()
.map(|f| (f.label(), snap.wal_bytes[f.index()])),
)?;

gauge_series(
enc,
"walshadow_buffer_bytes",
"Data held right now, by buffer. walsender_send_queue is the largest unsent walsender queue across connections with the reconnect backfill left out, read against the slow-client cap.",
"buffer",
Buffer::ALL
.into_iter()
.map(|b| (b.label(), snap.buffer_bytes[b.index()])),
)?;

counter_pairs(
enc,
"walshadow_worker_seconds_total",
Expand Down Expand Up @@ -1604,7 +1642,9 @@ mod tests {
shadow_apply_lag_bytes: 12345,
shadow_apply_lag_seconds: 1.23456,
shadow_stream_active_connections: 2,
shadow_stream_dropped_connections_total: 5,
shadow_stream_sockets_by_event: [9, 5, 3, 1],
wal_bytes: [4096],
buffer_bytes: [777],
..MetricsSnapshot::default()
};
let body = render(snap);
Expand All @@ -1617,9 +1657,19 @@ mod tests {
assert!(body.contains("# HELP walshadow_shadow_stream_active_connections"));
assert!(body.contains("# TYPE walshadow_shadow_stream_active_connections gauge"));
assert!(body.contains("walshadow_shadow_stream_active_connections 2"));
assert!(body.contains("# HELP walshadow_shadow_stream_dropped_connections"));
assert!(body.contains("# TYPE walshadow_shadow_stream_dropped_connections counter"));
assert!(body.contains("walshadow_shadow_stream_dropped_connections_total 5"));
assert!(body.contains("# TYPE walshadow_shadow_stream_sockets counter"));
assert!(body.contains("walshadow_shadow_stream_sockets_total{event=\"accepted\"} 9"));
assert!(
body.contains("walshadow_shadow_stream_sockets_total{event=\"closed_slow_client\"} 5")
);
assert!(
body.contains("walshadow_shadow_stream_sockets_total{event=\"closed_by_client\"} 3")
);
assert!(body.contains("walshadow_shadow_stream_sockets_total{event=\"closed_error\"} 1"));
assert!(body.contains("# TYPE walshadow_wal_bytes counter"));
assert!(body.contains("walshadow_wal_bytes_total{flow=\"walsender_backfill\"} 4096"));
assert!(body.contains("# TYPE walshadow_buffer_bytes gauge"));
assert!(body.contains("walshadow_buffer_bytes{buffer=\"walsender_send_queue\"} 777"));
}

/// Unknown rate stays infinite on the wire. prometheus-client spells it
Expand Down
Loading
Loading