Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion src/bin/stream/metrics_publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ use walshadow::ch_emitter::EmitterStats;
use walshadow::config::ConfigResolver;
use walshadow::metrics::WorkerState;
use walshadow::metrics::{
DbSeries, MetricsRegistry, MetricsSnapshot, RateEstimator, source_server_wal_end,
DbSeries, HistogramSnapshot, MetricsRegistry, MetricsSnapshot, RateEstimator,
source_server_wal_end,
};
use walshadow::pipeline::row_ledger::TableRowCounts;
use walshadow::pos::{
Expand Down Expand Up @@ -162,6 +163,15 @@ impl DbMetricSources {
let bridge_op_seconds = |pick: fn(&BridgeStats) -> &[AtomicU64; OP_COUNT]| {
bridge_ops(pick).map(|nanos| nanos as f64 / 1e9)
};
let bridge_latency = || -> [HistogramSnapshot; OP_COUNT] {
std::array::from_fn(|i| {
let mut folded = HistogramSnapshot::default();
for s in self.bridge.iter().flatten() {
folded.merge(s.request_latency[i].snapshot(1e9));
}
folded
})
};
let log_stats = self.desc_log.as_ref().map(|log| log.stats_handle());
let log = |pick: fn(&DescLogStats) -> &AtomicU64| -> u64 {
log_stats.as_ref().map_or(0, |s| ld(pick(s)))
Expand All @@ -183,6 +193,7 @@ impl DbMetricSources {
bridge_requests_by_op: bridge_ops(|s| &s.requests),
bridge_errors_by_op: bridge_ops(|s| &s.errors),
bridge_request_seconds_by_op: bridge_op_seconds(|s| &s.request_nanos),
request_duration_seconds: bridge_latency(),
bridge_lock_wait_seconds_by_op: bridge_op_seconds(|s| &s.lock_wait_nanos),
bridge_service_seconds_by_op: bridge_op_seconds(|s| &s.service_nanos),
bridge_request_bytes_by_op: bridge_ops(|s| &s.request_bytes),
Expand Down Expand Up @@ -564,6 +575,9 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M
toast_value_fetch_seconds: emitter_seconds(|s| &s.toast_value_fetch_nanos),
toast_values_read_ahead_total: emitter(|s| &s.toast_values_read_ahead),
toast_value_wait_seconds: emitter_seconds(|s| &s.toast_value_wait_nanos),
toast_values_per_fetch: v.emitter.map_or_else(HistogramSnapshot::default, |s| {
s.toast_values_per_fetch.snapshot(1.0)
}),
toast_image_rows_mirrored_total: emitter(|s| &s.toast_image_rows_mirrored),
toast_values_filled_superseded_total: emitter(|s| &s.toast_values_filled_superseded),
toast_values_filled_mismatch_total: emitter(|s| &s.toast_values_filled_mismatch),
Expand Down Expand Up @@ -1055,6 +1069,7 @@ mod tests {
let bump = |s: &BridgeStats, n: u64| {
s.up.store(1, Ordering::Relaxed);
s.requests[encode].fetch_add(n, Ordering::Relaxed);
s.request_latency[encode].observe(30_000);
s.native_bytes.fetch_add(n, Ordering::Relaxed);
};
let (live, boot) = (
Expand All @@ -1076,6 +1091,10 @@ mod tests {
let handed_over = sources([Some(live.clone()), Some(boot.clone())]).series();
assert_eq!(handed_over.database, "app");
assert_eq!(handed_over.bridge_native_bytes_total, 7);
assert_eq!(
handed_over.request_duration_seconds[encode].count, 2,
"both bridges' latency samples fold into one histogram"
);
assert_eq!(
handed_over.bridge_requests_by_op[encode], 7,
"both bridges' requests land on the database's series"
Expand Down
6 changes: 6 additions & 0 deletions src/emit/ch_emitter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2639,6 +2639,10 @@ fn oracle_cell(v: Option<&ColumnValue>, source_type_oid: u32) -> Result<OracleCe
})
}

/// Values-per-fetch buckets: 1 doubling to 256, where one means no batching
/// and the top is past a typical deferred-resolution batch
pub type ValuesPerFetch = crate::ops::metrics::Histogram<1, 9>;

crate::atomic_stats! {
/// CH emitter counters. `fetch_add(_, Relaxed)`; status loop reads
/// via `.load(Relaxed)`.
Expand Down Expand Up @@ -2680,6 +2684,8 @@ crate::atomic_stats! {
pub toast_values_read_ahead,
/// Planning blocked on reads started ahead of it
pub toast_value_wait_nanos,
/// Values requested per store fetch
pub toast_values_per_fetch: ValuesPerFetch,
/// Toasted values NULL/default-filled because no store could rebuild
/// them (disabled mode). Surfaced, never silent
pub toast_values_filled_default,
Expand Down
12 changes: 11 additions & 1 deletion src/ops/bridge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::UnixStream;
use tokio::sync::Mutex;

use crate::ops::metrics::Histogram;
use crate::toast::FetchedValue;

/// Frame and op layouts. Must equal `WS_PROTO_VERSION` in `pgext/walshadow.h`
Expand Down Expand Up @@ -258,6 +259,11 @@ pub struct ScanResult {
pub rows: Vec<Vec<Option<String>>>,
}

/// Bridge request latency in nanoseconds: 25us doubling to 102.4ms, which
/// spans a local socket round trip through a worker stalled on slow storage.
/// Slower requests land in `+Inf`
pub type RequestLatency = Histogram<25_000, 13>;

crate::atomic_stats! {
pub struct BridgeStats {
/// 1 while the last transport attempt succeeded
Expand All @@ -276,6 +282,8 @@ crate::atomic_stats! {
pub requests: [AtomicU64; OP_COUNT],
pub errors: [AtomicU64; OP_COUNT],
pub request_nanos: [AtomicU64; OP_COUNT],
/// Distribution behind `request_nanos`
pub request_latency: [RequestLatency; OP_COUNT],
/// Queued behind another caller's request on the one socket. Against
/// `service_nanos` this is what says whether the worker is the
/// limiter or the funnel in front of it is
Expand Down Expand Up @@ -751,7 +759,9 @@ impl Bridge {
fn record<T>(stats: &BridgeStats, op: Op, started: Instant, res: &Result<T, BridgeError>) {
let slot = op.slot();
stats.requests[slot].fetch_add(1, Ordering::Relaxed);
stats.request_nanos[slot].fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed);
let nanos = started.elapsed().as_nanos() as u64;
stats.request_nanos[slot].fetch_add(nanos, Ordering::Relaxed);
stats.request_latency[slot].observe(nanos);
if res.is_err() {
stats.errors[slot].fetch_add(1, Ordering::Relaxed);
}
Expand Down
Loading
Loading