From 4ffd1151e673219e44a48994747115de1bc339ee Mon Sep 17 00:00:00 2001 From: Harshil Goel Date: Tue, 10 Nov 2026 03:27:43 +0530 Subject: [PATCH] feat(metrics): bootstrap phase stages, live gate progress, hourly config log --- bench/src/initial_load.rs | 25 ++-- docs/operations.md | 22 ++++ plans/observability.md | 6 +- src/backfill/backup_backfill.rs | 10 +- src/backfill/visibility_gate.rs | 74 ++++++++--- src/bin/stream/bootstrap.rs | 96 ++++++++------ src/bin/stream/metrics_publish.rs | 34 ++++- src/bin/stream/session.rs | 49 ++++++++ src/emit/ch_emitter.rs | 2 + src/ops/metrics.rs | 29 +++-- src/ops/stages.rs | 202 ++++++++++++++++++++++++++---- 11 files changed, 448 insertions(+), 101 deletions(-) diff --git a/bench/src/initial_load.rs b/bench/src/initial_load.rs index e365717e..69a92e2d 100644 --- a/bench/src/initial_load.rs +++ b/bench/src/initial_load.rs @@ -142,10 +142,12 @@ fn lifecycle_complete( after: &BTreeMap, ) -> bool { let finished = |stage| { - let key = format!("walshadow_stage_completed_total{{stage=\"{stage}\"}}"); - after - .get(&key) - .is_some_and(|value| *value > *before.get(&key).unwrap_or(&0.0)) + let prefix = format!("walshadow_stage_completed_total{{stage=\"{stage}\""); + after.iter().any(|(key, value)| { + key.strip_prefix(&prefix) + .is_some_and(|rest| rest.starts_with(',') || rest.starts_with('}')) + && *value > *before.get(key).unwrap_or(&0.0) + }) }; if bootstrap { finished("bootstrap") && finished("shadow_replay") @@ -557,7 +559,7 @@ mod tests { #[test] fn completion_requires_new_stage_work_and_settlement() { let before = metrics_values( - "walshadow_stage_completed_total{stage=\"publish\"} 2\nwalshadow_stage_completed_total{stage=\"settle\"} 2\nwalshadow_config_backfills_pending 0\n", + "walshadow_stage_completed_total{stage=\"publish\",title=\"Publish\",order=\"090\"} 2\nwalshadow_stage_completed_total{stage=\"settle\",title=\"Settle\",order=\"100\"} 2\nwalshadow_config_backfills_pending 0\n", ); assert!(!lifecycle_complete( false, @@ -567,7 +569,8 @@ mod tests { )); let mut after = before.clone(); after.insert( - "walshadow_stage_completed_total{stage=\"publish\"}".into(), + "walshadow_stage_completed_total{stage=\"publish\",title=\"Publish\",order=\"090\"}" + .into(), 3.0, ); assert!(!lifecycle_complete( @@ -577,7 +580,8 @@ mod tests { &after )); after.insert( - "walshadow_stage_completed_total{stage=\"settle\"}".into(), + "walshadow_stage_completed_total{stage=\"settle\",title=\"Settle\",order=\"100\"}" + .into(), 3.0, ); assert!(lifecycle_complete( @@ -599,7 +603,7 @@ mod tests { fn bootstrap_requires_snapshot_and_shadow_replay() { let empty = BTreeMap::new(); let mut after = metrics_values( - "walshadow_stage_completed_total{stage=\"copy\"} 1\nwalshadow_config_backfills_pending 0\n", + "walshadow_stage_completed_total{stage=\"copy\",title=\"Table COPY\",order=\"070\"} 1\nwalshadow_config_backfills_pending 0\n", ); assert!(!lifecycle_complete( true, @@ -608,7 +612,8 @@ mod tests { &after )); after.insert( - "walshadow_stage_completed_total{stage=\"bootstrap\"}".into(), + "walshadow_stage_completed_total{stage=\"bootstrap\",title=\"Initial load\",order=\"010\"}" + .into(), 1.0, ); assert!(!lifecycle_complete( @@ -618,7 +623,7 @@ mod tests { &after )); after.insert( - "walshadow_stage_completed_total{stage=\"shadow_replay\"}".into(), + "walshadow_stage_completed_total{stage=\"shadow_replay\",title=\"Shadow start\",order=\"040\"}".into(), 1.0, ); assert!(lifecycle_complete( diff --git a/docs/operations.md b/docs/operations.md index 3061a52a..aebd9856 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -97,6 +97,28 @@ If decoder queue stays deep, check `walshadow_queue_records_out_total` for progress. Use rate of `walshadow_decoder_buffer_lock_wait_seconds_total` to measure time spent waiting for transaction buffer lock +An initial load reports its phase through `walshadow_stage_active{stage}`: +`bootstrap` spans the whole load, and inside it `bootstrap_walk` (page walk +off the backup), `bootstrap_window` (replay of the backup window's WAL, +streamed live beside the walk or read back from the landed WAL after it), +`bootstrap_deferred` (TOAST referrers resolved against the chunk store) and +`bootstrap_gate` (undecided tuples resolved against pg_xact) light up in +turn. `walshadow_stage_seconds_total` and `_completed_total` carry the same +labels. Every bootstrap gauge keeps updating until the load hands off to +streaming, so a frozen `bootstrap_pages_walked_total` under an active +`bootstrap_gate` is expected: the walk is over, the gate pass is running. +Gate pass progress is `walshadow_stage_progress{measure="work_done"}` against +`{measure="work_total"}` for `stage="bootstrap_gate"`, and its decisions +accrue live in `walshadow_bootstrap_gate_tuples_total{verdict}`. + +The daemon also logs its replication scope once at start and every hour +after, on target `walshadow::config`: per source database one +`event="shadow config"` line with the source endpoint, version, system id and +table count, then one `event="table config"` line per replicated table with +its destination. Log +pipelines can render the current table list from the newest of these lines +without a per-table metric series. + Metrics for work a source database owns carry a `database=` label: bridge requests, descriptor log and capture counters, and config overlay counters. Sum them when a panel wants one number for the daemon. Cluster WAL positions, diff --git a/plans/observability.md b/plans/observability.md index 9110c24f..07dff110 100644 --- a/plans/observability.md +++ b/plans/observability.md @@ -45,7 +45,7 @@ expected siblings is too narrow. | `commit_timestamp_seconds` | `at` | source, ack | shadow_replay once the pump keeps an LSN→commit-time ring | | `source_mode` | `mode` | the six one-hot states, `initial_load` included | complete | | `stage_active`, `stage_seconds_total`, `stage_completed_total`, `stage_interrupted_total` | `stage`, `title`, `order` | ten stages | copy and publish phases as the table backfill grows | -| `stage_count` | `stage`, `title`, `order`, `measure` | work_total, work_done, rows | complete | +| `stage_progress` | `stage`, `title`, `order`, `measure` | work_total, work_done, rows | complete | | `worker_seconds_total` | `worker`, `state` | pump, queue, batcher, inserter, backup_tap, backup_walk × busy, idle, blocked_in, blocked_out | reorder if it ever leaves the queue worker | | `errors_total` | `component`, `kind` | source: wal_removed, disconnect, walsender_exit, other | ClickHouse insert retries and reconnects, DDL retries, bridge errors, oracle errors, timeline switch failures, endpoint swap failures | | `request_duration_seconds` | `database`, `target`, `op` | bridge × op | clickhouse insert, archive fetch | @@ -71,8 +71,8 @@ family they read. | `bootstrap_bytes_tapped`, `backfill_backup_bytes_total`, `backfill_copy_bytes_total`, `timeline_prefix_bytes_verified_total` | `wal_bytes_total{flow}` | | `archive_wal_segments_total` | `wal_bytes_total{flow="archive_fetched"}` in bytes | | `xact_bytes_in_memory`, `spill_bytes_active`, `drain_resident_bytes`, `drain_chunk_resident_bytes`, `drain_row_resident_bytes`, `toast_xact_spool_bytes`, `resident_payload_bytes`, `resident_payload_peak_bytes`, `raw_pending_bytes`, `desc_log_tail_bytes` | `buffer_bytes{buffer}` | -| `bootstrap_deferred_bytes`, `bootstrap_deferred_spool_bytes`, `bootstrap_deferred_replay_bytes`, `bootstrap_deferred_replayed_bytes` | derive: `stage_count` for bootstrap_deferred already carries the replay ratio; the spool byte gauges also wrap below zero (u64 underflow in the deferred footprint), fix or drop | -| `bootstrap_parts_total`, `bootstrap_parts_done`, `bootstrap_tuples_emitted`, `bootstrap_pages_walked`, `bootstrap_files_walked`, `bootstrap_files_skipped_unmapped`, `bootstrap_replay_covered_rows_total` | derive from `stage_count`; pages and files walked to debug | +| `bootstrap_deferred_bytes`, `bootstrap_deferred_spool_bytes`, `bootstrap_deferred_replay_bytes`, `bootstrap_deferred_replayed_bytes` | derive: `stage_progress` for bootstrap_deferred already carries the replay ratio; the spool byte gauges also wrap below zero (u64 underflow in the deferred footprint), fix or drop | +| `bootstrap_parts_total`, `bootstrap_parts_done`, `bootstrap_tuples_emitted`, `bootstrap_pages_walked`, `bootstrap_files_walked`, `bootstrap_files_skipped_unmapped`, `bootstrap_replay_covered_rows_total` | derive from `stage_progress`; pages and files walked to debug | | `bootstrap_gate_chunks_gated_total`, `bootstrap_gate_multixact_emitted_total` | debug | | `emitter_retries_attempted_total`, `emitter_reconnects_total`, `bridge_errors_by_op`, `bridge_reconnects_total`, `oracle_errors_total`, `oracle_conversion_errors_total`, `timeline_switch_failures_by_reason`, `source_endpoint_swap_failures_total`, `catalog_boundary_hold_failures_total`, `toast_fetch_miss_total` | `errors_total{component,kind}` | | `inserter_ch_seconds_total`, `archive_fetch_seconds_total` | `request_duration_seconds{target}` histograms; the counters go once the histograms exist | diff --git a/src/backfill/backup_backfill.rs b/src/backfill/backup_backfill.rs index dee7cd53..b58809e4 100644 --- a/src/backfill/backup_backfill.rs +++ b/src/backfill/backup_backfill.rs @@ -722,7 +722,15 @@ async fn gate_task( }; let segments = accums.segment_count(); let view = accums.view(&patch); - resolve_phase(deferred, &view, &tx, Some(&mut pending), &mut stats).await?; + resolve_phase( + deferred, + &view, + &tx, + Some(&mut pending), + &mut stats, + &counters, + ) + .await?; Ok((stats, segments, pending)) } diff --git a/src/backfill/visibility_gate.rs b/src/backfill/visibility_gate.rs index 88b82eb8..ddedfba9 100644 --- a/src/backfill/visibility_gate.rs +++ b/src/backfill/visibility_gate.rs @@ -160,6 +160,7 @@ pub async fn resolve_phase( tx: &mpsc::Sender>, mut pending_spool: Option<&mut PendingSpool>, stats: &mut GateStats, + counters: &EmitterStats, ) -> Result<(), String> { let mut out = SlabTx::new(tx); let mut replay = deferred @@ -172,17 +173,25 @@ pub async fn resolve_phase( .await .map_err(|e| format!("visibility gate: deferred spool replay: {e}"))? { + counters.gate_rechecked.fetch_add(1, Ordering::Relaxed); match tuple_visibility(t.xid, t.xmax, t.infomask, Some(view)) { Visibility::Emit => { if t.infomask & HEAP_XMAX_IS_MULTI != 0 { stats.multixact_emitted += 1; + counters + .gate_multixact_emitted + .fetch_add(1, Ordering::Relaxed); } stats.emitted += 1; + counters.gate_emitted.fetch_add(1, Ordering::Relaxed); if !out.push(t).await { break; } } - Visibility::Skip => stats.gated += 1, + Visibility::Skip => { + stats.gated += 1; + counters.gate_gated.fetch_add(1, Ordering::Relaxed); + } Visibility::Defer => { let pending = deferred_xids(t.xid, t.xmax, t.infomask, view); let retained = match pending_spool.as_mut() { @@ -191,8 +200,10 @@ pub async fn resolve_phase( }; if retained { stats.pending += 1; + counters.gate_pending.fetch_add(1, Ordering::Relaxed); } else { stats.gated += 1; + counters.gate_gated.fetch_add(1, Ordering::Relaxed); } } Visibility::Unresolvable => { @@ -482,6 +493,7 @@ pub async fn resolve_greenfield( &txs[0], Some(&mut pending_spool), &mut gate_stats, + &sink.stats, ) .await; if resolved.is_err() { @@ -1086,9 +1098,16 @@ mod tests { if spool.records() == 0 { continue; } - resolve_phase(spool, &view, &tx, None, &mut stats) - .await - .unwrap(); + resolve_phase( + spool, + &view, + &tx, + None, + &mut stats, + &EmitterStats::default(), + ) + .await + .unwrap(); } drop(tx); @@ -1136,9 +1155,16 @@ mod tests { tuple(100, 0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID), ]) .await; - resolve_phase(spool, &view, &tx, Some(&mut pending_spool), &mut stats) - .await - .unwrap(); + resolve_phase( + spool, + &view, + &tx, + Some(&mut pending_spool), + &mut stats, + &EmitterStats::default(), + ) + .await + .unwrap(); drop(tx); let visible = rx.recv().await.expect("visible rows"); @@ -1161,9 +1187,16 @@ mod tests { let (tx, _rx) = mpsc::channel(8); let mut stats = GateStats::default(); let spool = spool_at("no-pending", vec![tuple(100, 0, 0)]).await; - resolve_phase(spool, &view, &tx, None, &mut stats) - .await - .unwrap(); + resolve_phase( + spool, + &view, + &tx, + None, + &mut stats, + &EmitterStats::default(), + ) + .await + .unwrap(); assert_eq!(stats.pending, 0); assert_eq!(stats.gated, 1); } @@ -1191,11 +1224,17 @@ mod tests { let (tx, mut rx) = mpsc::channel(4); let mut stats = GateStats::default(); let spool = spool_at("lock-only", vec![locked()]).await; - resolve_phase(spool, &view, &tx, None, &mut stats) + let counters = EmitterStats::default(); + resolve_phase(spool, &view, &tx, None, &mut stats, &counters) .await .unwrap(); drop(tx); assert_eq!((stats.emitted, stats.multixact_emitted), (1, 1)); + let ld = |a: &std::sync::atomic::AtomicU64| a.load(Ordering::Relaxed); + assert_eq!( + (ld(&counters.gate_rechecked), ld(&counters.gate_emitted)), + (1, 1) + ); assert_eq!(rx.recv().await.map(|slab| slab.len()), Some(1)); } @@ -1214,9 +1253,16 @@ mod tests { )]) .await; - let err = resolve_phase(spool, &view, &tx, None, &mut stats) - .await - .unwrap_err(); + let err = resolve_phase( + spool, + &view, + &tx, + None, + &mut stats, + &EmitterStats::default(), + ) + .await + .unwrap_err(); assert!(err.contains("pg_multixact"), "{err}"); assert_eq!(stats.emitted, 0); } diff --git a/src/bin/stream/bootstrap.rs b/src/bin/stream/bootstrap.rs index 14f31ca7..72bc1a8c 100644 --- a/src/bin/stream/bootstrap.rs +++ b/src/bin/stream/bootstrap.rs @@ -30,7 +30,7 @@ use walshadow::config::{CliOverrides, ConfigResolver, cli_over_toml}; use walshadow::decoder_sink::MetricsTupleObserver; use walshadow::emit::route::routing; use walshadow::mapping::MappingHandle; -use walshadow::metrics::{MetricsRegistry, MetricsSnapshot}; +use walshadow::metrics::{MetricsRegistry, MetricsSnapshot, SourceMode}; use walshadow::pipeline::Fatal; use walshadow::pipeline::tail::OwnedTail; use walshadow::runtime_config::InitialLoadMode; @@ -408,7 +408,7 @@ pub(crate) async fn run_bootstrap( resolver: None, backfiller: None, }; - let ticker = tokio_util::task::AbortOnDropHandle::new(tokio::spawn({ + let _ticker = tokio_util::task::AbortOnDropHandle::new(tokio::spawn({ let metrics = metrics.clone(); let progress = progress.clone(); let stats = bootstrap_stats.clone(); @@ -421,6 +421,7 @@ pub(crate) async fn run_bootstrap( metrics .set(MetricsSnapshot { by_database: vec![boot_db.series()], + source_mode: SourceMode::InitialLoad.one_hot(), ..stage_gauges(&StageCounters { emitter: Some(&stats), db_name: &|_| boot_db.database.clone(), @@ -434,6 +435,9 @@ pub(crate) async fn run_bootstrap( } } })); + let walk_timing = resume + .is_none() + .then(|| walshadow::ops::stages::BOOTSTRAP_WALK.start()); let (rx, pump) = match &resume { Some(done) => { let (_tx, rx) = tokio::sync::mpsc::channel(1); @@ -662,19 +666,30 @@ pub(crate) async fn run_bootstrap( }; let leg_fut = async { match live_cfg { - Some(cfg) => walshadow::backfill::bootstrap_window::stream_window( - cfg, - feed, - source_ident.xlogpos, - stop_rx, - ) - .await - .map(Some), + Some(cfg) => walshadow::ops::stages::BOOTSTRAP_WINDOW + .measure(walshadow::backfill::bootstrap_window::stream_window( + cfg, + feed, + source_ident.xlogpos, + stop_rx, + )) + .await + .map(Some), None => Ok(None), } }; - let (gate_res, stage_res, pump_res, leg_res) = - tokio::join!(gate, stages.join(), pump_then_stop, leg_fut); + let walk = async { + let (gate_res, stage_res, pump_res) = tokio::join!(gate, stages.join(), pump_then_stop); + if let Some(timing) = walk_timing + && gate_res.is_ok() + && stage_res.is_ok() + && matches!(pump_res, Ok(Ok(_))) + { + timing.finish(); + } + (gate_res, stage_res, pump_res) + }; + let ((gate_res, stage_res, pump_res), leg_res) = tokio::join!(walk, leg_fut); let prepared = (|| -> Result<_> { let drain_outcome = stage_res.map_err(|e| anyhow::anyhow!(e))?; let (gate_stats, gate_spool) = @@ -877,23 +892,24 @@ pub(crate) async fn run_bootstrap( // Replay backup WAL before resolving deferred value references if let Some(mut cfg) = window_cfg { cfg.timeline = outcome.start.timeline; - let replayed = async { - let segments = walshadow::backfill::bootstrap_window::segments_in_dir( - &window_wal, - outcome.start.timeline, - outcome.start.start_lsn, - outcome.end.end_lsn, - ) - .await?; - walshadow::backfill::bootstrap_window::replay_segments( - cfg, - &segments, - outcome.start.start_lsn, - outcome.end.end_lsn, - ) - .await - } - .await; + let replayed = walshadow::ops::stages::BOOTSTRAP_WINDOW + .measure(async { + let segments = walshadow::backfill::bootstrap_window::segments_in_dir( + &window_wal, + outcome.start.timeline, + outcome.start.start_lsn, + outcome.end.end_lsn, + ) + .await?; + walshadow::backfill::bootstrap_window::replay_segments( + cfg, + &segments, + outcome.start.start_lsn, + outcome.end.end_lsn, + ) + .await + }) + .await; match replayed { Ok(w) => window = Some(w), Err(e) => { @@ -936,7 +952,10 @@ pub(crate) async fn run_bootstrap( } let mut deferred_rows = 0; if !handback.is_empty() { - match sink.resolve_deferred(handback, coverage.clone()).await { + match walshadow::ops::stages::BOOTSTRAP_DEFERRED + .measure(sink.resolve_deferred(handback, coverage.clone())) + .await + { Ok(resolved) => { for (i, lane) in handback_at.into_iter().zip(resolved) { drain_outcome[i].next_seq = lane.next_seq; @@ -988,6 +1007,9 @@ pub(crate) async fn run_bootstrap( let outcome: BootstrapOutcome = pump_res .context("bootstrap pump join")? .context("bootstrap pump")?; + if let Some(timing) = walk_timing { + timing.finish(); + } (shipped, outcome, None, Some(landing_tracker)) }; @@ -1039,7 +1061,6 @@ pub(crate) async fn run_bootstrap( files_skipped_unmapped = outcome.page_walk.files_skipped_unmapped.load(Ordering::Relaxed), "bootstrap stage timings", ); - ticker.abort(); if let Some((settings, storage)) = wal_hydrate { fetch_wal_into_pg_wal( @@ -1094,10 +1115,15 @@ pub(crate) async fn run_bootstrap( // Resolve deferred tuples after window transaction overlay is complete let recorded = if let Some(pending) = pending_gate { let patch = std::mem::take(&mut *window_patch.lock().expect("window patch lock")).seal(); - let (gate, pending_tables) = - resolve_greenfield(pending, &shadow_data_dir, &patch, source_major) - .await - .context("bootstrap: visibility gate")?; + let (gate, pending_tables) = walshadow::ops::stages::BOOTSTRAP_GATE + .measure(resolve_greenfield( + pending, + &shadow_data_dir, + &patch, + source_major, + )) + .await + .context("bootstrap: visibility gate")?; // Persist the ledger before clearing the marker so pending rows // already in ClickHouse can be published after restart let mut ledger = PendingLedger::load(&args.spill_dir, source_ident.system_id) diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index ebd8d7d7..c6ecbfd8 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -530,7 +530,29 @@ fn skipped_by_rel( .collect() } +fn mirror_stage_work(v: &StageCounters<'_>) { + use walshadow::ops::stages::{ + BOOTSTRAP_DEFERRED, BOOTSTRAP_GATE, BOOTSTRAP_WALK, BOOTSTRAP_WINDOW, + }; + let ld = |a: &AtomicU64| a.load(Ordering::Relaxed); + let Some(p) = v.bootstrap else { + return; + }; + BOOTSTRAP_WALK.set_work(ld(&p.pump.parts_done), ld(&p.pump.parts_total)); + BOOTSTRAP_WALK.set_rows(ld(&p.page_walk.tuples_emitted)); + if let Some(s) = v.emitter { + BOOTSTRAP_WINDOW.set_rows(ld(&s.bootstrap_replay_covered_rows)); + BOOTSTRAP_DEFERRED.set_work( + ld(&s.bootstrap_deferred_replayed_bytes), + ld(&s.bootstrap_deferred_replay_bytes), + ); + let rechecked = ld(&s.gate_rechecked); + BOOTSTRAP_GATE.set_work(rechecked, ld(&s.gate_deferred).max(rechecked)); + } +} + pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> MetricsSnapshot { + mirror_stage_work(v); let (proc_cpu, proc_rss, proc_threads) = read_process_stats(); let by_table = |pick: fn(&EmitterStats) -> &TableRowCounts| { v.emitter.map_or_else(Vec::new, |s| { @@ -589,9 +611,15 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M toast_values_filled_oversize_total: emitter(|s| &s.toast_values_filled_oversize), toast_fetch_miss_total: emitter(|s| &s.toast_fetch_miss), bootstrap_replay_covered_rows_total: emitter(|s| &s.bootstrap_replay_covered_rows), - bootstrap_gate_emitted_total: emitter(|s| &s.gate_emitted), - bootstrap_gate_gated_total: emitter(|s| &s.gate_gated), - bootstrap_gate_deferred_total: emitter(|s| &s.gate_deferred), + bootstrap_gate_tuples_by_verdict: emitter_counts( + v.emitter, + [ + |s| &s.gate_emitted, + |s| &s.gate_gated, + |s| &s.gate_deferred, + |s| &s.gate_pending, + ], + ), bootstrap_gate_chunks_gated_total: emitter(|s| &s.gate_chunks_gated), bootstrap_gate_multixact_emitted_total: emitter(|s| &s.gate_multixact_emitted), toast_mirror_truncates_total: emitter(|s| &s.toast_mirror_truncates), diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index b4eb3254..ae1fad5a 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -77,6 +77,8 @@ use crate::source_recovery::{ resume_source_feed, stream_branch, stream_error_kind, swap_reason, }; +const CONFIG_LOG_INTERVAL: Duration = Duration::from_secs(3600); + /// Wait out an unreachable ClickHouse at boot rather than exit, which leaves /// a supervisor restart-looping the daemon until the destination returns async fn ensure_boot_database_waiting(cfg: &EmitterConfig) -> Result<()> { @@ -1527,6 +1529,7 @@ async fn session_body( } let mut segments_shipped = 0u64; + let mut config_logged: Option = None; let mut prev_dispatched = stream.dispatched_lsn(); let mut rate_estimator = RateEstimator::default(); // Manifest write cadence. Slot safety doesn't ride on it: advertised @@ -2325,6 +2328,17 @@ async fn session_body( let drain_for_metric = xact_stats.drain_lsn; let ack_commit_ts = ack_probe.borrow().ack_commit_ts; let source_wal_end = matches!(path, SourcePath::Live).then(|| feed.last_server_wal_end()); + if config_logged.is_none_or(|t| t.elapsed() >= CONFIG_LOG_INTERVAL) { + config_logged = Some(Instant::now()); + for resolver in &config_resolvers { + log_replication_config( + &resolver.subscribe().borrow(), + &source_conn.endpoint(), + feed.server_version_num(), + &ident.system_id, + ); + } + } populate_metrics( &metrics, received, @@ -2590,6 +2604,41 @@ impl SessionTasks { } } +fn log_replication_config( + cfg: &walshadow::config::ResolvedConfig, + source_endpoint: &str, + source_version_num: i32, + system_id: &dyn std::fmt::Display, +) { + tracing::info!( + target: "walshadow::config", + event = "shadow config", + source_endpoint, + source_version_num, + system_id = %system_id, + source_dbname = %cfg.source.dbname, + tables = cfg.tables.len(), + namespaces = cfg.namespaces.len(), + paused = cfg.paused, + row_budget = cfg.row_budget, + byte_budget = cfg.byte_budget, + ); + let mut tables: Vec<_> = cfg.tables.iter().collect(); + tables.sort_by(|a, b| a.0.cmp(b.0)); + for (rel, mapping) in tables { + tracing::info!( + target: "walshadow::config", + event = "table config", + source_table = %rel, + target_database = %mapping.target.database, + target_table = %mapping.target.table, + target_instance = mapping.target.instance.as_deref().unwrap_or(""), + columns = mapping.columns.len(), + tees = mapping.tees.len(), + ); + } +} + /// Error for a task that stopped before its shutdown, preferring the fatal /// it set on the way out pub(crate) fn task_stopped( diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 7f473430..6aeb1110 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -2702,6 +2702,8 @@ crate::atomic_stats! { pub gate_deferred, pub gate_chunks_gated, pub gate_multixact_emitted, + pub gate_pending, + pub gate_rechecked, /// Chunk rows mirrored from a restored TOAST page image, repairing /// backup page copies read mid-write pub toast_image_rows_mirrored, diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 2f225df9..f352e58b 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -135,6 +135,15 @@ label_class! { } } +label_class! { + pub enum GateVerdict { + Emitted => "emitted", + Gated => "gated", + Deferred => "deferred", + Pending => "pending", + } +} + label_class! { pub enum WorkerState { Busy => "busy", @@ -421,8 +430,9 @@ snapshot! { counter bootstrap_files_skipped_unmapped: u64 = "User-heap segments declined at begin because no mapped relation owns them; their bytes drain unread.", counter spill_evictions_total: u64 = "Total evictions in→spill since daemon start.", - counter xacts_committed_total: u64 = "Total xacts drained as commits since daemon start.", - counter xacts_aborted_total: u64 = "Total xacts dropped as aborts since daemon start.", + counter xacts_committed_total: u64 = + "Commits drained that carried buffered rows for mapped tables; walshadow_emitter_xacts_total counts every drained commit.", + counter xacts_aborted_total: u64 = "Aborts that discarded buffered rows for mapped tables.", counter decoder_decoded_total: u64 = "Heap records decoded since daemon start.", counter decoder_records_by_op: [u64; 5] ["op" = ["insert", "update", "hot_update", "delete", "truncate"]] = @@ -480,12 +490,9 @@ snapshot! { "TOAST resolution failures caused by missing chunks or size mismatches.", counter bootstrap_replay_covered_rows_total: u64 = "Backup rows dropped during deferred TOAST resolution because values were unreadable and window replay emitted a tombstone or newer version for each tuple's slot.", - counter bootstrap_gate_emitted_total: u64 = - "Heap tuples forwarded by bootstrap visibility gate using hint bits.", - counter bootstrap_gate_gated_total: u64 = - "Heap tuples dropped by bootstrap visibility gate using hint bits.", - counter bootstrap_gate_deferred_total: u64 = - "Heap tuples with undecided hint-bit visibility, spooled for transaction-status checks.", + counter bootstrap_gate_tuples_by_verdict: [u64; GateVerdict::COUNT] + ["verdict" = GateVerdict::LABELS] = + "Heap tuples by the bootstrap visibility gate's verdict: emitted (forwarded, from hint bits during the walk or the pg_xact pass after it), gated (dropped as dead), deferred (undecided by hint bits, spooled for the pg_xact pass), pending (writer or deleter still open at backup end, parked for the live stream to settle). A deferred tuple is counted again under the verdict the pg_xact pass gives it, so the verdicts do not sum to tuples seen. Gate pass progress is walshadow_stage_progress{measure=\"work_done\"} over {measure=\"work_total\"} for stage bootstrap_gate.", counter bootstrap_gate_chunks_gated_total: u64 = "TOAST chunk tuples excluded from mirror writes because hint bits proved them dead.", counter bootstrap_gate_multixact_emitted_total: u64 = @@ -896,7 +903,7 @@ const BACKFILL_MODES: [&str; 3] = ["copy", "base_backup", "object_store"]; /// Family declaration off a snapshot field name: a `_by_