Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions bench/src/initial_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,12 @@ fn lifecycle_complete(
after: &BTreeMap<String, f64>,
) -> bool {
let finished = |stage| {
let key = format!("walshadow_stage_completed_total{{stage=\"{stage}\"}}");
after
.get(&key)
.is_some_and(|value| *value > *before.get(&key).unwrap_or(&0.0))
let prefix = format!("walshadow_stage_completed_total{{stage=\"{stage}\"");
after.iter().any(|(key, value)| {
key.strip_prefix(&prefix)
.is_some_and(|rest| rest.starts_with(',') || rest.starts_with('}'))
&& *value > *before.get(key).unwrap_or(&0.0)
})
};
if bootstrap {
finished("bootstrap") && finished("shadow_replay")
Expand Down Expand Up @@ -557,7 +559,7 @@ mod tests {
#[test]
fn completion_requires_new_stage_work_and_settlement() {
let before = metrics_values(
"walshadow_stage_completed_total{stage=\"publish\"} 2\nwalshadow_stage_completed_total{stage=\"settle\"} 2\nwalshadow_config_backfills_pending 0\n",
"walshadow_stage_completed_total{stage=\"publish\",title=\"Publish\",order=\"090\"} 2\nwalshadow_stage_completed_total{stage=\"settle\",title=\"Settle\",order=\"100\"} 2\nwalshadow_config_backfills_pending 0\n",
);
assert!(!lifecycle_complete(
false,
Expand All @@ -567,7 +569,8 @@ mod tests {
));
let mut after = before.clone();
after.insert(
"walshadow_stage_completed_total{stage=\"publish\"}".into(),
"walshadow_stage_completed_total{stage=\"publish\",title=\"Publish\",order=\"090\"}"
.into(),
3.0,
);
assert!(!lifecycle_complete(
Expand All @@ -577,7 +580,8 @@ mod tests {
&after
));
after.insert(
"walshadow_stage_completed_total{stage=\"settle\"}".into(),
"walshadow_stage_completed_total{stage=\"settle\",title=\"Settle\",order=\"100\"}"
.into(),
3.0,
);
assert!(lifecycle_complete(
Expand All @@ -599,7 +603,7 @@ mod tests {
fn bootstrap_requires_snapshot_and_shadow_replay() {
let empty = BTreeMap::new();
let mut after = metrics_values(
"walshadow_stage_completed_total{stage=\"copy\"} 1\nwalshadow_config_backfills_pending 0\n",
"walshadow_stage_completed_total{stage=\"copy\",title=\"Table COPY\",order=\"070\"} 1\nwalshadow_config_backfills_pending 0\n",
);
assert!(!lifecycle_complete(
true,
Expand All @@ -608,7 +612,8 @@ mod tests {
&after
));
after.insert(
"walshadow_stage_completed_total{stage=\"bootstrap\"}".into(),
"walshadow_stage_completed_total{stage=\"bootstrap\",title=\"Initial load\",order=\"010\"}"
.into(),
1.0,
);
assert!(!lifecycle_complete(
Expand All @@ -618,7 +623,7 @@ mod tests {
&after
));
after.insert(
"walshadow_stage_completed_total{stage=\"shadow_replay\"}".into(),
"walshadow_stage_completed_total{stage=\"shadow_replay\",title=\"Shadow start\",order=\"040\"}".into(),
1.0,
);
assert!(lifecycle_complete(
Expand Down
22 changes: 22 additions & 0 deletions docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,28 @@ If decoder queue stays deep, check `walshadow_queue_records_out_total` for
progress. Use rate of `walshadow_decoder_buffer_lock_wait_seconds_total` to
measure time spent waiting for transaction buffer lock

An initial load reports its phase through `walshadow_stage_active{stage}`:
`bootstrap` spans the whole load, and inside it `bootstrap_walk` (page walk
off the backup), `bootstrap_window` (replay of the backup window's WAL,
streamed live beside the walk or read back from the landed WAL after it),
`bootstrap_deferred` (TOAST referrers resolved against the chunk store) and
`bootstrap_gate` (undecided tuples resolved against pg_xact) light up in
turn. `walshadow_stage_seconds_total` and `_completed_total` carry the same
labels. Every bootstrap gauge keeps updating until the load hands off to
streaming, so a frozen `bootstrap_pages_walked_total` under an active
`bootstrap_gate` is expected: the walk is over, the gate pass is running.
Gate pass progress is `walshadow_stage_progress{measure="work_done"}` against
`{measure="work_total"}` for `stage="bootstrap_gate"`, and its decisions
accrue live in `walshadow_bootstrap_gate_tuples_total{verdict}`.

The daemon also logs its replication scope once at start and every hour
after, on target `walshadow::config`: per source database one
`event="shadow config"` line with the source endpoint, version, system id and
table count, then one `event="table config"` line per replicated table with
its destination. Log
pipelines can render the current table list from the newest of these lines
without a per-table metric series.

Metrics for work a source database owns carry a `database=` label: bridge
requests, descriptor log and capture counters, and config overlay counters.
Sum them when a panel wants one number for the daemon. Cluster WAL positions,
Expand Down
6 changes: 3 additions & 3 deletions plans/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ expected siblings is too narrow.
| `commit_timestamp_seconds` | `at` | source, ack | shadow_replay once the pump keeps an LSN→commit-time ring |
| `source_mode` | `mode` | the six one-hot states, `initial_load` included | complete |
| `stage_active`, `stage_seconds_total`, `stage_completed_total`, `stage_interrupted_total` | `stage`, `title`, `order` | ten stages | copy and publish phases as the table backfill grows |
| `stage_count` | `stage`, `title`, `order`, `measure` | work_total, work_done, rows | complete |
| `stage_progress` | `stage`, `title`, `order`, `measure` | work_total, work_done, rows | complete |
| `worker_seconds_total` | `worker`, `state` | pump, queue, batcher, inserter, backup_tap, backup_walk × busy, idle, blocked_in, blocked_out | reorder if it ever leaves the queue worker |
| `errors_total` | `component`, `kind` | source: wal_removed, disconnect, walsender_exit, other | ClickHouse insert retries and reconnects, DDL retries, bridge errors, oracle errors, timeline switch failures, endpoint swap failures |
| `request_duration_seconds` | `database`, `target`, `op` | bridge × op | clickhouse insert, archive fetch |
Expand All @@ -71,8 +71,8 @@ family they read.
| `bootstrap_bytes_tapped`, `backfill_backup_bytes_total`, `backfill_copy_bytes_total`, `timeline_prefix_bytes_verified_total` | `wal_bytes_total{flow}` |
| `archive_wal_segments_total` | `wal_bytes_total{flow="archive_fetched"}` in bytes |
| `xact_bytes_in_memory`, `spill_bytes_active`, `drain_resident_bytes`, `drain_chunk_resident_bytes`, `drain_row_resident_bytes`, `toast_xact_spool_bytes`, `resident_payload_bytes`, `resident_payload_peak_bytes`, `raw_pending_bytes`, `desc_log_tail_bytes` | `buffer_bytes{buffer}` |
| `bootstrap_deferred_bytes`, `bootstrap_deferred_spool_bytes`, `bootstrap_deferred_replay_bytes`, `bootstrap_deferred_replayed_bytes` | derive: `stage_count` for bootstrap_deferred already carries the replay ratio; the spool byte gauges also wrap below zero (u64 underflow in the deferred footprint), fix or drop |
| `bootstrap_parts_total`, `bootstrap_parts_done`, `bootstrap_tuples_emitted`, `bootstrap_pages_walked`, `bootstrap_files_walked`, `bootstrap_files_skipped_unmapped`, `bootstrap_replay_covered_rows_total` | derive from `stage_count`; pages and files walked to debug |
| `bootstrap_deferred_bytes`, `bootstrap_deferred_spool_bytes`, `bootstrap_deferred_replay_bytes`, `bootstrap_deferred_replayed_bytes` | derive: `stage_progress` for bootstrap_deferred already carries the replay ratio; the spool byte gauges also wrap below zero (u64 underflow in the deferred footprint), fix or drop |
| `bootstrap_parts_total`, `bootstrap_parts_done`, `bootstrap_tuples_emitted`, `bootstrap_pages_walked`, `bootstrap_files_walked`, `bootstrap_files_skipped_unmapped`, `bootstrap_replay_covered_rows_total` | derive from `stage_progress`; pages and files walked to debug |
| `bootstrap_gate_chunks_gated_total`, `bootstrap_gate_multixact_emitted_total` | debug |
| `emitter_retries_attempted_total`, `emitter_reconnects_total`, `bridge_errors_by_op`, `bridge_reconnects_total`, `oracle_errors_total`, `oracle_conversion_errors_total`, `timeline_switch_failures_by_reason`, `source_endpoint_swap_failures_total`, `catalog_boundary_hold_failures_total`, `toast_fetch_miss_total` | `errors_total{component,kind}` |
| `inserter_ch_seconds_total`, `archive_fetch_seconds_total` | `request_duration_seconds{target}` histograms; the counters go once the histograms exist |
Expand Down
10 changes: 9 additions & 1 deletion src/backfill/backup_backfill.rs
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,15 @@ async fn gate_task(
};
let segments = accums.segment_count();
let view = accums.view(&patch);
resolve_phase(deferred, &view, &tx, Some(&mut pending), &mut stats).await?;
resolve_phase(
deferred,
&view,
&tx,
Some(&mut pending),
&mut stats,
&counters,
)
.await?;
Ok((stats, segments, pending))
}

Expand Down
74 changes: 60 additions & 14 deletions src/backfill/visibility_gate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ pub async fn resolve_phase(
tx: &mpsc::Sender<Vec<BackfillTuple>>,
mut pending_spool: Option<&mut PendingSpool>,
stats: &mut GateStats,
counters: &EmitterStats,
) -> Result<(), String> {
let mut out = SlabTx::new(tx);
let mut replay = deferred
Expand All @@ -172,17 +173,25 @@ pub async fn resolve_phase(
.await
.map_err(|e| format!("visibility gate: deferred spool replay: {e}"))?
{
counters.gate_rechecked.fetch_add(1, Ordering::Relaxed);
match tuple_visibility(t.xid, t.xmax, t.infomask, Some(view)) {
Visibility::Emit => {
if t.infomask & HEAP_XMAX_IS_MULTI != 0 {
stats.multixact_emitted += 1;
counters
.gate_multixact_emitted
.fetch_add(1, Ordering::Relaxed);
}
stats.emitted += 1;
counters.gate_emitted.fetch_add(1, Ordering::Relaxed);
if !out.push(t).await {
break;
}
}
Visibility::Skip => stats.gated += 1,
Visibility::Skip => {
stats.gated += 1;
counters.gate_gated.fetch_add(1, Ordering::Relaxed);
}
Visibility::Defer => {
let pending = deferred_xids(t.xid, t.xmax, t.infomask, view);
let retained = match pending_spool.as_mut() {
Expand All @@ -191,8 +200,10 @@ pub async fn resolve_phase(
};
if retained {
stats.pending += 1;
counters.gate_pending.fetch_add(1, Ordering::Relaxed);
} else {
stats.gated += 1;
counters.gate_gated.fetch_add(1, Ordering::Relaxed);
}
}
Visibility::Unresolvable => {
Expand Down Expand Up @@ -482,6 +493,7 @@ pub async fn resolve_greenfield(
&txs[0],
Some(&mut pending_spool),
&mut gate_stats,
&sink.stats,
)
.await;
if resolved.is_err() {
Expand Down Expand Up @@ -1086,9 +1098,16 @@ mod tests {
if spool.records() == 0 {
continue;
}
resolve_phase(spool, &view, &tx, None, &mut stats)
.await
.unwrap();
resolve_phase(
spool,
&view,
&tx,
None,
&mut stats,
&EmitterStats::default(),
)
.await
.unwrap();
}
drop(tx);

Expand Down Expand Up @@ -1136,9 +1155,16 @@ mod tests {
tuple(100, 0, HEAP_XMIN_COMMITTED | HEAP_XMAX_INVALID),
])
.await;
resolve_phase(spool, &view, &tx, Some(&mut pending_spool), &mut stats)
.await
.unwrap();
resolve_phase(
spool,
&view,
&tx,
Some(&mut pending_spool),
&mut stats,
&EmitterStats::default(),
)
.await
.unwrap();
drop(tx);

let visible = rx.recv().await.expect("visible rows");
Expand All @@ -1161,9 +1187,16 @@ mod tests {
let (tx, _rx) = mpsc::channel(8);
let mut stats = GateStats::default();
let spool = spool_at("no-pending", vec![tuple(100, 0, 0)]).await;
resolve_phase(spool, &view, &tx, None, &mut stats)
.await
.unwrap();
resolve_phase(
spool,
&view,
&tx,
None,
&mut stats,
&EmitterStats::default(),
)
.await
.unwrap();
assert_eq!(stats.pending, 0);
assert_eq!(stats.gated, 1);
}
Expand Down Expand Up @@ -1191,11 +1224,17 @@ mod tests {
let (tx, mut rx) = mpsc::channel(4);
let mut stats = GateStats::default();
let spool = spool_at("lock-only", vec![locked()]).await;
resolve_phase(spool, &view, &tx, None, &mut stats)
let counters = EmitterStats::default();
resolve_phase(spool, &view, &tx, None, &mut stats, &counters)
.await
.unwrap();
drop(tx);
assert_eq!((stats.emitted, stats.multixact_emitted), (1, 1));
let ld = |a: &std::sync::atomic::AtomicU64| a.load(Ordering::Relaxed);
assert_eq!(
(ld(&counters.gate_rechecked), ld(&counters.gate_emitted)),
(1, 1)
);
assert_eq!(rx.recv().await.map(|slab| slab.len()), Some(1));
}

Expand All @@ -1214,9 +1253,16 @@ mod tests {
)])
.await;

let err = resolve_phase(spool, &view, &tx, None, &mut stats)
.await
.unwrap_err();
let err = resolve_phase(
spool,
&view,
&tx,
None,
&mut stats,
&EmitterStats::default(),
)
.await
.unwrap_err();
assert!(err.contains("pg_multixact"), "{err}");
assert_eq!(stats.emitted, 0);
}
Expand Down
Loading
Loading