From caadd0933894ff868eb551eb0fecb7bf38735de4 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:57:58 +0000 Subject: [PATCH] Fix bootstrap restart gaps A crash after bootstrap can skip rows when streaming restarts. Keep bootstrap position until streaming saves its manifest and descriptor logs --- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 2 +- docs/limitations.md | 25 ++- plans/bootstrap.md | 18 +-- src/backfill/bootstrap_marker.rs | 91 ++++++++++- src/bin/stream/bootstrap.rs | 41 ++++- src/bin/stream/session.rs | 11 +- src/crash_test.rs | 41 +++-- tests/bootstrap_handoff_crash_ch.rs | 108 +++++++++++++ tests/bootstrap_pending_shapes_ch.rs | 233 +++++++++++++++++++++++++++ tests/bootstrap_toast_reuse_ch.rs | 169 +++++++++++++++++++ 11 files changed, 693 insertions(+), 50 deletions(-) create mode 100644 tests/bootstrap_handoff_crash_ch.rs create mode 100644 tests/bootstrap_pending_shapes_ch.rs create mode 100644 tests/bootstrap_toast_reuse_ch.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c43f9267..918e7bc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,7 @@ jobs: - name: Install PostgreSQL ${{ matrix.pg-major }} (PGDG) run: | sudo systemctl stop postgresql 2>/dev/null || true - curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + curl -fsSL --retry 5 --retry-all-errors https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | sudo gpg --dearmor -o /usr/share/keyrings/postgres-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main ${{ matrix.pg-major }}" \ | sudo tee /etc/apt/sources.list.d/pgdg.list @@ -163,7 +163,7 @@ jobs: - name: Install ClickHouse (server + client) run: | sudo apt-get install -y apt-transport-https ca-certificates - curl -fsSL https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key \ + curl -fsSL --retry 5 --retry-all-errors https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key \ | sudo gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" \ | sudo tee /etc/apt/sources.list.d/clickhouse.list diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28d674a2..95713c35 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -182,7 +182,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ build-essential curl gnupg lsb-release - curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + curl -fsSL --retry 5 --retry-all-errors https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | sudo gpg --dearmor -o /usr/share/keyrings/postgres-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main ${{ matrix.pg-major }}" \ | sudo tee /etc/apt/sources.list.d/pgdg.list diff --git a/docs/limitations.md b/docs/limitations.md index dcef7d69..95d0cf3e 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -76,6 +76,9 @@ insert batch - restart can resend acknowledged-nearby rows, generated table engine deduplicates them during merge or `FINAL` - `tee` tables using engines other than `ReplacingMergeTree` keep duplicate rows after retries - destination queries without `FINAL` can observe multiple row versions until background merge +- initial-load rows use backup start as `_lsn` and can show stale or incorrect + values until backup-window replay writes newer row versions, see + [large values](#large-values) - bounded ClickHouse retry exhaustion stops daemon, supervisor restart continues from persisted floor ## Initial loads @@ -94,8 +97,12 @@ insert batch - restored TOAST page images in backup-window WAL mirror every chunk they carry, repairing pages the backup copied mid-write. They date from the page's own version, so walked chunks never outrank them -- an external value the mirror cannot reassemble stops the load rather than - substituting one; rerun against a fresher backup, or load the table with +- greenfield bootstrap reads deferred external values at heap page LSN or + backup start, whichever is later. It skips a backup row whose value cannot be + reassembled only if backup-window replay deleted or updated that row's heap + slot on a route that keeps deletes. Skipped rows increment + `walshadow_bootstrap_replay_covered_rows_total`. Other failures stop loading, + as do all such failures during per-table loads. Use a newer backup or `initial_load = 'copy'` - a multixact xmax the backup's `pg_multixact` cannot bound stops the load, with the same remedies @@ -137,11 +144,15 @@ Default `clickhouse` value mode uses persistent TOAST chunk mirrors. Missing required mirror tables stop replication, see [large toasted values](destination-tables.md#large-toasted-values) -Reused TOAST value IDs can leave ambiguous generations in backup chunk mirrors -when hint bits do not prove older chunks dead. Baseline rows and later -unchanged-pointer updates both resolve from those mirrors. Chunk lookup orders -by `(ver, blkno, offnum)` for determinism, not generation correctness: a newer -generation at a lower TID can lose when versions tie +PostgreSQL reuses a TOAST value ID only after all chunks of its previous value +are gone. During backup, chunk mirrors can contain both old and new values +under that ID. Initial-load rows read values at heap page LSN or backup start, +so chunks from restored page images can cause them to read an old value. +Backup-window replay writes correct values at commit LSN. If reassembly fails, +greenfield bootstrap skips a backup row only when replay deleted or updated its +slot. +After replay, `FINAL` returns correct values; reads without it can still return +old values until merges finish ### Shadow value mode diff --git a/plans/bootstrap.md b/plans/bootstrap.md index 8fba0658..b4654ff8 100644 --- a/plans/bootstrap.md +++ b/plans/bootstrap.md @@ -26,11 +26,11 @@ rows and deferred TOAST. Scratch writes alone cannot acknowledge deferred tuples ## Completion -Extend [pending-row settlement test](../tests/bootstrap_pending_settle_ch.rs) -beyond direct-bootstrap INSERT commit and rollback. Cover UPDATE and DELETE, -multixact updaters, subtransactions, deferred external values, and transactions -begun inside backup window. Repeat for object-store bootstrap and per-table -backup loads with explicit retained-history assumptions +[Pending-row tests](../tests/bootstrap_pending_shapes_ch.rs) cover updates, +deletes, multixact updaters, subtransactions, and external values during direct +bootstrap. Add transactions begun during backup. Repeat for object-store +bootstrap and per-table backup loads, stating which WAL history must remain +available Restart before settlement and after promotion but before ledger persistence. Interrupt pending-table cleanup and staged publication. Cover outcomes arriving @@ -46,11 +46,9 @@ convergence boundaries Backup-based initial loads issue no source SQL scans for user rows. Explicit `initial_load = "copy"` still scans selected table and remains `init`'s default. -Baseline external values in backup modes resolve from walked chunk mirrors, -so reused TOAST generations need a physical proof of age: backup transaction -logs and tuple-location order do not -establish it, and no source read compensates. Keep WAL replay ordered behind -required chunk history +Backup rows read external values from chunk mirrors populated during page walk +and window replay. [Bootstrap ordering](bootstrap_ordering.md) covers read +bounds and replacement evidence. Keep WAL replay behind required chunk writes To remove all source SQL, also build descriptors from copied catalogs, convert types while preserving source OIDs without reading source catalogs, and replace diff --git a/src/backfill/bootstrap_marker.rs b/src/backfill/bootstrap_marker.rs index 36c11a1d..df0cf447 100644 --- a/src/backfill/bootstrap_marker.rs +++ b/src/backfill/bootstrap_marker.rs @@ -17,6 +17,18 @@ pub struct BootstrapMarker { pub attempts: u32, #[serde(default, skip_serializing_if = "Option::is_none")] pub backup_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub handoff: Option, +} + +/// Keep bootstrap position until streaming saves its manifest and descriptor log +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Handoff { + pub end_lsn: u64, + pub timeline: u32, + /// Earliest record among transactions open at window seal + #[serde(default, skip_serializing_if = "Option::is_none")] + pub open_floor: Option, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -109,10 +121,21 @@ impl BootstrapMarker { .context("persist bootstrap marker") } - /// End the attempt. Pending rows already in ClickHouse must be recorded - /// first, so a restart can still publish them - pub async fn complete(dir: &Path, _: PendingRecorded) -> Result<()> { - ExtractedCheckpoint::clear(dir).await?; + /// Record pending rows first so restart can publish them. + /// Keep marker until streaming saves its resume state + pub async fn hand_off(dir: &Path, handoff: Handoff, _: PendingRecorded) -> Result<()> { + let marker = Self::read(dir)?.context("bootstrap marker vanished before handoff")?; + Self { + handoff: Some(handoff), + ..marker + } + .write(dir) + .await?; + ExtractedCheckpoint::clear(dir).await + } + + /// Clear marker after streaming saves its manifest and descriptor log + pub async fn complete(dir: &Path) -> Result<()> { Self::clear(dir).await } @@ -129,6 +152,7 @@ impl BootstrapMarker { Self { attempts: self.attempts + 1, backup_name: self.backup_name.clone(), + handoff: None, } } @@ -152,8 +176,12 @@ impl BootstrapMarker { } } +pub fn handed_off(dir: &Path) -> Result> { + Ok(BootstrapMarker::read(dir)?.and_then(|m| m.handoff)) +} + pub fn pending_attempt(dir: &Path, mode: BootstrapMode) -> Result> { - let Some(marker) = BootstrapMarker::read(dir)? else { + let Some(marker) = BootstrapMarker::read(dir)?.filter(|m| m.handoff.is_none()) else { return Ok(None); }; anyhow::ensure!( @@ -271,6 +299,7 @@ async fn first_attempt(data_dir: &Path, pin: Option) -> Result) -> Self { + Self { + end_lsn: handoff.end_lsn, + timeline: handoff.timeline, + open_floor: handoff.open_floor, + shadow, + } + } + /// Source or archive must retain crossing transaction records pub(crate) fn resume_lsn(&self) -> u64 { self.open_floor.unwrap_or(self.end_lsn).min(self.end_lsn) @@ -1256,6 +1267,8 @@ pub(crate) async fn bootstrap_build_mapping( pub(crate) enum ShadowStart { Bootstrap(PathBuf), Rebootstrap(PathBuf, BootstrapMarker), + /// Load finished, streaming never persisted its resume state + Handoff(PathBuf, bootstrap_marker::Handoff), Resume(PathBuf), } @@ -1266,7 +1279,10 @@ impl ShadowStart { pub(crate) fn data_dir(&self) -> &Path { match self { - Self::Bootstrap(d) | Self::Rebootstrap(d, _) | Self::Resume(d) => d, + Self::Bootstrap(d) + | Self::Rebootstrap(d, _) + | Self::Handoff(d, _) + | Self::Resume(d) => d, } } } @@ -1285,6 +1301,15 @@ pub(crate) fn resolve_shadow_start(args: &Args, mode: BootstrapMode) -> Result bool { + std::fs::read_to_string(path).is_ok_and(|armed| armed == format!("{point:?}")) +} + +fn crash(path: &Path, commit: u64) -> ! { + std::fs::write(path.with_extension("hit"), commit.to_string()).expect("record crash boundary"); + // SIGKILL skips atexit, where llvm-cov would otherwise dump counters + #[cfg(coverage)] + unsafe { + __llvm_profile_write_file() + }; + // SIGKILL prevents unwinding, shutdown drains, and final manifest writes. + unsafe { libc::raise(libc::SIGKILL) }; + unreachable!("SIGKILL returned"); +} diff --git a/tests/bootstrap_handoff_crash_ch.rs b/tests/bootstrap_handoff_crash_ch.rs new file mode 100644 index 00000000..8d69af61 --- /dev/null +++ b/tests/bootstrap_handoff_crash_ch.rs @@ -0,0 +1,108 @@ +//! Resume from saved bootstrap position after a crash before streaming starts + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use walshadow::mapping::TableTarget; +use walshadow::schema::RelName; + +const N_ROWS: i32 = 64; +const SCHEMA: &str = "s26"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn crash_at_bootstrap_handoff_keeps_rows_written_before_restart() { + if !fx::tools::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + fx::load_source_workload(&source, SCHEMA, N_ROWS).expect("load source workload"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + fx::create_ch_dest_table(&ch, "default", "t").expect("create ch table"); + + let ch_config_path = tmp.path().join("ch-config.toml"); + fx::write_ch_config_toml( + &ch_config_path, + "127.0.0.1", + slot.ch_tcp, + "default", + &RelName::new(SCHEMA, "t"), + &TableTarget::new("default", "t"), + ) + .expect("write ch-config"); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let arm = tmp.path().join("crash.arm"); + std::fs::write(&arm, "BootstrapHandoff").unwrap(); + // Slot retains WAL between crash and restart + let flags = ["--slot", "walshadow_handoff"]; + let child = daemon + .spawn_mode( + &source, + &ch_config_path, + slot.walsender, + "direct", + &flags, + &[("WALSHADOW_TEST_CRASH", arm.display().to_string())], + ) + .expect("spawn crashing daemon"); + let mut guard = fx::ChildGuard::new(child); + + let result: Result<()> = async { + let status = fx::wait_with_timeout( + guard.child.as_mut().expect("child"), + Duration::from_secs(60), + )?; + ensure!( + arm.with_extension("hit").exists(), + "daemon exited ({status}) without reaching bootstrap completion" + ); + guard.child = None; + + source + .psql_one(&format!( + "INSERT INTO {SCHEMA}.t SELECT g, 'gap-'||g::text \ + FROM generate_series(1001, 1010) g" + )) + .context("insert while daemon is down")?; + std::fs::remove_file(&arm)?; + + guard.child = Some( + daemon + .spawn(&source, &ch_config_path, slot.walsender, &flags) + .context("respawn daemon")?, + ); + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(30)) + .context("restarted daemon metrics endpoint never came up")?; + source + .psql_one(&format!("INSERT INTO {SCHEMA}.t VALUES (2001, 'after')")) + .context("insert after restart")?; + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE id = 2001", + "1", + Duration::from_secs(30), + )?; + fx::wait_ch_matches_source( + &ch, + &source, + &format!("{SCHEMA}.t"), + "default.t", + Duration::from_secs(10), + ) + } + .await; + + fx::finish_daemon(guard, &daemon, result); +} diff --git a/tests/bootstrap_pending_shapes_ch.rs b/tests/bootstrap_pending_shapes_ch.rs new file mode 100644 index 00000000..a4b0362e --- /dev/null +++ b/tests/bootstrap_pending_shapes_ch.rs @@ -0,0 +1,233 @@ +//! Settle pending updates, deletes, subtransactions, multixact updaters, and +//! external values after bootstrap + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::net::SocketAddr; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use tokio_postgres::Client; +use walshadow::mapping::TableTarget; +use walshadow::schema::RelName; +use walshadow::shadow::Shadow; +use walshadow::source_feed::open_sql_client; + +const N_ROWS: i32 = 64; +/// Keep writers open across backup window +const MAX_RATE_KIB: &str = "32768"; +const SCHEMA: &str = "s27"; + +fn wait_for_metric(addr: SocketAddr, name: &str, least: u64, timeout: Duration) -> Result { + let start = Instant::now(); + let mut seen = None; + while start.elapsed() < timeout { + let body = fx::http_get(addr, "/metrics")?; + let value = fx::parse_metric(&body, name); + if value.is_some_and(|v| v >= least) { + return Ok(value.expect("checked")); + } + seen = value; + std::thread::sleep(Duration::from_millis(200)); + } + bail!("{name} never reached {least}, last {seen:?}"); +} + +struct Writer { + client: Client, + commit: bool, +} + +async fn open_writer(source: &Shadow, tag: &str, commit: bool, sql: &str) -> Writer { + let client = open_sql_client(&fx::pg_cfg(source, tag)) + .await + .unwrap_or_else(|e| panic!("{tag} connect: {e}")); + client + .batch_execute(&format!("BEGIN; {sql}")) + .await + .unwrap_or_else(|e| panic!("{tag} writes before backup redo: {e}")); + Writer { client, commit } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pending_updates_deletes_subxacts_multixacts_and_external_values_settle() { + if !fx::tools::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + fx::load_source_workload(&source, SCHEMA, N_ROWS).expect("load source workload"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + fx::create_ch_dest_table(&ch, "default", "t").expect("create ch table"); + + let ch_config_path = tmp.path().join("ch-config.toml"); + fx::write_ch_config_toml( + &ch_config_path, + "127.0.0.1", + slot.ch_tcp, + "default", + &RelName::new(SCHEMA, "t"), + &TableTarget::new("default", "t"), + ) + .expect("write ch-config"); + + let t = format!("{SCHEMA}.t"); + // Keep row locks until updaters finish to preserve multixacts + let lockers = [ + open_writer( + &source, + "lock-a", + true, + &format!("SELECT * FROM {t} WHERE id = 22 FOR KEY SHARE"), + ) + .await, + open_writer( + &source, + "lock-b", + true, + &format!("SELECT * FROM {t} WHERE id = 23 FOR KEY SHARE"), + ) + .await, + ]; + let writers = [ + open_writer( + &source, + "upd-commit", + true, + &format!("UPDATE {t} SET name = 'upd-'||id WHERE id BETWEEN 1 AND 5"), + ) + .await, + open_writer( + &source, + "upd-abort", + false, + &format!("UPDATE {t} SET name = 'lost-'||id WHERE id BETWEEN 6 AND 10"), + ) + .await, + open_writer( + &source, + "del-commit", + true, + &format!("DELETE FROM {t} WHERE id BETWEEN 11 AND 15"), + ) + .await, + open_writer( + &source, + "del-abort", + false, + &format!("DELETE FROM {t} WHERE id BETWEEN 16 AND 20"), + ) + .await, + open_writer( + &source, + "subxact", + true, + &format!( + "INSERT INTO {t} SELECT g, 'sub-'||g FROM generate_series(3001, 3005) g; \ + SAVEPOINT a; \ + INSERT INTO {t} SELECT g, 'sub-'||g FROM generate_series(3006, 3010) g; \ + ROLLBACK TO a; \ + SAVEPOINT b; \ + UPDATE {t} SET name = 'sub-upd' WHERE id = 21; \ + RELEASE b" + ), + ) + .await, + open_writer( + &source, + "multi-commit", + true, + &format!("UPDATE {t} SET name = 'multi-'||id WHERE id = 22"), + ) + .await, + open_writer( + &source, + "multi-abort", + false, + &format!("UPDATE {t} SET name = 'multi-lost' WHERE id = 23"), + ) + .await, + // Incompressible enough to stay external after pglz + open_writer( + &source, + "external", + true, + &format!( + "INSERT INTO {t} SELECT 4001, string_agg(md5(g::text || 'x'), '') \ + FROM generate_series(1, 1000) g; \ + UPDATE {t} SET name = (SELECT string_agg(md5(g::text), '') \ + FROM generate_series(1, 1000) g) WHERE id = 24" + ), + ) + .await, + ]; + source + .apply_schema_dump("CHECKPOINT;\n") + .expect("checkpoint uncommitted pages"); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn( + &source, + &ch_config_path, + slot.walsender, + &[ + "--bootstrap-max-rate-kib", + MAX_RATE_KIB, + "--bootstrap-wind-down-secs", + "0", + "--slot", + "walshadow_pending_shapes", + ], + ) + .expect("spawn walshadow-stream"); + let guard = fx::ChildGuard::new(child); + + let result: Result<()> = async { + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(120)) + .context("daemon metrics endpoint never came up")?; + wait_for_metric( + daemon.metrics_addr, + "walshadow_pending_rows_total", + 40, + Duration::from_secs(60), + ) + .context("pending rows never reached ClickHouse")?; + + for w in writers.iter().chain(&lockers) { + let end = if w.commit { "COMMIT" } else { "ROLLBACK" }; + w.client.batch_execute(end).await.context(end)?; + } + // Write after settlement so replay can advance past it + source + .psql_one(&format!("INSERT INTO {t} VALUES (5001, 'tail')")) + .context("tail insert")?; + + fx::wait_for_ch_value( + &ch, + "SELECT count() FROM default.t FINAL WHERE id = 5001", + "1", + Duration::from_secs(60), + )?; + fx::wait_for_ch_value( + &ch, + "EXISTS TABLE default.t__wspending", + "0", + Duration::from_secs(60), + ) + .context("pending table still exists after all rows settled")?; + fx::wait_ch_matches_source(&ch, &source, &t, "default.t", Duration::from_secs(30)) + } + .await; + + fx::finish_daemon(guard, &daemon, result); +} diff --git a/tests/bootstrap_toast_reuse_ch.rs b/tests/bootstrap_toast_reuse_ch.rs new file mode 100644 index 00000000..c841b927 --- /dev/null +++ b/tests/bootstrap_toast_reuse_ch.rs @@ -0,0 +1,169 @@ +//! Reuse a TOAST value ID during backup. A hint-bit page image preserves old +//! chunks with versions before backup start, so backup rows read old values + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::io::Write; +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use walshadow::mapping::TableTarget; +use walshadow::schema::RelName; + +const SCHEMA: &str = "s28"; +/// Keep rows inline so new rows append near end of main heap +const FILLER_ROWS: i32 = 120_000; +const MAX_RATE_KIB: &str = "16384"; +const LIVE_ID: i32 = 200_000; +/// 32-byte MD5 digests per old value +const DEAD_DIGESTS: u32 = 94; + +/// Different chunk counts prevent reassembly; replay supplies replacement row +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn reissued_value_id_of_another_size_inside_backup_window_converges() { + reissued_value_id(160).await; +} + +/// Equal sizes return old value; replay must replace it +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn reissued_value_id_of_same_size_inside_backup_window_converges() { + reissued_value_id(DEAD_DIGESTS).await; +} + +async fn reissued_value_id(live_digests: u32) { + if !fx::tools::requirements_available() { + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + let source = fx::start_source(&tmp); + let _src_stop = fx::StopOnDrop { sh: &source }; + + let t = format!("{SCHEMA}.t"); + // EXTERNAL skips compression, so value length determines chunk count + source + .apply_schema_dump(&format!( + "CREATE SCHEMA {SCHEMA};\n\ + CREATE TABLE {t} (id int4 PRIMARY KEY, name text NOT NULL) \ + WITH (autovacuum_enabled = false, toast.autovacuum_enabled = false);\n\ + ALTER TABLE {t} ALTER COLUMN name SET STORAGE EXTERNAL;\n\ + ALTER TABLE {t} REPLICA IDENTITY FULL;\n\ + INSERT INTO {t} SELECT g, repeat('f', 1000) FROM generate_series(1, {FILLER_ROWS}) g;\n\ + INSERT INTO {t} SELECT 0, string_agg(md5(g::text || 'dead'), '') \ + FROM generate_series(1, {DEAD_DIGESTS}) g;\n" + )) + .expect("load workload"); + let toast = source + .psql_one(&format!( + "SELECT reltoastrelid::regclass::text FROM pg_class \ + WHERE oid = '{t}'::regclass" + )) + .expect("toast relation") + .trim() + .to_string(); + let dead_id = source + .psql_one(&format!("SELECT DISTINCT chunk_id FROM {toast}")) + .expect("dead value id") + .trim() + .to_string(); + source + .apply_schema_dump(&format!("DELETE FROM {t} WHERE id = 0;\nCHECKPOINT;\n")) + .expect("delete dead generation"); + + // Log hint-bit page images and reuse old value ID after vacuum removes chunks + let data_dir = source.config().data_dir.clone(); + writeln!( + std::fs::OpenOptions::new() + .append(true) + .open(data_dir.join("postgresql.conf")) + .unwrap(), + "wal_log_hints = on" + ) + .unwrap(); + source.stop().expect("stop source"); + let reset = Command::new("pg_resetwal") + .args(["-o", &dead_id, "-D"]) + .arg(&data_dir) + .status() + .expect("run pg_resetwal"); + assert!(reset.success(), "pg_resetwal -o {dead_id} failed"); + source.start().expect("restart source"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + fx::create_ch_dest_table(&ch, "default", "t").expect("create ch table"); + let ch_config_path = tmp.path().join("ch-config.toml"); + fx::write_ch_config_toml( + &ch_config_path, + "127.0.0.1", + slot.ch_tcp, + "default", + &RelName::new(SCHEMA, "t"), + &TableTarget::new("default", "t"), + ) + .expect("write ch-config"); + + let daemon = fx::DaemonRun::prepare(tmp.path(), slot.metrics).expect("daemon layout"); + let child = daemon + .spawn( + &source, + &ch_config_path, + slot.walsender, + &[ + "--bootstrap-max-rate-kib", + MAX_RATE_KIB, + "--slot", + "walshadow_toast_reuse", + ], + ) + .expect("spawn walshadow-stream"); + let guard = fx::ChildGuard::new(child); + + let result: Result<()> = async { + fx::wait_for_backup_streaming(&source, Duration::from_secs(60))?; + // First read after backup start logs page image containing old chunks + source + .psql_one(&format!("SELECT count(*) FROM {toast}")) + .context("hint dead chunks")?; + // Vacuum only TOAST heap so new rows append near end of main heap + source + .apply_schema_dump(&format!("VACUUM {toast};\n")) + .context("vacuum toast heap")?; + source + .psql_one(&format!( + "INSERT INTO {t} SELECT {LIVE_ID}, string_agg(md5(g::text || 'live'), '') \ + FROM generate_series(1, {live_digests}) g" + )) + .context("insert live generation")?; + let live_id = source + .psql_one(&format!("SELECT DISTINCT chunk_id FROM {toast}")) + .context("live value id")?; + ensure!( + live_id.trim() == dead_id, + "value id not reissued: dead {dead_id}, live {}", + live_id.trim() + ); + ensure!( + fx::backup_in_progress(&source), + "backup finished before new value was written" + ); + + fx::wait_for_listen(daemon.metrics_addr, Duration::from_secs(120)) + .context("daemon metrics endpoint never came up")?; + fx::wait_for_ch_value( + &ch, + &format!("SELECT count() FROM default.t FINAL WHERE id = {LIVE_ID}"), + "1", + Duration::from_secs(60), + )?; + fx::wait_ch_matches_source(&ch, &source, &t, "default.t", Duration::from_secs(30)) + } + .await; + + fx::finish_daemon(guard, &daemon, result); +}