Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions architecture/recovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ Restart may resend rows already inserted near checkpoint. Source row keys and
LSN versions make normal replay converge. Losing required WAL or durable history
is an error, not permission to skip ahead

Starting below descriptor log's seed would skip seeded relations' records
undecoded: refuse when a manifest exists, re-seed when none ever landed

## Separate retention needs

Decoder history and TOAST cleanup follow persisted restart floor. Filtered WAL
Expand Down
4 changes: 3 additions & 1 deletion docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ Do not reuse state against unrelated PostgreSQL cluster. Source system-ID
mismatch fails startup

`--ignore-cursor` and `--start-lsn` intentionally discard normal resume
position. Reserve them for recovery drills or operator-directed rebuilds
position. Reserve them for recovery drills or operator-directed rebuilds.
`--start-lsn` below descriptor history fails once a manifest exists; add
`--ignore-cursor` to re-seed history at that position

## Retain source WAL

Expand Down
115 changes: 104 additions & 11 deletions src/bin/stream/source_db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,12 +332,7 @@ pub(crate) async fn open_db_desc_log(
dir.display(),
);
if args.ignore_cursor {
for f in [
walshadow::desc_log::CKPT_FILE,
walshadow::desc_log::TAIL_FILE,
] {
let _ = tokio::fs::remove_file(dir.join(f)).await;
}
discard_desc_log(dir).await?;
}
if walshadow::desc_log::DescriptorLog::predates_partitions(dir).await?
&& let Some(d) = (input.catalog.lock().await.fetch_all_descriptors().await?.1)
Expand All @@ -351,14 +346,38 @@ pub(crate) async fn open_db_desc_log(
d.rel_name,
);
}
let desc_log = Arc::new(
walshadow::desc_log::DescriptorLog::open_on_branch(dir, input.identity, input.lineage)
.await
.context(format!("open descriptor log for database {}", input.dbname))?,
);
let open = || async {
walshadow::desc_log::DescriptorLog::open_on_branch(
dir,
input.identity.clone(),
input.lineage,
)
.await
.map(Arc::new)
.with_context(|| format!("open descriptor log for database {}", input.dbname))
};
let mut desc_log = open().await?;
if let Some(lsn) = input.start_lsn_override {
check_start_lsn(lsn, desc_log.floor_at_write(), desc_log.head())?;
}
if let Some(seeded) = reseed_below_start(
desc_log.seed_valid_from(),
input.aligned,
input.manifest_present,
)
.with_context(|| format!("descriptor log in {}", dir.display()))?
{
tracing::warn!(
target: "walshadow::desc_log",
dbname = %input.dbname,
start = %input.aligned,
seeded_from = format_pg_lsn(seeded).to_string(),
"descriptor log seeded above start with no manifest behind it; re-seeding",
);
drop(desc_log);
discard_desc_log(dir).await?;
desc_log = open().await?;
}
if desc_log.is_empty() {
// Baseline snapshot: every eligible rel as of shadow's position,
// valid from the aligned start so the prefix re-read decodes
Expand Down Expand Up @@ -427,6 +446,42 @@ fn check_start_lsn(lsn: Pos<RawStart>, floor: Pos<Floor>, head: u64) -> anyhow::
Ok(())
}

/// Below the seed every seeded relation's record reads as not covered and
/// skips undecoded. With a manifest behind the log that is lost history, so
/// refuse; without one no run behind it made progress durable, so `Some`
/// asks for a re-seed at start
fn reseed_below_start(
seeded: Option<u64>,
start: Pos<Floor>,
manifest_present: bool,
) -> anyhow::Result<Option<u64>> {
let Some(seeded) = seeded.filter(|&s| start.get() < s) else {
return Ok(None);
};
anyhow::ensure!(
!manifest_present,
"start {start} below descriptor log coverage from {}; records in between would \
skip undecoded — --ignore-cursor re-seeds the log at the start, or re-bootstrap",
format_pg_lsn(seeded),
);
Ok(Some(seeded))
}

async fn discard_desc_log(dir: &Path) -> anyhow::Result<()> {
for f in [
walshadow::desc_log::CKPT_FILE,
walshadow::desc_log::TAIL_FILE,
] {
let path = dir.join(f);
match tokio::fs::remove_file(&path).await {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e).with_context(|| format!("discard {}", path.display())),
}
}
Ok(())
}

pub(crate) const ATTACH_MARKER: &str = "attach-pending";

pub(crate) async fn list_databases(
Expand Down Expand Up @@ -479,4 +534,42 @@ mod tests {
"{beyond}"
);
}

const SEED: u64 = 0x5FA_6B00_0000;

#[test]
fn start_at_or_above_seed_keeps_the_log() {
for manifest in [false, true] {
assert_eq!(
reseed_below_start(Some(SEED), Pos::new(SEED), manifest).unwrap(),
None
);
assert_eq!(
reseed_below_start(Some(SEED), Pos::new(SEED + 1), manifest).unwrap(),
None
);
assert_eq!(
reseed_below_start(None, Pos::new(1), manifest).unwrap(),
None
);
}
}

/// A restart that never wrote a manifest seeded at the source head; the
/// operator's earlier `--start-lsn` must not decode against it
#[test]
fn start_below_seed_without_manifest_reseeds() {
assert_eq!(
reseed_below_start(Some(SEED), Pos::new(0x5AA_FF00_0000), false).unwrap(),
Some(SEED)
);
}

#[test]
fn start_below_seed_with_manifest_is_refused() {
let err = reseed_below_start(Some(SEED), Pos::new(0x5AA_FF00_0000), true).unwrap_err();
let msg = format!("{err:#}");
assert!(msg.contains("5FA/6B000000"), "{msg}");
assert!(msg.contains("--ignore-cursor"), "{msg}");
}
}
45 changes: 45 additions & 0 deletions src/catalog/desc_log.rs
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,18 @@ impl DescriptorLog {
self.index.read().unwrap().floor_at_write
}

/// Lowest LSN the boot seed describes. A seeded relation's record below
/// it reads as `NotCovered` and skips. `None` for an empty seed, or once
/// GC dropped every seeded entry and `floor_at_write` bounds history
pub fn seed_valid_from(&self) -> Option<u64> {
let idx = self.index.read().unwrap();
idx.batches
.range(..=idx.covered_through)
.flat_map(|(_, b)| &b.entries)
.map(|e| e.valid_from)
.min()
}

/// Highest `captured_at`, 0 when empty
pub fn head(&self) -> u64 {
self.index
Expand Down Expand Up @@ -1942,6 +1954,39 @@ mod tests {
assert!(log.batch_at(150).is_none());
}

#[tokio::test(flavor = "current_thread")]
async fn seed_valid_from_tracks_surviving_seed_entries() {
let tmp = tempfile::tempdir().unwrap();
let d1 = desc(101, 6001, false);
let d2 = desc(101, 6001, true);
let d3 = desc(102, 6002, false);
{
let log = open(tmp.path()).await;
assert_eq!(log.seed_valid_from(), None);
log.seed(batch(100, vec![present(90, &d1), present(90, &d3)]), 100)
.await
.unwrap();
log.append_batch(batch(200, vec![present(180, &d2)]))
.await
.unwrap();
assert_eq!(log.seed_valid_from(), Some(90));
assert_eq!(log.descriptor_at(rfn(6002), 89), LookupResult::NotCovered);
}
let log = open(tmp.path()).await;
assert_eq!(log.seed_valid_from(), Some(90));
// 6001's seed entry goes, 6002's is still the state at the floor
log.force_gc(Pos::new(250)).await.unwrap();
assert_eq!(log.seed_valid_from(), Some(90));
log.append_batch(batch(
300,
vec![tombstone(290, 102, 6002, LogValue::Dropped)],
))
.await
.unwrap();
log.force_gc(Pos::new(350)).await.unwrap();
assert_eq!(log.seed_valid_from(), None);
}

#[tokio::test(flavor = "current_thread")]
async fn rotation_retires_old_rfn_interval() {
let tmp = tempfile::tempdir().unwrap();
Expand Down
Loading