Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 27 additions & 20 deletions src/emit/pipeline/bootstrap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
//! before resolving deferred values, which may read past `start_lsn`. Wait for
//! synthetic sequences to become durable before advancing resume LSN to backup end

use std::collections::hash_map::Entry;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};

Expand Down Expand Up @@ -794,21 +795,26 @@ fn unreadable(fetched: &[Option<FetchedValue>]) -> bool {

/// A batch's values, with how many referrers each one still owes
struct FetchedValues {
/// `None` without store: nothing to consult, every referrer fills
values: Option<HashMap<ValueKey, FetchedValue>>,
uses: HashMap<ValueKey, u32>,
values: HashMap<ValueKey, FetchedEntry>,
}

#[derive(Default)]
struct FetchedEntry {
remaining: u32,
value: Option<FetchedValue>,
}

impl FetchedValues {
/// Last referrer of a value takes its bytes, earlier ones copy
fn take(&mut self, key: ValueKey) -> Option<FetchedValue> {
let left = self.uses.get_mut(&key)?;
*left -= 1;
let values = self.values.as_mut()?;
if *left == 0 {
values.remove(&key)
let Entry::Occupied(mut entry) = self.values.entry(key) else {
return None;
};
entry.get_mut().remaining -= 1;
if entry.get().remaining == 0 {
entry.remove().value
} else {
values.get(&key).cloned()
entry.get().value.clone()
}
}

Expand All @@ -834,38 +840,39 @@ async fn fetch_values_at<'a>(
rows: impl Iterator<Item = (&'a ReplayRow, u64)>,
resolver: &ToastResolver,
) -> Result<FetchedValues, String> {
let mut uses: HashMap<ValueKey, u32> = HashMap::new();
let mut values: HashMap<ValueKey, FetchedEntry> = HashMap::new();
let mut wanted: HashMap<(u32, u32, u64), Vec<(u32, usize)>> = HashMap::new();
for (row, bound) in rows {
for site in &row.pointers {
let key = value_key(row, site, bound);
let seen = uses.entry(key).or_default();
*seen += 1;
if *seen == 1 {
let entry = values.entry(key).or_default();
entry.remaining += 1;
if entry.remaining == 1 {
wanted
.entry((key.0, key.1, bound))
.or_default()
.push((key.2, pointer_extsize(&site.p)));
}
}
}
let mut values = HashMap::with_capacity(uses.len());
for ((db_oid, toast_relid, bound), batch) in wanted {
let Some(got) = resolver
.fetch_values(db_oid, toast_relid, &batch, bound)
.await
.map_err(|e| format!("bootstrap: toast store fetch: {e}"))?
else {
return Ok(FetchedValues { values: None, uses });
return Ok(FetchedValues {
values: HashMap::new(),
});
};
for ((value_id, _), value) in batch.iter().zip(got) {
values.insert((db_oid, toast_relid, *value_id, bound), value);
values
.get_mut(&(db_oid, toast_relid, *value_id, bound))
.expect("counted in fetch_values_at")
.value = Some(value);
}
}
Ok(FetchedValues {
values: Some(values),
uses,
})
Ok(FetchedValues { values })
}

/// Turn one fetch outcome into the column its row routes, counting fills.
Expand Down
49 changes: 27 additions & 22 deletions src/toast/resolver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -975,8 +975,8 @@ impl ClickHouseChunkStore {
toast_relid: u32,
ids: &[u32],
max_lsn: u64,
expected: &HashMap<u32, usize>,
) -> Result<HashMap<u32, ChunkAssembler>, ChunkStoreError> {
expected: &HashMap<u32, (usize, usize)>,
) -> Result<Vec<(u32, ChunkAssembler)>, ChunkStoreError> {
let sql = self.fetch_sql(db_oid, toast_relid, ids, max_lsn);
let mut state = self.slot().await;
self.query_locked(&mut state, &sql, |block, out| {
Expand Down Expand Up @@ -1005,13 +1005,11 @@ fn is_missing_mirror(error: &EmitterError) -> bool {
)
}

/// Feed one result block into per-value assemblers, sized off `expected` at
/// a value's first chunk. Seq order is validated per value across block
/// boundaries (final `ORDER BY chunk_id, chunk_seq` is the contract)
/// SQL orders chunks by value id and sequence across result blocks
fn read_value_block(
block: &Block,
expected: &HashMap<u32, usize>,
out: &mut HashMap<u32, ChunkAssembler>,
expected: &HashMap<u32, (usize, usize)>,
out: &mut Vec<(u32, ChunkAssembler)>,
) -> Result<(), EmitterError> {
let n = block.n_rows();
if n == 0 {
Expand Down Expand Up @@ -1039,8 +1037,18 @@ fn read_value_block(
let seq = u32::from_le_bytes(seq_bytes[i * 4..i * 4 + 4].try_into().unwrap());
let start = if i == 0 { 0 } else { offsets[i - 1] as usize };
let end = offsets[i] as usize;
out.entry(id)
.or_insert_with(|| ChunkAssembler::new(expected.get(&id).copied().unwrap_or_default()))
if out.last().is_none_or(|(last, _)| *last != id) {
if out.last().is_some_and(|(last, _)| *last > id) {
return Err(EmitterError::Type(format!(
"toast fetch: chunk_id {id}: fetch order contract broken"
)));
}
let size = expected.get(&id).map_or(0, |(_, size)| *size);
out.push((id, ChunkAssembler::new(size)));
}
out.last_mut()
.expect("assembler for current value")
.1
.push(seq, &data[start..end])
.map_err(|e| EmitterError::Type(format!("toast fetch: {e}")))?;
}
Expand Down Expand Up @@ -1075,9 +1083,9 @@ impl ChunkStore for ClickHouseChunkStore {
if values.is_empty() {
return Ok(Vec::new());
}
let mut expected: HashMap<u32, usize> = HashMap::with_capacity(values.len());
for &(id, size) in values {
let prev = expected.insert(id, size);
let mut expected = HashMap::with_capacity(values.len());
for (index, &(id, size)) in values.iter().enumerate() {
let prev = expected.insert(id, (index, size));
debug_assert!(
prev.is_none(),
"batch repeats value {id} from {}",
Expand All @@ -1087,7 +1095,7 @@ impl ChunkStore for ClickHouseChunkStore {
let mut ids: Vec<u32> = expected.keys().copied().collect();
// Ascending: a split's ids stay adjacent in the mirror's granules
ids.sort_unstable();
let mut assembled: HashMap<u32, ChunkAssembler> = HashMap::with_capacity(ids.len());
let mut assembled = vec![FetchedValue::Missing; values.len()];
let queries = ids.len().div_ceil(FETCH_QUERY_IDS).max(1);
let width = ids.len().div_ceil(queries);
let concurrency = queries.min(self.states.len());
Expand All @@ -1097,16 +1105,13 @@ impl ChunkStore for ClickHouseChunkStore {
.collect();
let mut batches = futures::stream::iter(split).buffer_unordered(concurrency);
while let Some(batch) = batches.next().await {
assembled.extend(batch?);
for (id, chunks) in batch? {
if let Some(&(index, _)) = expected.get(&id) {
assembled[index] = chunks.finish();
}
}
}
Ok(values
.iter()
.map(|(id, _)| {
assembled
.remove(id)
.map_or(FetchedValue::Missing, ChunkAssembler::finish)
})
.collect())
Ok(assembled)
}

async fn truncate_mirror(&self, db_oid: u32, toast_relid: u32) -> Result<(), ChunkStoreError> {
Expand Down
72 changes: 46 additions & 26 deletions src/xact/xact_buffer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2453,16 +2453,11 @@ pub async fn detoast_heap(
// resolution peak (every retained decoded value + the largest
// single-value transient), shrunk to retained bytes before return
let mut leaf = crate::budget::acquire_opt(resolver.budget(), leaf_need).await;
let mut uses: HashMap<(u32, u32), u32> = HashMap::with_capacity(pointers.len());
for p in &pointers {
*uses.entry((p.va_toastrelid, p.va_valueid)).or_default() += 1;
}
let cache = prefetch_store_values(&pointers, chunk_maps, resolver, db, lsn).await?;
let mut res = ValueResolution {
spool,
xact_maps: chunk_maps,
resolver,
uses,
cache,
retained: 0,
};
Expand Down Expand Up @@ -2526,7 +2521,7 @@ async fn prefetch_store_values(
if resolver.fill_on_miss() {
return Ok(cache);
}
let mut wanted: HashMap<u32, HashMap<u32, ToastPointer>> = HashMap::new();
let mut wanted: HashMap<u32, HashMap<u32, (ToastPointer, u32)>> = HashMap::new();
for p in pointers {
let key = (p.va_toastrelid, p.va_valueid);
if let Some(v) = chunk_maps.iter().find_map(|m| m.get(&key)) {
Expand All @@ -2541,36 +2536,48 @@ async fn prefetch_store_values(
"toast value's leading chunks precede the stream start; resolving it from the store",
);
}
wanted
let (pointer, uses) = wanted
.entry(p.va_toastrelid)
.or_default()
.insert(p.va_valueid, *p);
.entry(p.va_valueid)
.or_insert((*p, 0));
*pointer = *p;
*uses += 1;
}
for (toast_relid, ptrs) in wanted {
let ptrs: Vec<ToastPointer> = ptrs.into_values().collect();
let ptrs: Vec<_> = ptrs.into_values().collect();
let batch: Vec<(u32, usize)> = ptrs
.iter()
.map(|p| (p.va_valueid, pointer_extsize(p)))
.map(|(p, _)| (p.va_valueid, pointer_extsize(p)))
.collect();
let got = resolver
.fetch_values(db_oid, toast_relid, &batch, bound)
.await
.map_err(|e| XactBufferError::Detoast(format!("toast store fetch: {e}")))?
.expect("store checked via fill_on_miss");
for (p, fetched) in ptrs.iter().zip(got) {
for ((p, uses), fetched) in ptrs.iter().zip(got) {
let cached = match StoreMiss::split(Some(fetched)) {
Ok(stored) => Ok(finish_value(p, stored)?),
Err(miss) => Err(miss),
};
cache.insert((p.va_toastrelid, p.va_valueid), cached);
cache.insert(
(p.va_toastrelid, p.va_valueid),
CachedValue {
remaining: *uses,
value: cached,
},
);
}
}
Ok(cache)
}

/// Store-fetched value decoded once per key; cloned for all but the last
/// use, which moves the buffer
type CachedValue = std::result::Result<Vec<u8>, StoreMiss>;
struct CachedValue {
remaining: u32,
value: std::result::Result<Vec<u8>, StoreMiss>,
}

/// Who owns chunk store contents, deciding what an unresolved value means
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
Expand Down Expand Up @@ -2659,9 +2666,6 @@ struct ValueResolution<'a> {
spool: Option<&'a BodySpoolFile>,
xact_maps: &'a [&'a ChunkRefMap],
resolver: &'a ToastResolver,
/// Pointer occurrences per key across both tuples, sizing cache
/// retention (last use moves instead of cloning)
uses: HashMap<(u32, u32), u32>,
cache: HashMap<(u32, u32), CachedValue>,
retained: usize,
}
Expand Down Expand Up @@ -2736,14 +2740,15 @@ impl ValueResolution<'_> {
let cached = if self.resolver.fill_on_miss() {
Err(StoreMiss::NoStore)
} else {
let uses = self.uses.get_mut(&key).expect("counted in detoast_heap");
*uses -= 1;
if *uses > 0 {
self.cache.get(&key).cloned()
let Entry::Occupied(mut entry) = self.cache.entry(key) else {
panic!("prefetched with the heap");
};
entry.get_mut().remaining -= 1;
if entry.get().remaining > 0 {
entry.get().value.clone()
} else {
self.cache.remove(&key)
entry.remove().value
}
.expect("prefetched with the heap")
};
match cached {
Ok(raw) => {
Expand Down Expand Up @@ -4717,14 +4722,24 @@ mod tests {
resolver: &'a ToastResolver,
spool: Option<&'a BodySpoolFile>,
xact_maps: &'a [&'a ChunkRefMap],
cache: HashMap<(u32, u32), CachedValue>,
cache: HashMap<(u32, u32), std::result::Result<Vec<u8>, StoreMiss>>,
) -> ValueResolution<'a> {
ValueResolution {
spool,
xact_maps,
resolver,
uses: HashMap::from_iter([((16500u32, 55u32), 1)]),
cache,
cache: cache
.into_iter()
.map(|(key, value)| {
(
key,
CachedValue {
remaining: 1,
value,
},
)
})
.collect(),
retained: 0,
}
}
Expand Down Expand Up @@ -5002,11 +5017,11 @@ mod tests {
let cache = prefetch_store_values(&[p, p], &[], &resolver, 0, 10)
.await
.unwrap();
let stored = cache[&(16500, 55)].value.as_ref().unwrap().as_ptr();
let mut r = ValueResolution {
spool: None,
xact_maps: &[],
resolver: &resolver,
uses: HashMap::from_iter([((16500u32, 55u32), 2)]),
cache,
retained: 0,
};
Expand All @@ -5015,6 +5030,11 @@ mod tests {
let second = r.resolve_store(&p, 17).unwrap();
assert_eq!(first, ColumnValue::Bytea(b"abcd".to_vec()));
assert_eq!(second, ColumnValue::Bytea(b"abcd".to_vec()));
let (ColumnValue::Bytea(first), ColumnValue::Bytea(second)) = (&first, &second) else {
unreachable!();
};
assert_ne!(first.as_ptr(), stored);
assert_eq!(second.as_ptr(), stored);
assert!(r.cache.is_empty(), "last use moves the buffer out");
assert_eq!(r.retained, 8, "both uses retain their copy");
assert_eq!(
Expand Down
Loading
Loading