From cfe6e4f6510b0166682cab78f54545bdef4a7be8 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Wed, 18 Nov 2026 06:38:50 +0000 Subject: [PATCH] Combine hashmaps TOAST resolution had unnecessary bookkeeping and repeated lookups, combine related state & use existing query order to avoid that --- src/emit/pipeline/bootstrap.rs | 47 ++++++++++++---------- src/toast/resolver.rs | 49 ++++++++++++----------- src/xact/xact_buffer.rs | 72 ++++++++++++++++++++++------------ tests/toast_resolvers.rs | 15 ++++--- 4 files changed, 107 insertions(+), 76 deletions(-) diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index f533ed71..7f4b7902 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -4,6 +4,7 @@ //! before resolving deferred values, which may read past `start_lsn`. Wait for //! synthetic sequences to become durable before advancing resume LSN to backup end +use std::collections::hash_map::Entry; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -794,21 +795,26 @@ fn unreadable(fetched: &[Option]) -> bool { /// A batch's values, with how many referrers each one still owes struct FetchedValues { - /// `None` without store: nothing to consult, every referrer fills - values: Option>, - uses: HashMap, + values: HashMap, +} + +#[derive(Default)] +struct FetchedEntry { + remaining: u32, + value: Option, } impl FetchedValues { /// Last referrer of a value takes its bytes, earlier ones copy fn take(&mut self, key: ValueKey) -> Option { - let left = self.uses.get_mut(&key)?; - *left -= 1; - let values = self.values.as_mut()?; - if *left == 0 { - values.remove(&key) + let Entry::Occupied(mut entry) = self.values.entry(key) else { + return None; + }; + entry.get_mut().remaining -= 1; + if entry.get().remaining == 0 { + entry.remove().value } else { - values.get(&key).cloned() + entry.get().value.clone() } } @@ -834,14 +840,14 @@ async fn fetch_values_at<'a>( rows: impl Iterator, resolver: &ToastResolver, ) -> Result { - let mut uses: HashMap = HashMap::new(); + let mut values: HashMap = HashMap::new(); let mut wanted: HashMap<(u32, u32, u64), Vec<(u32, usize)>> = HashMap::new(); for (row, bound) in rows { for site in &row.pointers { let key = value_key(row, site, bound); - let seen = uses.entry(key).or_default(); - *seen += 1; - if *seen == 1 { + let entry = values.entry(key).or_default(); + entry.remaining += 1; + if entry.remaining == 1 { wanted .entry((key.0, key.1, bound)) .or_default() @@ -849,23 +855,24 @@ async fn fetch_values_at<'a>( } } } - let mut values = HashMap::with_capacity(uses.len()); for ((db_oid, toast_relid, bound), batch) in wanted { let Some(got) = resolver .fetch_values(db_oid, toast_relid, &batch, bound) .await .map_err(|e| format!("bootstrap: toast store fetch: {e}"))? else { - return Ok(FetchedValues { values: None, uses }); + return Ok(FetchedValues { + values: HashMap::new(), + }); }; for ((value_id, _), value) in batch.iter().zip(got) { - values.insert((db_oid, toast_relid, *value_id, bound), value); + values + .get_mut(&(db_oid, toast_relid, *value_id, bound)) + .expect("counted in fetch_values_at") + .value = Some(value); } } - Ok(FetchedValues { - values: Some(values), - uses, - }) + Ok(FetchedValues { values }) } /// Turn one fetch outcome into the column its row routes, counting fills. diff --git a/src/toast/resolver.rs b/src/toast/resolver.rs index ec600003..6a6627bf 100644 --- a/src/toast/resolver.rs +++ b/src/toast/resolver.rs @@ -975,8 +975,8 @@ impl ClickHouseChunkStore { toast_relid: u32, ids: &[u32], max_lsn: u64, - expected: &HashMap, - ) -> Result, ChunkStoreError> { + expected: &HashMap, + ) -> Result, ChunkStoreError> { let sql = self.fetch_sql(db_oid, toast_relid, ids, max_lsn); let mut state = self.slot().await; self.query_locked(&mut state, &sql, |block, out| { @@ -1005,13 +1005,11 @@ fn is_missing_mirror(error: &EmitterError) -> bool { ) } -/// Feed one result block into per-value assemblers, sized off `expected` at -/// a value's first chunk. Seq order is validated per value across block -/// boundaries (final `ORDER BY chunk_id, chunk_seq` is the contract) +/// SQL orders chunks by value id and sequence across result blocks fn read_value_block( block: &Block, - expected: &HashMap, - out: &mut HashMap, + expected: &HashMap, + out: &mut Vec<(u32, ChunkAssembler)>, ) -> Result<(), EmitterError> { let n = block.n_rows(); if n == 0 { @@ -1039,8 +1037,18 @@ fn read_value_block( let seq = u32::from_le_bytes(seq_bytes[i * 4..i * 4 + 4].try_into().unwrap()); let start = if i == 0 { 0 } else { offsets[i - 1] as usize }; let end = offsets[i] as usize; - out.entry(id) - .or_insert_with(|| ChunkAssembler::new(expected.get(&id).copied().unwrap_or_default())) + if out.last().is_none_or(|(last, _)| *last != id) { + if out.last().is_some_and(|(last, _)| *last > id) { + return Err(EmitterError::Type(format!( + "toast fetch: chunk_id {id}: fetch order contract broken" + ))); + } + let size = expected.get(&id).map_or(0, |(_, size)| *size); + out.push((id, ChunkAssembler::new(size))); + } + out.last_mut() + .expect("assembler for current value") + .1 .push(seq, &data[start..end]) .map_err(|e| EmitterError::Type(format!("toast fetch: {e}")))?; } @@ -1075,9 +1083,9 @@ impl ChunkStore for ClickHouseChunkStore { if values.is_empty() { return Ok(Vec::new()); } - let mut expected: HashMap = HashMap::with_capacity(values.len()); - for &(id, size) in values { - let prev = expected.insert(id, size); + let mut expected = HashMap::with_capacity(values.len()); + for (index, &(id, size)) in values.iter().enumerate() { + let prev = expected.insert(id, (index, size)); debug_assert!( prev.is_none(), "batch repeats value {id} from {}", @@ -1087,7 +1095,7 @@ impl ChunkStore for ClickHouseChunkStore { let mut ids: Vec = expected.keys().copied().collect(); // Ascending: a split's ids stay adjacent in the mirror's granules ids.sort_unstable(); - let mut assembled: HashMap = HashMap::with_capacity(ids.len()); + let mut assembled = vec![FetchedValue::Missing; values.len()]; let queries = ids.len().div_ceil(FETCH_QUERY_IDS).max(1); let width = ids.len().div_ceil(queries); let concurrency = queries.min(self.states.len()); @@ -1097,16 +1105,13 @@ impl ChunkStore for ClickHouseChunkStore { .collect(); let mut batches = futures::stream::iter(split).buffer_unordered(concurrency); while let Some(batch) = batches.next().await { - assembled.extend(batch?); + for (id, chunks) in batch? { + if let Some(&(index, _)) = expected.get(&id) { + assembled[index] = chunks.finish(); + } + } } - Ok(values - .iter() - .map(|(id, _)| { - assembled - .remove(id) - .map_or(FetchedValue::Missing, ChunkAssembler::finish) - }) - .collect()) + Ok(assembled) } async fn truncate_mirror(&self, db_oid: u32, toast_relid: u32) -> Result<(), ChunkStoreError> { diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index 6f678087..d52e9786 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -2453,16 +2453,11 @@ pub async fn detoast_heap( // resolution peak (every retained decoded value + the largest // single-value transient), shrunk to retained bytes before return let mut leaf = crate::budget::acquire_opt(resolver.budget(), leaf_need).await; - let mut uses: HashMap<(u32, u32), u32> = HashMap::with_capacity(pointers.len()); - for p in &pointers { - *uses.entry((p.va_toastrelid, p.va_valueid)).or_default() += 1; - } let cache = prefetch_store_values(&pointers, chunk_maps, resolver, db, lsn).await?; let mut res = ValueResolution { spool, xact_maps: chunk_maps, resolver, - uses, cache, retained: 0, }; @@ -2526,7 +2521,7 @@ async fn prefetch_store_values( if resolver.fill_on_miss() { return Ok(cache); } - let mut wanted: HashMap> = HashMap::new(); + let mut wanted: HashMap> = HashMap::new(); for p in pointers { let key = (p.va_toastrelid, p.va_valueid); if let Some(v) = chunk_maps.iter().find_map(|m| m.get(&key)) { @@ -2541,28 +2536,37 @@ async fn prefetch_store_values( "toast value's leading chunks precede the stream start; resolving it from the store", ); } - wanted + let (pointer, uses) = wanted .entry(p.va_toastrelid) .or_default() - .insert(p.va_valueid, *p); + .entry(p.va_valueid) + .or_insert((*p, 0)); + *pointer = *p; + *uses += 1; } for (toast_relid, ptrs) in wanted { - let ptrs: Vec = ptrs.into_values().collect(); + let ptrs: Vec<_> = ptrs.into_values().collect(); let batch: Vec<(u32, usize)> = ptrs .iter() - .map(|p| (p.va_valueid, pointer_extsize(p))) + .map(|(p, _)| (p.va_valueid, pointer_extsize(p))) .collect(); let got = resolver .fetch_values(db_oid, toast_relid, &batch, bound) .await .map_err(|e| XactBufferError::Detoast(format!("toast store fetch: {e}")))? .expect("store checked via fill_on_miss"); - for (p, fetched) in ptrs.iter().zip(got) { + for ((p, uses), fetched) in ptrs.iter().zip(got) { let cached = match StoreMiss::split(Some(fetched)) { Ok(stored) => Ok(finish_value(p, stored)?), Err(miss) => Err(miss), }; - cache.insert((p.va_toastrelid, p.va_valueid), cached); + cache.insert( + (p.va_toastrelid, p.va_valueid), + CachedValue { + remaining: *uses, + value: cached, + }, + ); } } Ok(cache) @@ -2570,7 +2574,10 @@ async fn prefetch_store_values( /// Store-fetched value decoded once per key; cloned for all but the last /// use, which moves the buffer -type CachedValue = std::result::Result, StoreMiss>; +struct CachedValue { + remaining: u32, + value: std::result::Result, StoreMiss>, +} /// Who owns chunk store contents, deciding what an unresolved value means #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -2659,9 +2666,6 @@ struct ValueResolution<'a> { spool: Option<&'a BodySpoolFile>, xact_maps: &'a [&'a ChunkRefMap], resolver: &'a ToastResolver, - /// Pointer occurrences per key across both tuples, sizing cache - /// retention (last use moves instead of cloning) - uses: HashMap<(u32, u32), u32>, cache: HashMap<(u32, u32), CachedValue>, retained: usize, } @@ -2736,14 +2740,15 @@ impl ValueResolution<'_> { let cached = if self.resolver.fill_on_miss() { Err(StoreMiss::NoStore) } else { - let uses = self.uses.get_mut(&key).expect("counted in detoast_heap"); - *uses -= 1; - if *uses > 0 { - self.cache.get(&key).cloned() + let Entry::Occupied(mut entry) = self.cache.entry(key) else { + panic!("prefetched with the heap"); + }; + entry.get_mut().remaining -= 1; + if entry.get().remaining > 0 { + entry.get().value.clone() } else { - self.cache.remove(&key) + entry.remove().value } - .expect("prefetched with the heap") }; match cached { Ok(raw) => { @@ -4717,14 +4722,24 @@ mod tests { resolver: &'a ToastResolver, spool: Option<&'a BodySpoolFile>, xact_maps: &'a [&'a ChunkRefMap], - cache: HashMap<(u32, u32), CachedValue>, + cache: HashMap<(u32, u32), std::result::Result, StoreMiss>>, ) -> ValueResolution<'a> { ValueResolution { spool, xact_maps, resolver, - uses: HashMap::from_iter([((16500u32, 55u32), 1)]), - cache, + cache: cache + .into_iter() + .map(|(key, value)| { + ( + key, + CachedValue { + remaining: 1, + value, + }, + ) + }) + .collect(), retained: 0, } } @@ -5002,11 +5017,11 @@ mod tests { let cache = prefetch_store_values(&[p, p], &[], &resolver, 0, 10) .await .unwrap(); + let stored = cache[&(16500, 55)].value.as_ref().unwrap().as_ptr(); let mut r = ValueResolution { spool: None, xact_maps: &[], resolver: &resolver, - uses: HashMap::from_iter([((16500u32, 55u32), 2)]), cache, retained: 0, }; @@ -5015,6 +5030,11 @@ mod tests { let second = r.resolve_store(&p, 17).unwrap(); assert_eq!(first, ColumnValue::Bytea(b"abcd".to_vec())); assert_eq!(second, ColumnValue::Bytea(b"abcd".to_vec())); + let (ColumnValue::Bytea(first), ColumnValue::Bytea(second)) = (&first, &second) else { + unreachable!(); + }; + assert_ne!(first.as_ptr(), stored); + assert_eq!(second.as_ptr(), stored); assert!(r.cache.is_empty(), "last use moves the buffer out"); assert_eq!(r.retained, 8, "both uses retain their copy"); assert_eq!( diff --git a/tests/toast_resolvers.rs b/tests/toast_resolvers.rs index 2df126c1..cc4bf527 100644 --- a/tests/toast_resolvers.rs +++ b/tests/toast_resolvers.rs @@ -442,10 +442,9 @@ async fn ch_chunk_store_fetch_many_aligns_and_splits() { None, )); - // 200 single-chunk values: past one query's id width, so a fetch has to - // split across the pool and merge its splits back + // Exceed 1024 ids per query to exercise concurrent splits let body = |id: u32| format!("v{id}").into_bytes(); - let rows: Vec = (1..=200) + let rows: Vec = (1..=2050) .map(|id| { row( 16900, @@ -464,7 +463,7 @@ async fn ch_chunk_store_fetch_many_aligns_and_splits() { .expect("stop merges"); store.put(&[tomb(16900, (0, 6), 0x2000)]).await.unwrap(); store - .put(&[row(16900, 301, 0, (0, 6), 0x3000, b"reborn")]) + .put(&[row(16900, 3001, 0, (0, 6), 0x3000, b"reborn")]) .await .unwrap(); @@ -474,8 +473,8 @@ async fn ch_chunk_store_fetch_many_aligns_and_splits() { (200, 4), (1, 2), (5, 2), - (301, 6), - (999, 3), + (3001, 6), + (9999, 3), (42, 3), (7, 9), ]; @@ -498,7 +497,7 @@ async fn ch_chunk_store_fetch_many_aligns_and_splits() { // not yet born assert_eq!( store - .fetch_many(SRC_DB, 16900, &[(5, 2), (301, 6)], 0x1fff) + .fetch_many(SRC_DB, 16900, &[(5, 2), (3001, 6)], 0x1fff) .await .unwrap(), vec![assembled(b"v5"), FetchedValue::Missing], @@ -512,7 +511,7 @@ async fn ch_chunk_store_fetch_many_aligns_and_splits() { }) )); - let all: Vec<(u32, usize)> = (1..=200).map(|id| (id, body(id).len())).collect(); + let all: Vec<(u32, usize)> = (1..=2050).rev().map(|id| (id, body(id).len())).collect(); let got = store .fetch_many(SRC_DB, 16900, &all, u64::MAX) .await