From d37efe5dbfa30d726e01cfa52a9f13d0cab99def Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:06:12 +0000 Subject: [PATCH 01/20] plan --- plans/INDEX.md | 1 + plans/native_encoding.md | 279 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 plans/native_encoding.md diff --git a/plans/INDEX.md b/plans/INDEX.md index 5b4c139b..2b8a2852 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -39,6 +39,7 @@ writes. A documented limitation does not imply code rejects it | [Multi-database loads and metrics](multi_database.md) | Extend heap-page bootstrap and backup loads beyond primary database, attribute metrics | | [Fuzzing](fuzzing.md) | Find parser and schema-transition interactions beyond fixed regressions | | [Performance](performance.md) | Establish healthy sustained WAL baseline with exact reconciliation before tuning | +| [Direct Native encoding](native_encoding.md) | Build on `more-types`: retain compact PG inputs, transcode into column chunks, compress sealed output | | [Failover](failover.md) | Fence unplanned promotion and prove remaining archive/restart fault cases | | [Shadow TOAST reclamation](shadow_toast.md) | Keep historical values readable under lag and restart | | [Replay callback](custom_rmgr.md) | Reduce measured command-boundary capture stalls | diff --git a/plans/native_encoding.md b/plans/native_encoding.md new file mode 100644 index 00000000..e34f954f --- /dev/null +++ b/plans/native_encoding.md @@ -0,0 +1,279 @@ +# Encode PostgreSQL values directly into ClickHouse blocks + +Build on `more-types` at `69f1700` (`Decode more PostgreSQL fixed-width types +locally`). Replace long-lived decoded row graphs with compact PG inputs and +direct column encoding. Retain transaction coordination, then compress sealed +output blocks while queued or retained for retry. + +Keep three changes independently reviewable: transaction storage, transcoding, +and compressed output ownership. Measure each against `more-types` before +combining results. Follow [shared constraints](coordination.md), +[value coercion](value_coercion.md), and [performance acceptance](performance.md). + +## Preserve more types behavior + +Reuse branch codecs, target mappings, defaults, and type coverage: + +| Source types | Existing branch behavior to preserve | +|---|---| +| `oid`, `xid`, `cid` | `UInt32` | +| `oid8`, `xid8`, `pg_lsn` | `UInt64`; retain `oid8` availability on PG 19+ | +| `money` | `Int64` containing source minor units; do not introduce decimal rescaling | +| `name` | Trim NUL padding and preserve existing text behavior | +| `macaddr`, `macaddr8`, `tid` | Render existing PostgreSQL-compatible text | + +Keep source type identity beside storage class. An eight-byte value alone +cannot distinguish timestamp epoch conversion, unsigned integers, and money. +Share rendering helpers from branch's `src/decode/codecs/fixed.rs` rather than +copying their logic into new encoders. Preserve COPY coverage without assuming +every newly local heap type also uses binary COPY. + +## Replace representation and retain coordination + +Current [heap decoder](../src/decode/heap_decoder.rs) constructs owned +`ColumnValue` cells. [Transaction buffer](../src/xact/xact_buffer.rs) retains +decoded heaps and TOAST entries until commit or abort. +[Planner](../src/emit/pipeline/planner.rs) resolves TOAST and routing, then +[plan spool](../src/emit/pipeline/plan_spool.rs) retains or serializes decoded +heaps. [Batcher](../src/emit/pipeline/batcher.rs) traverses rows again to fill +`ColumnBuf` slabs. [Resolver](../src/emit/pipeline/resolver.rs) converts remaining +oracle columns before [inserter](../src/emit/pipeline/inserter.rs) builds and +sends Native blocks. Sealed batches retain uncompressed slabs. + +Use this proposed flow: + +```text +WAL -> compact PG inputs owned by xid -> commit merge + -> resolve descriptor, route, TOAST, old and new key + -> append directly to column chunks in private transaction plan + -> validate and seal plan + -> coalesce compatible committed chunks across transactions + -> resolve oracle columns -> serialize and compress Native blocks + -> retain compressed blocks through INSERT acknowledgement +``` + +Encode after commit initially so subtransaction rollback never requires editing +compressed columns, and descriptor/TOAST resolution precedes final encoding. +Compression here reduces retained output, not open transaction inputs. Evaluate +compression of raw transaction segments separately if those inputs dominate RSS; +keep spill available regardless of compression ratio. Defer Native encoding +before commit until measurements justify its unresolved-cell and rollback state. + +Keep `XactBuffer` responsible for aborts, subtransactions, WAL ordering, spill, +and resume floor. Replace its heap payload, not its correctness responsibilities. +Keep `ColumnValue` for runtime configuration, SQL defaults, diagnostics, and +temporary adapters where useful. Remove it from ordinary retained row payloads; +deleting enum globally is not a completion requirement. + +## Store compact inputs before commit + +Introduce an owned heap input containing writer xid, source LSN, physical +identity, descriptor span key when covered, and bytes needed to reconstruct +logical old/new images. Preserve absent versus SQL NULL, dropped attributes, +missing defaults, operation flags, and partial-image diagnostics. + +Start from existing `RawRecord` ownership and reconstruction in +[spill codec](../src/xact/spill.rs). Reuse `drop_redundant_images`; retain images +when decoding actually requires them. Do not retain a complete WAL segment or +backup page through a small borrowed tuple slice. Compact further only after +measuring retained bytes against decoded rows, including full-page images and +multi-insert records. + +Give covered raw heaps a distinct representation from unresolved catalog stash +records. Existing `SpillEntry::Raw` carries commit-time stash classification; +routing every ordinary heap through that arm would change discard and rejection +semantics. Pin covered descriptor versions exactly as `DescribedHeap` does; +resolve uncovered records through existing fail-closed stash verdicts. + +Keep TOAST chunk and delete handling intact during initial migration. Preserve +chunk generation fences, store references, same-transaction births, and mirror +row ordering. Decode small runtime configuration rows through existing adapter +until a separate reader is useful. + +Replace speculative owned `Predecoded` row graphs in +[queue](../src/source/queueing_record_sink.rs) and `ReorderSink::predecoder` +with bounded framing or borrowed scans. Preserve TOAST read-ahead for unchanged +external values without retaining decoded copies. Measure lost decode overlap +and read-ahead effectiveness before removing existing path. + +## Transcode during transaction planning + +Split tuple walking from owned cell construction. Expose short-lived views over +fixed bytes, varlena bodies, SQL NULL, missing attributes, and external pointers. +Borrow from current input or bounded detoast scratch; never let views outlive +their owners. Retain a cell-producing adapter for remaining callers. + +Compile conversion decisions once per descriptor version and frozen route, +using source type and destination type together. Keep mapping changes and +schema epochs in cache identity. Reuse existing codecs for epoch shifts, UUID +layout, numeric rendering, JSONB, interval, inet, and branch's added types. +Append formatted output directly where helpers allow it; accept bounded scratch +when a codec needs materialization. + +Coordinate [local container codecs](tier2.md) around recursive column builders. +Do not require a retained nested `ColumnValue` graph when array or map support +can append directly; preserve oracle fallback for uncovered source/target pairs. + +| Input | Direct encoding rule | +|---|---| +| Compatible fixed heap datum | Copy validated little-endian bytes into matching column storage | +| Binary COPY field | Decode big-endian `typsend` representation; never reuse heap byte-copy rule | +| Inline text or bytea | Strip storage framing and append body after existing validation | +| Compressed or external varlena | Resolve and decompress before encoding; bound scratch and preserve size policy | +| Missing default | Apply shared default conversion with source descriptor and target policy | +| Oracle type or configured composite target | Retain compact raw/text conversion input; preserve oracle fallback | + +Do not treat equal width as type compatibility. Preserve configured substitutes +and existing errors; compose outstanding coercion fixes without introducing a +second policy implementation. Keep absence defaults for tombstones distinct +from real source NULL. Preserve NaN payloads and signed zero where existing +fixed-width mappings permit bit-preserving output. + +Inspect old and new replica identity before releasing either view. Emit old-key +tombstones for key changes, choose correct image for DELETE, and keep configured +append-only delete policy. Preserve source `_lsn` and writer `_xid`; attach commit +timestamp after commit is known. Keep load-specific synthetic versions unchanged. + +Store encoded chunks and oracle inputs in private transaction plan. Seal chunks +at row/byte limits, schema or route changes, and ordered control entries. Preserve +TOAST mirror row positions and truncate fences. Keep all existing planning +rejections before transaction effects; do not send partially planned chunks. +Oracle transport and conversion remain explicit later failure points. + +Replace heap frames in plan spool with versioned encoded frames once adapter +parity passes. Retain plan memory/disk caps and bounded replay. Reuse chunk +metadata across rows instead of copying descriptor and routing ownership per +cell. Treat plans and transaction spills as scratch recovered from source WAL, +not as a new durable destination queue. + +## Preserve batching across transactions + +Accept column chunks in batcher instead of `RoutedRow` graphs. Coalesce adjacent +compatible small chunks by appending fixed bytes and rebasing string offsets, +null maps, and oracle cell ranges. Preserve `(seq, row_count)` contributions. +Keep compatibility keyed by destination, schema epoch, route, and encoding plan. + +Separate Native block size from INSERT size. Seal bounded blocks, then group +compatible blocks from multiple transactions into one INSERT under existing row, +byte, and deadline triggers. Do not compress tiny per-transaction blocks and +assume concatenating their byte streams produces one merged Native block. +Send several Data blocks under one query when grouping sealed blocks. + +Preserve FIFO `FlushAll`, destructive-operation drains, and configuration +barriers. Never move chunks across DDL or TRUNCATE to improve compression. +Measure actual destination parts: several blocks in one INSERT do not establish +unchanged part creation by themselves. + +## Compress sealed output blocks + +Resolve oracle columns before final serialization. Serialize final Native block +body and compress once; release column slabs and oracle output after compressed +owner is admitted. Keep schema/route metadata and acknowledgement counts beside +payload. Retain original logical row/byte counts for batching independently of +compressed resident size. Start with existing configured codecs and measure +incompressible expansion as well as useful compression. + +Add a narrow prepare/send interface in `clickhouse-c` and `clickhouse-c-rs`. +Current client accepts `BlockBuilder` through `send_data`; it does not expose an +owned prepared payload send API. Keep serialization, compression framing, packet +state, and transport handling in dependency rather than duplicating protocol in +walshadow. Read existing [client](../pgext/pg-clickhouse-c/clickhouse-c/doc/clickhouse-client.md) +and [compression](../pgext/pg-clickhouse-c/clickhouse-c/doc/clickhouse-compression.md) +contracts before choosing interface. + +Prepare immutable block payload with explicit codec and protocol compatibility +requirements; generate connection-specific packet framing at send. Validate +negotiated revision and serialization settings on reconnect and across named +destinations. For incompatible connections, reject reuse or rebuild through a +bounded decode/re-encode path. Do not replay captured Query packets, session +state, or unchecked raw socket bytes. + +Stream prepared payload through bounded transport buffers. Account for any C +output-buffer copy and retained capacity; adding compressed owner while keeping +slabs or an unbounded transport copy defeats resident-memory goal. If prepared +send cannot land first, allow storage compression with one bounded decompression +buffer per inserter as an intermediate step, and measure its CPU cost separately. + +Retain every block in an INSERT until `EndOfStream` from every required +destination. Retry entire INSERT after ambiguous failure using unchanged rows, +versions, block order, and metadata. Acknowledge its combined sequence counts +only after success. Keep counts logical, independent of compression frames. +Compression, cancellation, and transport errors must neither drop ownership nor +advance contiguous acknowledgement. + +## Bound resident memory and preserve progress + +Track allocated capacity for raw inputs, open column builders, oracle inputs and +responses, sealed compressed blocks, compression scratch, and transport buffers. +Keep decoded/encoded sizes distinct from compressed allocation and process RSS. +Bound queue bytes as well as item counts; include spare slab capacity and +per-chunk metadata when measuring small rows and many tables. + +Preserve separate transaction spill and output admission mechanisms. If retaining +an open transaction would exhaust resident allowance, spill and continue reading +WAL; never wait for its own COMMIT while blocking ingestion. Reserve bounded +conversion/compression workspace separately from queued output so memory held by +unfinished work cannot prevent its completion. Specify acquisition and release +order before changing pools, including incompressible output and oversized rows. + +Keep startup scratch cleanup and source-WAL recovery. Version changed spill and +plan formats; never interpret old frames under new layout. Retain first-record +resume floor until corresponding transaction output is durable. Test restart +from recorded source positions rather than treating a compressed buffer as proof +of persistence. + +## Implementation sequence + +1. Record baseline on `more-types`. Extend existing `xact_spill`, `spool_read`, + and `bootstrap_pump` benchmarks with retained allocation measurements and + exact result comparison. Establish source-to-destination baseline using + existing benchmark driver. +2. Add borrowed tuple views and direct column append behind internal adapters. + Keep production ownership unchanged while comparing against existing encoder. + Cover branch's added types and configured target overrides first. +3. Replace ordinary transaction heap storage with owned PG inputs. Reuse spill, + keep covered descriptors separate from unresolved stash, and remove retained + predecoded copies. Preserve temporary cell adapter at commit during migration. +4. Fuse commit planning and encoding. Store column chunks in private plan, + replace typed heap frames, and accept chunks in batcher. Delete superseded + row materialization paths after parity; avoid a permanent second pipeline. +5. Introduce prepared payload support in dependencies, then compressed block + ownership and multi-block INSERTs. Validate retries and memory handoff before + enabling resident compression by default. +6. Apply shared direct append to COPY and heap-page/bootstrap loads. Keep + visibility, pending rows, deferred TOAST, replacement evidence, and staging + publication unchanged. Stop retaining enum rows in ordinary load queues. +7. Reconcile end-to-end results and publish per-stage resource comparisons. + Update settled architecture and operator documentation; remove completed + proposal sections and retain unresolved acceptance work. + +## Acceptance + +Extend existing behavioral suites rather than creating a parallel harness: + +| Concern | Required evidence | +|---|---| +| Type fidelity | Compare old/direct output across `types_sweep`, `emitter_native_types`, oracle and bootstrap type suites; include branch types, defaults, NULL, empty values, signed boundaries, float bits, timestamp epochs, UUIDs, numeric, and JSONB | +| Source representation | Compare WAL/heap and binary COPY outputs; exercise inline, compressed, external, and unchanged TOAST values | +| Transaction outcomes | Extend `xact_buffer`, `subxact`, and bootstrap crossing-transaction cases with interleaved parent/subxid rows, savepoint rollback, abort after spill, and late subxid assignment | +| Ordering | Extend grouping/spill, schema evolution, parallel DDL, runtime config, and truncate tests with chunk boundaries on both sides of controls | +| Planning failures | Force missing descriptor, malformed input, missing TOAST, and spool-cap failure after earlier chunks have sealed; observe no transaction effects from rejected plan | +| Oracle coexistence | Mix direct and oracle columns, oracle retries, local literal resolution, and conversion errors without rebuilding retained row graphs | +| Compressed output | Decode prepared blocks through existing Native reader; verify none/LZ4/ZSTD, incompressible data, large values, nullable strings, frame boundaries, and multiple blocks per INSERT | +| Retry and cancellation | Disconnect after an intermediate Data block and after server acceptance before reply; cancel compression, queue handoff, and send; verify replay, ownership cleanup, and pinned acknowledgement | +| Recovery | Restart with open raw transactions, spilled plans, and ambiguous INSERT outcomes; verify exact destination state and resume floor across required destinations | +| Memory progress | Exhaust transaction/output budgets with many tables, a large transaction, and stalled inserters; prove spill reaches COMMIT and compression can finish without circular waits | +| Load coverage | Exercise COPY, direct bootstrap, base backup, object store, pending transactions, deferred TOAST, and restart using existing load fixtures | + +Measure narrow fixed rows, short strings, wide JSONB/text, TOAST-heavy updates, +mixed oracle tables, many small transactions, and a spilling large transaction. +Run with unchanged worker counts, batching limits, and codec settings. Record +rows/s, source WAL bytes/s, CPU, allocation count, peak/steady RSS, retained bytes +by owner, spill volume, compression ratio, durable lag, blocks/INSERT, and actual +ClickHouse part creation. Compare compressible and incompressible payloads with +both healthy and stalled destination. + +Require exact reconciliation and no new progress stalls before accepting resource +gains. Attribute savings separately to removal of enum rows, compact transaction +storage, and compression. Choose block size and default compression from measured +CPU/memory/part tradeoffs; do not claim gains from wire compression alone. From 861dfefca5aeebe77cf70e6578608bd14c7eb72f Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:12:53 +0000 Subject: [PATCH 02/20] Meter allocations in buffer, plan, and bootstrap benches Wrap mimalloc in a counting global allocator so xact_spill, spool_read and bootstrap_pump report allocation count, retained bytes and peak live bytes per stage. Plan and deferred replay compare every row's values; bootstrap pump compares a digest of decoded values against written pages and drain asserts routed row count. Baseline (more-types, defaults unless noted): xact_spill --buffer-bytes 1048576: 950k rows/s, buffer peak 1.62 MB, 23.6k allocs per 7812-row xact, drain 21.1k allocs spool_read --plan-only --planner --mapped --relations 8 --control-every 10000 --rows 200000: plan 400k allocs, replay+place 400k allocs, replay peak 2.13 MB bootstrap_pump: pump 18.5M tuples/s, 822k allocs, peak 15.8 MiB; drain 14.2M tuples/s, 829k allocs, peak 3.5 MiB Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 4 ++ benches/bootstrap_pump.rs | 66 +++++++++++++------ benches/common/alloc_meter.rs | 119 ++++++++++++++++++++++++++++++++++ benches/spool_read.rs | 29 ++++++++- benches/xact_spill.rs | 22 ++++++- 5 files changed, 216 insertions(+), 24 deletions(-) create mode 100644 benches/common/alloc_meter.rs diff --git a/bench/README.md b/bench/README.md index c6d94f45..b1890474 100644 --- a/bench/README.md +++ b/bench/README.md @@ -16,6 +16,8 @@ xid, LSN order, descriptor, and file cleanup. Write timing includes allocation and admission; read timing includes reader setup, verified drain, and unlink. Report spill bytes alongside throughput: final memory tails can avoid disk. Compare equal spill bytes when isolating I/O improvements. +`buffer` and `drain` report allocation count, bytes still live, and peak live +bytes for largest transaction, counted by a wrapping global allocator Files follow disposable-spill semantics: no fsync, immediate replay, then unlink. Results include page cache and codec costs; they do not measure sustained device @@ -45,6 +47,8 @@ placement with `--planner`. For plans in memory, byte counts estimate memory use for spilled plans, they report encoded file size. Route caching measures descriptor identity checks and reuse of cached references. Measurements exclude mapping resolution, WAL draining, TOAST I/O, control execution, and PostgreSQL/ClickHouse I/O. +`plan` reports allocations and bytes retained by sealed plan; `replay` reports +peak bytes while replaying it. Replay compares every row's values exactly ## pgbench workload diff --git a/benches/bootstrap_pump.rs b/benches/bootstrap_pump.rs index bed8ae44..8a185320 100644 --- a/benches/bootstrap_pump.rs +++ b/benches/bootstrap_pump.rs @@ -27,7 +27,11 @@ // serializes on that pattern. A bench on a different allocator measures a // different program #[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +static GLOBAL: alloc_meter::Metered = + alloc_meter::Metered::new(mimalloc::MiMalloc); + +#[path = "common/alloc_meter.rs"] +mod alloc_meter; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -150,11 +154,12 @@ fn mapping(filenode: u32, columns: usize) -> TableMapping { /// One 8 KiB heap page carrying `tuples` live int4-only tuples, laid out /// as PG writes them: line pointers up from the header, tuple bodies down -/// from the page end. -fn heap_page(tuples: usize, columns: usize, seed: i32) -> Vec { +/// from the page end. Returns page and wrapping sum of its values +fn heap_page(tuples: usize, columns: usize, seed: i32) -> (Vec, i64) { let body = TUPLE_HEADER + 4 * columns; let mut page = vec![0u8; PAGE_BYTES]; let mut written = 0usize; + let mut digest = 0i64; for i in 0..tuples { let off = PAGE_BYTES - (i + 1) * body; if off < SIZE_OF_PAGE_HEADER + (i + 1) * SIZE_OF_ITEM_ID { @@ -169,6 +174,7 @@ fn heap_page(tuples: usize, columns: usize, seed: i32) -> Vec { let at = off + TUPLE_HEADER + c * 4; let v = seed + (i as i32) * 31 + c as i32; page[at..at + 4].copy_from_slice(&v.to_le_bytes()); + digest = digest.wrapping_add(i64::from(v)); } let slot = SIZE_OF_PAGE_HEADER + i * SIZE_OF_ITEM_ID; let raw = ((off as u32) & 0x7FFF) | (LP_NORMAL << 15) | (((body as u32) & 0x7FFF) << 17); @@ -179,27 +185,27 @@ fn heap_page(tuples: usize, columns: usize, seed: i32) -> Vec { let pd_upper = PAGE_BYTES - written * body; page[12..14].copy_from_slice(&(pd_lower as u16).to_le_bytes()); page[14..16].copy_from_slice(&(pd_upper as u16).to_le_bytes()); - page + (page, digest) } -/// One tar part holding `segments_per_part` `base//` entries. -async fn build_part(shape: Shape, part: usize) -> Vec { +/// One tar part holding `segments_per_part` `base//` entries, +/// with wrapping sum of values written +async fn build_part(shape: Shape, part: usize) -> (Vec, i64) { let mut writer = Writer::new(Vec::new()); + let mut digest = 0i64; for s in 0..shape.segments_per_part { let filenode = FIRST_FILENODE + (part * shape.segments_per_part + s) as u32; let mut body = Vec::with_capacity(shape.pages_per_segment * PAGE_BYTES); for p in 0..shape.pages_per_segment { - body.extend_from_slice(&heap_page( - shape.tuples_per_page, - shape.columns, - (p * 1_000) as i32, - )); + let (page, d) = heap_page(shape.tuples_per_page, shape.columns, (p * 1_000) as i32); + body.extend_from_slice(&page); + digest = digest.wrapping_add(d); } let path = format!("base/{DB_NODE}/{filenode}"); let entry = Entry::new(path, Kind::Regular, body.len() as u64, 0o600, 0); writer.append(&entry, body.as_slice()).await.unwrap(); } - writer.finish().await.unwrap() + (writer.finish().await.unwrap(), digest) } /// Object-store-shaped source over pre-built tars: one `pump_tar_to_sink` @@ -275,8 +281,11 @@ impl Report { async fn bench_pump(shape: Shape) -> Report { let mut parts = Vec::with_capacity(shape.parts); + let mut expected = 0i64; for p in 0..shape.parts { - parts.push(build_part(shape, p).await); + let (part, digest) = build_part(shape, p).await; + parts.push(part); + expected = expected.wrapping_add(digest); } let mut catalog = CatalogMap::new(); @@ -292,19 +301,29 @@ async fn bench_pump(shape: Shape) -> Report { parallelism: shape.parallelism, }); + let base = GLOBAL.reset_peak(); let started = Instant::now(); let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog, false); let drained = tokio::spawn(async move { - let mut n = 0u64; + let (mut n, mut digest) = (0u64, 0i64); while let Some(slab) = rx.recv().await { - // Touch the payload so the decode can't be optimised away - n += slab.iter().map(|t| t.columns.len() as u64).sum::(); + for t in &slab { + n += t.columns.len() as u64; + for c in &t.columns { + let Some(ColumnValue::Int4(v)) = c else { + panic!("int4 column decoded as {c:?}"); + }; + digest = digest.wrapping_add(i64::from(*v)); + } + } } - n + (n, digest) }); pump.await.unwrap().unwrap(); - let touched = drained.await.unwrap(); + let (touched, digest) = drained.await.unwrap(); let elapsed = started.elapsed().as_secs_f64(); + let metered = GLOBAL.snapshot().since(base); + assert_eq!(digest, expected, "decoded values differ from written pages"); let ld = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64; Report { @@ -320,6 +339,8 @@ async fn bench_pump(shape: Shape) -> Report { "channel_block_secs", ld(&progress.page_walk.channel_block_nanos) / 1e9, ), + ("allocs", metered.allocs as f64), + ("peak_mib", metered.peak as f64 / (1 << 20) as f64), ], } } @@ -372,6 +393,7 @@ async fn bench_drain(shape: Shape) -> Report { }); let stats = Arc::new(walshadow::ch_emitter::EmitterStats::default()); + let base = GLOBAL.reset_peak(); let started = Instant::now(); let outcome = bootstrap::drain( tup_rx, @@ -390,6 +412,8 @@ async fn bench_drain(shape: Shape) -> Report { let elapsed = started.elapsed().as_secs_f64(); feeder.await.unwrap(); tail.close(msg_tx, ack).await; + let metered = GLOBAL.snapshot().since(base); + assert_eq!(outcome.rows_routed, shape.tuples()); Report { label: "drain".into(), @@ -397,7 +421,11 @@ async fn bench_drain(shape: Shape) -> Report { tuples: outcome.rows_routed, // Int4 payload only; the interesting rate here is rows, not bytes bytes: outcome.rows_routed * 4 * shape.columns as u64, - detail: vec![("seqs", outcome.next_seq as f64)], + detail: vec![ + ("seqs", outcome.next_seq as f64), + ("allocs", metered.allocs as f64), + ("peak_mib", metered.peak as f64 / (1 << 20) as f64), + ], } } diff --git a/benches/common/alloc_meter.rs b/benches/common/alloc_meter.rs new file mode 100644 index 00000000..4b89062b --- /dev/null +++ b/benches/common/alloc_meter.rs @@ -0,0 +1,119 @@ +//! Global allocator wrapper counting allocations and live bytes, so benches +//! report retained memory by stage instead of payload estimates + +use std::alloc::{GlobalAlloc, Layout}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +pub struct Metered { + inner: A, + allocs: AtomicU64, + live: AtomicUsize, + peak: AtomicUsize, +} + +impl Metered { + pub const fn new(inner: A) -> Self { + Self { + inner, + allocs: AtomicU64::new(0), + live: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + } + } + + fn grow(&self, n: usize) { + let live = self.live.fetch_add(n, Ordering::Relaxed) + n; + self.peak.fetch_max(live, Ordering::Relaxed); + } + + fn shrink(&self, n: usize) { + self.live.fetch_sub(n, Ordering::Relaxed); + } + + pub fn snapshot(&self) -> Meter { + Meter { + allocs: self.allocs.load(Ordering::Relaxed), + live: self.live.load(Ordering::Relaxed), + peak: self.peak.load(Ordering::Relaxed), + } + } + + /// Restart peak tracking from current live bytes + pub fn reset_peak(&self) -> Meter { + self.peak + .store(self.live.load(Ordering::Relaxed), Ordering::Relaxed); + self.snapshot() + } +} + +/// Requested sizes, not allocator size classes or RSS +#[derive(Clone, Copy, Debug)] +pub struct Meter { + pub allocs: u64, + pub live: usize, + pub peak: usize, +} + +impl Meter { + /// Allocations, bytes still live, and peak growth since `base` + pub fn since(self, base: Meter) -> Delta { + Delta { + allocs: self.allocs - base.allocs, + retained: self.live as i64 - base.live as i64, + peak: self.peak.saturating_sub(base.live), + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Delta { + pub allocs: u64, + pub retained: i64, + pub peak: usize, +} + +impl std::fmt::Display for Delta { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "allocs={} retained_bytes={} peak_bytes={}", + self.allocs, self.retained, self.peak + ) + } +} + +// SAFETY: forwards to `inner`; counters never touch the returned memory +unsafe impl GlobalAlloc for Metered { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { self.inner.alloc(layout) }; + if !p.is_null() { + self.allocs.fetch_add(1, Ordering::Relaxed); + self.grow(layout.size()); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { self.inner.alloc_zeroed(layout) }; + if !p.is_null() { + self.allocs.fetch_add(1, Ordering::Relaxed); + self.grow(layout.size()); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { self.inner.dealloc(ptr, layout) }; + self.shrink(layout.size()); + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let p = unsafe { self.inner.realloc(ptr, layout, new_size) }; + if !p.is_null() { + self.allocs.fetch_add(1, Ordering::Relaxed); + self.shrink(layout.size()); + self.grow(new_size); + } + p + } +} diff --git a/benches/spool_read.rs b/benches/spool_read.rs index 111e802d..a00c59ec 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -24,8 +24,12 @@ use walshadow::spool::DeferredSpool; use walshadow::toast::ToastResolver; use walshadow::xact_buffer::{DrainEntry, DrainedBatch, OrderedEvent}; +#[path = "common/alloc_meter.rs"] +mod alloc_meter; + #[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +static GLOBAL: alloc_meter::Metered = + alloc_meter::Metered::new(mimalloc::MiMalloc); #[derive(Debug, Parser)] struct Args { @@ -91,6 +95,7 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let dir = tempfile::tempdir_in(&args.dir)?; let payload = "x".repeat(args.payload_bytes); let mut spool = DeferredSpool::new(dir.path().join("deferred.spool"), 1 << 20); + let base = GLOBAL.reset_peak(); let start = Instant::now(); for i in 0..args.rows { spool @@ -113,17 +118,25 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let bytes = spool.spooled_bytes(); let mut reader = spool.into_reader().await?; let write = start.elapsed(); + let spooled = GLOBAL.snapshot().since(base); + let base = GLOBAL.reset_peak(); let start = Instant::now(); let mut n = 0; while let Some(t) = reader.next().await? { assert_eq!(t.xid, n as u32); - assert!(matches!(&t.columns[1], Some(ColumnValue::Text(s)) if s.len() == payload.len())); + assert!(matches!( + t.columns.as_slice(), + [Some(ColumnValue::Int4(v)), Some(ColumnValue::Text(s))] + if *v == n as i32 && *s == payload + )); n += 1; } reader.finish().await?; let read = start.elapsed(); + let replayed = GLOBAL.snapshot().since(base); assert_eq!(n, args.rows); report("deferred", repeat, n, bytes, write, read); + println!("deferred_alloc repeat={repeat} spool {spooled} replay {replayed}"); Ok(()) } @@ -226,6 +239,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { calls: 0, }; let resolver = ToastResolver::disabled(); + let base = GLOBAL.reset_peak(); let start = Instant::now(); let sealed = if args.planner { let mut planner = Planner::create( @@ -274,12 +288,14 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { writer.seal(u64::MAX, 0)? }; let write = start.elapsed(); + let planned = GLOBAL.snapshot().since(base); let (size_bytes, spilled) = (sealed.size_bytes, sealed.path().is_some()); let stats = EmitterStats::default(); let mut staged = Vec::new(); let mut pending = Vec::with_capacity(4096); let mut routed = 0; let mut controls = 0; + let base = GLOBAL.reset_peak(); let start = Instant::now(); let mut rd = sealed.replay()?; let mut n = 0; @@ -290,6 +306,12 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { assert_eq!(h.decoded.source_lsn, n as u64); assert_eq!(h.decoded.rfn, descriptors[n % descriptors.len()].rfn); assert_eq!(h.decoded.xid, 7 + (n % 4) as u32); + let columns = &h.decoded.new.as_ref().expect("insert image").columns; + assert!(matches!( + columns.as_slice(), + [Some(ColumnValue::Int4(v)), Some(ColumnValue::Text(s))] + if *v == n as i32 && *s == payload + )); n += 1; if args.planner { pending.push(h); @@ -332,6 +354,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { assert_eq!(routed, if args.mapped { args.rows as u64 } else { 0 }); } let replay = start.elapsed(); + let replayed = GLOBAL.snapshot().since(base); assert_eq!(n, if args.mapped { args.rows } else { 0 }); assert_eq!( controls, @@ -361,7 +384,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { + (args.rows % interval).min(descriptors.len()); assert_eq!(view.calls, expected_calls); println!( - "plan_total repeat={repeat} total_s={:.6} route_calls={} tables={} routed={routed} spilled={}", + "plan_total repeat={repeat} total_s={:.6} route_calls={} tables={} routed={routed} spilled={} plan {planned} replay {replayed}", (write + replay).as_secs_f64(), view.calls, rd.tables.len(), diff --git a/benches/xact_spill.rs b/benches/xact_spill.rs index 1deba7f4..3d1568c9 100644 --- a/benches/xact_spill.rs +++ b/benches/xact_spill.rs @@ -11,8 +11,12 @@ use walshadow::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, DescribedH use walshadow::schema::{RelDescriptor, RelName, ReplIdent}; use walshadow::xact_buffer::{StashResolved, XactBuffer, XactBufferConfig}; +#[path = "common/alloc_meter.rs"] +mod alloc_meter; + #[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +static GLOBAL: alloc_meter::Metered = + alloc_meter::Metered::new(mimalloc::MiMalloc); #[derive(Debug, Parser)] struct Args { @@ -70,7 +74,9 @@ async fn run(args: Args) -> anyhow::Result<()> { let mut read_time = Duration::ZERO; let mut spill_bytes = 0; let mut peak_memory = 0; + let (mut buffered, mut drained) = (Vec::new(), Vec::new()); for xid in 1..=args.transactions { + let base = GLOBAL.reset_peak(); let start = Instant::now(); for row in 0..args.rows { buffer @@ -94,6 +100,8 @@ async fn run(args: Args) -> anyhow::Result<()> { } write_time += start.elapsed(); spill_bytes += buffer.stats().spill_bytes_active; + buffered.push(GLOBAL.snapshot().since(base)); + let base = GLOBAL.reset_peak(); let start = Instant::now(); let commit_lsn = u64::from(xid + 1) * (args.rows as u64 + 1); let mut drain = buffer @@ -127,10 +135,18 @@ async fn run(args: Args) -> anyhow::Result<()> { drain.finish().await?; buffer.resume_safe_lsn(walshadow::pos::Pos::new(commit_lsn)); read_time += start.elapsed(); + drained.push(GLOBAL.snapshot().since(base)); } + // Largest transaction footprint, buffered rows still owned by buffer + let worst = |d: &[alloc_meter::Delta]| { + d.iter() + .copied() + .max_by_key(|d| d.peak) + .expect("one transaction") + }; let rows = args.rows as f64 * f64::from(args.transactions); println!( - "repeat={} rows={} spill_bytes={} evictions={} peak_buffer_bytes={} write_s={:.6} read_s={:.6} rows_per_s={:.0} spill_write_mib_s={:.2} spill_read_mib_s={:.2}", + "repeat={} rows={} spill_bytes={} evictions={} peak_buffer_bytes={} write_s={:.6} read_s={:.6} rows_per_s={:.0} spill_write_mib_s={:.2} spill_read_mib_s={:.2} buffer {} drain {}", repeat + 1, rows as u64, spill_bytes, @@ -141,6 +157,8 @@ async fn run(args: Args) -> anyhow::Result<()> { rows / (write_time + read_time).as_secs_f64(), spill_bytes as f64 / (1 << 20) as f64 / write_time.as_secs_f64(), spill_bytes as f64 / (1 << 20) as f64 / read_time.as_secs_f64(), + worst(&buffered), + worst(&drained), ); assert_eq!(buffer.stats().bytes_in_memory, 0); assert_eq!(buffer.stats().spill_bytes_active, 0); From bb5cf63c5ea3fc8878d38e3bf5004a9f6b3d210b Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:31:18 +0000 Subject: [PATCH 03/20] Walk tuples into borrowed datums and encode them directly Split heap tuple walking from owned cell construction. One generic walker feeds either DecodedTuple cells or a TupleView of DatumRefs borrowing the record: fixed bytes, inline or compressed varlena bodies, external pointers, SQL NULL, absent and missing attributes. Multi-insert tuples no longer copy into a synthesized header buffer. Encoding policy now runs over ValueRef, a borrowed mirror of ColumnValue, through one shared cell core. append_routed adapts owned cells to it; a test-only append_view feeds it from datums, materializing only fast defaults, compressed bodies, numeric, jsonb and PostGIS renders. Each column plan records its source attribute and compiles a Direct choice: copy little-endian bytes only when source type and target storage kind match exactly, else decode through shared policy. Production ownership is unchanged. Parity tests compare blocks and errors from both paths across more-types additions, float bits, epochs, numeric and jsonb, compressed and external values, defaults, dropped and absent attributes, overrides, substitutes and oracle columns; every heap decoder test also checks views build equal cells. Co-Authored-By: Claude Opus 5.5 --- src/decode/codecs/inet.rs | 2 +- src/decode/datum.rs | 166 ++++++++ src/decode/heap_decoder.rs | 700 +++++++++++++++++-------------- src/decode/mod.rs | 1 + src/emit/ch_emitter.rs | 376 +++++++++++++---- src/emit/direct_parity.rs | 833 +++++++++++++++++++++++++++++++++++++ src/emit/mod.rs | 2 + src/ops/oracle.rs | 15 +- 8 files changed, 1696 insertions(+), 399 deletions(-) create mode 100644 src/decode/datum.rs create mode 100644 src/emit/direct_parity.rs diff --git a/src/decode/codecs/inet.rs b/src/decode/codecs/inet.rs index 9d2bbd0b..b1674450 100644 --- a/src/decode/codecs/inet.rs +++ b/src/decode/codecs/inet.rs @@ -18,7 +18,7 @@ use crate::ascii_buf::AsciiBuf; pub const PGSQL_AF_INET: u8 = 2; pub const PGSQL_AF_INET6: u8 = 3; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct InetValue { pub family: u8, pub bits: u8, diff --git a/src/decode/datum.rs b/src/decode/datum.rs new file mode 100644 index 00000000..5894590c --- /dev/null +++ b/src/decode/datum.rs @@ -0,0 +1,166 @@ +//! Borrowed tuple attributes and values, so encoders read source bytes in +//! place instead of building owned cells per row + +use crate::decode::codecs::{InetValue, IntervalValue, NumericKind}; +use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; + +/// One attribute as stored in a tuple image, storage framing stripped +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum DatumRef<'a> { + /// Not in WAL image: prefix/suffix elision or truncated tail + Absent, + /// Bitmap bit clear + Null, + /// Past tuple `natts`, resolves through attribute's fast default + Missing, + Dropped, + /// `typlen > 0` bytes as stored + Fixed(&'a [u8]), + /// Uncompressed varlena body + Varlena(&'a [u8]), + /// Inline compressed varlena; `datum` spans header and body + Compressed { + method: u8, + raw_len: usize, + body: &'a [u8], + datum: &'a [u8], + }, + External(ToastPointer), + /// `typlen = -2` body without trailing NUL + Cstring(&'a [u8]), + /// Varlena tag PostgreSQL never writes to disk; rest of buffer + BadTag(&'a [u8]), +} + +/// Borrowed counterpart of [`ColumnValue`], one variant per variant +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ValueRef<'a> { + Null, + Bool(bool), + Char(i8), + Int2(i16), + Int4(i32), + Int8(i64), + Float4(f32), + Float8(f64), + Oid(u32), + Oid8(u64), + MacAddr([u8; 6]), + MacAddr8([u8; 8]), + Tid { block: u32, offset: u16 }, + Date(i32), + Time(i64), + Timestamp(i64), + TimestampTz(i64), + TimeTz { micros: i64, tz_seconds: i32 }, + Uuid([u8; 16]), + Name(&'a str), + Bytea(&'a [u8]), + Text(&'a str), + Numeric(&'a NumericKind), + Inet(InetValue), + Interval(IntervalValue), + Json(&'a str), + PgPending { type_oid: u32, raw: &'a [u8] }, + PgPendingText { type_oid: u32, text: &'a str }, + ExternalToast(ToastPointer), + Unsupported { type_oid: u32, raw: &'a [u8] }, +} + +impl ColumnValue { + pub fn view(&self) -> ValueRef<'_> { + match self { + Self::Null => ValueRef::Null, + Self::Bool(v) => ValueRef::Bool(*v), + Self::Char(v) => ValueRef::Char(*v), + Self::Int2(v) => ValueRef::Int2(*v), + Self::Int4(v) => ValueRef::Int4(*v), + Self::Int8(v) => ValueRef::Int8(*v), + Self::Float4(v) => ValueRef::Float4(*v), + Self::Float8(v) => ValueRef::Float8(*v), + Self::Oid(v) => ValueRef::Oid(*v), + Self::Oid8(v) => ValueRef::Oid8(*v), + Self::MacAddr(v) => ValueRef::MacAddr(*v), + Self::MacAddr8(v) => ValueRef::MacAddr8(*v), + Self::Tid { block, offset } => ValueRef::Tid { + block: *block, + offset: *offset, + }, + Self::Date(v) => ValueRef::Date(*v), + Self::Time(v) => ValueRef::Time(*v), + Self::Timestamp(v) => ValueRef::Timestamp(*v), + Self::TimestampTz(v) => ValueRef::TimestampTz(*v), + Self::TimeTz { micros, tz_seconds } => ValueRef::TimeTz { + micros: *micros, + tz_seconds: *tz_seconds, + }, + Self::Uuid(v) => ValueRef::Uuid(*v), + Self::Name(s) => ValueRef::Name(s), + Self::Bytea(b) => ValueRef::Bytea(b), + Self::Text(s) => ValueRef::Text(s), + Self::Numeric(n) => ValueRef::Numeric(n), + Self::Inet(v) => ValueRef::Inet(*v), + Self::Interval(v) => ValueRef::Interval(*v), + Self::Json(s) => ValueRef::Json(s), + Self::PgPending { type_oid, raw } => ValueRef::PgPending { + type_oid: *type_oid, + raw, + }, + Self::PgPendingText { type_oid, text } => ValueRef::PgPendingText { + type_oid: *type_oid, + text, + }, + Self::ExternalToast(p) => ValueRef::ExternalToast(*p), + Self::Unsupported { type_oid, raw } => ValueRef::Unsupported { + type_oid: *type_oid, + raw, + }, + } + } +} + +impl ValueRef<'_> { + pub fn to_owned_value(self) -> ColumnValue { + match self { + Self::Null => ColumnValue::Null, + Self::Bool(v) => ColumnValue::Bool(v), + Self::Char(v) => ColumnValue::Char(v), + Self::Int2(v) => ColumnValue::Int2(v), + Self::Int4(v) => ColumnValue::Int4(v), + Self::Int8(v) => ColumnValue::Int8(v), + Self::Float4(v) => ColumnValue::Float4(v), + Self::Float8(v) => ColumnValue::Float8(v), + Self::Oid(v) => ColumnValue::Oid(v), + Self::Oid8(v) => ColumnValue::Oid8(v), + Self::MacAddr(v) => ColumnValue::MacAddr(v), + Self::MacAddr8(v) => ColumnValue::MacAddr8(v), + Self::Tid { block, offset } => ColumnValue::Tid { block, offset }, + Self::Date(v) => ColumnValue::Date(v), + Self::Time(v) => ColumnValue::Time(v), + Self::Timestamp(v) => ColumnValue::Timestamp(v), + Self::TimestampTz(v) => ColumnValue::TimestampTz(v), + Self::TimeTz { micros, tz_seconds } => ColumnValue::TimeTz { micros, tz_seconds }, + Self::Uuid(v) => ColumnValue::Uuid(v), + Self::Name(s) => ColumnValue::Name(s.to_owned()), + Self::Bytea(b) => ColumnValue::Bytea(b.to_vec()), + Self::Text(s) => ColumnValue::Text(s.to_owned()), + Self::Numeric(n) => ColumnValue::Numeric(n.clone()), + Self::Inet(v) => ColumnValue::Inet(v), + Self::Interval(v) => ColumnValue::Interval(v), + Self::Json(s) => ColumnValue::Json(s.to_owned()), + Self::PgPending { type_oid, raw } => ColumnValue::PgPending { + type_oid, + raw: raw.to_vec(), + }, + Self::PgPendingText { type_oid, text } => ColumnValue::PgPendingText { + type_oid, + text: text.to_owned(), + }, + Self::ExternalToast(p) => ColumnValue::ExternalToast(p), + Self::Unsupported { type_oid, raw } => ColumnValue::Unsupported { + type_oid, + raw: raw.to_vec(), + }, + } + } +} diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index 9681f54f..a96b5ff9 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -68,6 +68,8 @@ use std::string::FromUtf8Error; use std::sync::Arc; use smallvec::{SmallVec, smallvec}; + +use crate::decode::datum::{DatumRef, ValueRef}; use thiserror::Error; use walrus::pg::walparser::{RelFileNode, RmId, XLogRecord}; @@ -452,6 +454,80 @@ pub struct DecodedTuple { pub partial: bool, } +/// Builds one tuple image from walked attributes: owned cells for +/// [`DecodedTuple`], borrowed datums for [`TupleView`] +pub trait TupleBuild<'a>: Sized { + fn with_capacity(n: usize) -> Self; + fn push(&mut self, att: &RelAttr, datum: DatumRef<'a>); + fn set_partial(&mut self); +} + +impl<'a> TupleBuild<'a> for DecodedTuple { + fn with_capacity(n: usize) -> Self { + Self { + columns: Vec::with_capacity(n), + partial: false, + } + } + + #[inline(always)] + fn push(&mut self, att: &RelAttr, datum: DatumRef<'a>) { + self.columns.push(datum_value(att, datum)); + } + + fn set_partial(&mut self) { + self.partial = true; + } +} + +/// Tuple image borrowing its record. Indexed by attnum-1 like +/// [`DecodedTuple::columns`] +#[derive(Clone, Debug, PartialEq)] +pub struct TupleView<'a> { + pub datums: Vec>, + pub partial: bool, +} + +impl<'a> TupleBuild<'a> for TupleView<'a> { + fn with_capacity(n: usize) -> Self { + Self { + datums: Vec::with_capacity(n), + partial: false, + } + } + + fn push(&mut self, _: &RelAttr, datum: DatumRef<'a>) { + self.datums.push(datum); + } + + fn set_partial(&mut self) { + self.partial = true; + } +} + +impl TupleView<'_> { + /// Owned cells, as [`decode_heap_record`] builds them + pub fn to_tuple(&self, rel: &RelDescriptor) -> DecodedTuple { + DecodedTuple { + columns: self + .datums + .iter() + .zip(&rel.attributes) + .map(|(d, att)| datum_value(att, *d)) + .collect(), + partial: self.partial, + } + } +} + +/// Images of one heap tuple change, before identity and position attach +#[derive(Clone, Debug, PartialEq)] +pub struct HeapImages { + pub op: HeapOp, + pub new: Option, + pub old: Option, +} + /// Top-level decode entry point. One-element [`DecodedHeaps`] for /// INSERT/UPDATE/HOT_UPDATE/DELETE, N-element for `XLOG_HEAP2_MULTI_INSERT`, /// empty for ops with no tuple payload (LOCK, INPLACE, TRUNCATE, other @@ -465,26 +541,50 @@ pub fn decode_heap_record( source_lsn: u64, rel: &RelDescriptor, ) -> Result { - let rm = record.header.resource_manager_id; - let info_op = record.header.info & XLOG_HEAP_OPMASK; let rfn = record .blocks .first() .map(|b| b.header.location.rel) .unwrap_or_default(); let xid = record.header.xact_id; + Ok(parse_heap_record::(record, rel)? + .into_iter() + .map(|images| DecodedHeap { + rfn, + xid, + source_lsn, + op: images.op, + new: images.new, + old: images.old, + }) + .collect()) +} + +/// [`decode_heap_record`] borrowing attribute bytes from `record` +pub fn view_heap_record<'a>( + record: &'a XLogRecord<'_>, + rel: &RelDescriptor, +) -> Result>; 1]>, DecodeError> { + parse_heap_record(record, rel) +} +fn parse_heap_record<'a, T: TupleBuild<'a>>( + record: &'a XLogRecord<'_>, + rel: &RelDescriptor, +) -> Result; 1]>, DecodeError> { + let rm = record.header.resource_manager_id; + let info_op = record.header.info & XLOG_HEAP_OPMASK; if rm == RmId::Heap2 as u8 && info_op == XLOG_HEAP2_MULTI_INSERT { - return decode_multi_insert(record, source_lsn, rfn, xid, rel); + return decode_multi_insert(record, rel); } if rm != RmId::Heap as u8 { return Ok(SmallVec::new()); } let op = match info_op { - XLOG_HEAP_INSERT => decode_insert(record, source_lsn, rfn, xid, rel)?, - XLOG_HEAP_UPDATE => decode_update(record, source_lsn, rfn, xid, rel, false)?, - XLOG_HEAP_HOT_UPDATE => decode_update(record, source_lsn, rfn, xid, rel, true)?, - XLOG_HEAP_DELETE => decode_delete(record, source_lsn, rfn, xid, rel)?, + XLOG_HEAP_INSERT => decode_insert(record, rel)?, + XLOG_HEAP_UPDATE => decode_update(record, rel, false)?, + XLOG_HEAP_HOT_UPDATE => decode_update(record, rel, true)?, + XLOG_HEAP_DELETE => decode_delete(record, rel)?, _ => return Ok(SmallVec::new()), }; Ok(smallvec![op]) @@ -501,13 +601,10 @@ pub const SIZE_OF_MULTI_INSERT_TUPLE: usize = 7; /// sequential `FirstOffsetNumber + i`), PG `heapam.c::heap_multi_insert`. pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80; -fn decode_multi_insert( - record: &XLogRecord, - source_lsn: u64, - rfn: RelFileNode, - xid: u32, +fn decode_multi_insert<'a, T: TupleBuild<'a>>( + record: &'a XLogRecord<'_>, rel: &RelDescriptor, -) -> Result { +) -> Result; 1]>, DecodeError> { let md = &record.main_data; if md.len() < SIZE_OF_HEAP_MULTI_INSERT { return Err(DecodeError::Truncated { @@ -551,9 +648,9 @@ fn decode_multi_insert( let Some(block) = record.blocks.first() else { return Ok(SmallVec::new()); }; - let data = &block.data; + let data: &'a [u8] = &block.data; let mut cursor = 0usize; - let mut out: DecodedHeaps = SmallVec::with_capacity(ntuples); + let mut out = SmallVec::with_capacity(ntuples); for _ in 0..ntuples { // Each xl_multi_insert_tuple SHORTALIGN'd (2-byte) at write time, // PG `heap_xlog_multi_insert` @@ -584,10 +681,7 @@ fn decode_multi_insert( } let tup = walk_tuple(&data[..cursor + datalen], header, cursor, rel, 0)?; cursor += datalen; - out.push(DecodedHeap { - rfn, - xid, - source_lsn, + out.push(HeapImages { op: HeapOp::Insert, new: Some(tup), old: None, @@ -596,44 +690,28 @@ fn decode_multi_insert( Ok(out) } -fn decode_insert( - record: &XLogRecord, - source_lsn: u64, - rfn: RelFileNode, - xid: u32, +fn decode_insert<'a, T: TupleBuild<'a>>( + record: &'a XLogRecord<'_>, rel: &RelDescriptor, -) -> Result { - let Some(block) = record.blocks.first() else { - // PG always references target page; emit empty new rather than - // fail the stream - return Ok(DecodedHeap { - rfn, - xid, - source_lsn, - op: HeapOp::Insert, - new: None, - old: None, - }); +) -> Result, DecodeError> { + // PG always references target page; emit empty new rather than fail + // the stream + let new = match record.blocks.first() { + Some(block) => Some(decode_tuple_payload(&block.data, 0, rel, 0)?), + None => None, }; - let new = decode_new_tuple_block(&block.data, 0, rel)?; - Ok(DecodedHeap { - rfn, - xid, - source_lsn, + Ok(HeapImages { op: HeapOp::Insert, - new: Some(new), + new, old: None, }) } -fn decode_update( - record: &XLogRecord, - source_lsn: u64, - rfn: RelFileNode, - xid: u32, +fn decode_update<'a, T: TupleBuild<'a>>( + record: &'a XLogRecord<'_>, rel: &RelDescriptor, hot: bool, -) -> Result { +) -> Result, DecodeError> { if record.main_data.len() < SIZE_OF_HEAP_UPDATE { return Err(DecodeError::Truncated { offset: 0, @@ -648,19 +726,20 @@ fn decode_update( let has_old_key = flags & XLH_UPDATE_CONTAINS_OLD_KEY != 0; let new = if let Some(b) = record.blocks.first() { + let data: &'a [u8] = &b.data; let mut cursor = 0; let prefixlen = if has_prefix { - read_u16(&b.data, &mut cursor)? as usize + read_u16(data, &mut cursor)? as usize } else { 0 }; - // Suffix bytes lie past WAL slice, so its end already bounds the walk + // Suffix bytes are past the WAL slice, so its length goes unread if has_suffix { - read_u16(&b.data, &mut cursor)?; + read_u16(data, &mut cursor)?; } - let mut t = decode_tuple_payload(&b.data, cursor, rel, prefixlen)?; + let mut t: T = decode_tuple_payload(data, cursor, rel, prefixlen)?; if has_prefix || has_suffix { - t.partial = true; + t.set_partial(); } Some(t) } else { @@ -678,23 +757,13 @@ fn decode_update( } else { HeapOp::Update }; - Ok(DecodedHeap { - rfn, - xid, - source_lsn, - op, - new, - old, - }) + Ok(HeapImages { op, new, old }) } -fn decode_delete( - record: &XLogRecord, - source_lsn: u64, - rfn: RelFileNode, - xid: u32, +fn decode_delete<'a, T: TupleBuild<'a>>( + record: &'a XLogRecord<'_>, rel: &RelDescriptor, -) -> Result { +) -> Result, DecodeError> { if record.main_data.len() < SIZE_OF_HEAP_DELETE { return Err(DecodeError::Truncated { offset: 0, @@ -709,37 +778,18 @@ fn decode_delete( } else { None }; - Ok(DecodedHeap { - rfn, - xid, - source_lsn, + Ok(HeapImages { op: HeapOp::Delete, new: None, old, }) } -/// Block-0 data shape: `xl_heap_header (5)` + bitmap[+pad] + col data. -fn decode_new_tuple_block( - data: &[u8], - data_start: usize, - rel: &RelDescriptor, -) -> Result { - decode_tuple_payload(data, data_start, rel, 0) -} - -fn decode_old_tuple_from_main_data( - main_data: &[u8], +fn decode_old_tuple_from_main_data<'a, T: TupleBuild<'a>>( + main_data: &'a [u8], header_off: usize, rel: &RelDescriptor, -) -> Result, DecodeError> { - if main_data.len() < header_off + SIZE_OF_HEAP_HEADER { - return Err(DecodeError::Truncated { - offset: header_off, - need: SIZE_OF_HEAP_HEADER, - have: main_data.len() - header_off, - }); - } +) -> Result, DecodeError> { Ok(Some(decode_tuple_payload(main_data, header_off, rel, 0)?)) } @@ -754,6 +804,14 @@ pub(crate) fn decode_block_data( decode_tuple_payload(block_data, 0, rel, 0) } +/// Tuple of a `block.data`-shaped buffer, borrowing attribute bytes +pub fn view_block_data<'a>( + block_data: &'a [u8], + rel: &RelDescriptor, +) -> Result, DecodeError> { + decode_tuple_payload(block_data, 0, rel, 0) +} + /// Fields PG strips into `xl_heap_header` or `xl_multi_insert_tuple` #[derive(Clone, Copy)] struct TupleHeader { @@ -767,15 +825,15 @@ struct TupleHeader { /// Walk a tuple payload (`xl_heap_header` at `header_off`, then bitmap, /// padding, col data). `prefixlen` is PG `XLH_UPDATE_PREFIX_FROM_OLD` byte /// count elided from front of logical col-data region; columns wholly inside -/// it or past end of `buf` surface `None` and caller flips `partial`. +/// it or past end of `buf` surface absent and caller flips `partial`. /// Bitmap + xl_heap_header precede prefix elision so are always present at /// `header_off`. -fn decode_tuple_payload( - buf: &[u8], +fn decode_tuple_payload<'a, T: TupleBuild<'a>>( + buf: &'a [u8], header_off: usize, rel: &RelDescriptor, prefixlen: usize, -) -> Result { +) -> Result { if buf.len() < header_off + SIZE_OF_HEAP_HEADER { return Err(DecodeError::Truncated { offset: header_off, @@ -798,15 +856,15 @@ fn decode_tuple_payload( ) } -/// Walk bitmap, padding and col data starting at `bitmap_off`, one column per +/// Walk bitmap, padding and col data starting at `bitmap_off`, one datum per /// attribute of `rel`. `buf` ends where tuple ends -fn walk_tuple( - buf: &[u8], +fn walk_tuple<'a, T: TupleBuild<'a>>( + buf: &'a [u8], header: TupleHeader, bitmap_off: usize, rel: &RelDescriptor, prefixlen: usize, -) -> Result { +) -> Result { let t_hoff = usize::from(header.hoff); if t_hoff < SIZE_OF_HEAP_TUPLE_HEADER { return Err(DecodeError::BadHoff { @@ -833,13 +891,20 @@ fn walk_tuple( // carried verbatim in WAL let col_data_off = bitmap_off + (t_hoff - SIZE_OF_HEAP_TUPLE_HEADER); - let mut columns = Vec::with_capacity(rel.attributes.len()); + let attrs = &rel.attributes; + let mut out = T::with_capacity(attrs.len()); + // Mark this + every later col absent + let absent_from = |out: &mut T, idx: usize| { + for att in &attrs[idx..] { + out.push(att, DatumRef::Absent); + } + out.set_partial(); + }; // `cur` is offset within the *logical* col-data region (relative to // t_hoff); align matches PG `att_align_nominal` against this offset. t_hoff // is MAXALIGN'd(8) so col 1 starts clean. WAL offset of logical `cur` is // `col_data_off + (cur - prefixlen)` when `cur >= prefixlen` let mut cur: usize = 0; - let attrs = effective_attrs(rel, natts); let wal_col_data_avail = buf.len().saturating_sub(col_data_off); let logical_end = wal_col_data_avail + prefixlen; for (idx, att) in attrs.iter().enumerate() { @@ -847,11 +912,11 @@ fn walk_tuple( // wal natts < catalog natts: trailing cols added via ALTER ADD // COLUMN after the record. Mirror PG `getmissingattr` // (heaptuple.c) `attmissingval[1]` fast-path default - columns.push(Some(missing_value_for(att))); + out.push(att, DatumRef::Missing); continue; } if has_null && !bitmap_is_set(bitmap, idx) { - columns.push(Some(ColumnValue::Null)); + out.push(att, DatumRef::Null); continue; } let col_size_hint = att.type_len; @@ -871,46 +936,16 @@ fn walk_tuple( cur = align_for(cur, att.type_align); } - if cur < prefixlen { - let len = decoded_size_or_skip(att); - match len { - Some(n) => { - columns.push(None); - cur += n; - } - None => { - // Varlena in prefix: can't advance cur without the bytes. - // Mark this + every later col absent, bail; xact buffer - // reconstructs - columns.push(None); - for _ in (idx + 1)..attrs.len() { - columns.push(None); - } - return Ok(DecodedTuple { - columns, - partial: true, - }); - } - } - continue; - } - - if cur >= logical_end { - // In suffix / past EOF - columns.push(None); - // Advance cur for fixed-width accounting only; all emit None - if let Some(n) = decoded_size_or_skip(att) { - cur += n; - } else { - // Varlena tail; remaining cols stay absent - for _ in (idx + 1)..attrs.len() { - columns.push(None); - } - return Ok(DecodedTuple { - columns, - partial: true, - }); - } + if cur < prefixlen || cur >= logical_end { + // In prefix, or suffix / past EOF. Advance cur for fixed-width + // accounting only; varlena width needs its bytes so later cols + // stay absent too + let Some(n) = decoded_size_or_skip(att) else { + absent_from(&mut out, idx); + return Ok(out); + }; + out.push(att, DatumRef::Absent); + cur += n; continue; } @@ -918,28 +953,19 @@ fn walk_tuple( if att.type_len > 0 && !att.dropped { let len = att.type_len as usize; if let Some(body) = buf.get(abs..abs + len) { - columns.push(Some(fixed_value(att, body))); + out.push(att, DatumRef::Fixed(body)); cur += len; continue; } } - let Ok((value, consumed)) = decode_one_value(att, buf, abs) else { - columns.push(None); - for _ in (idx + 1)..attrs.len() { - columns.push(None); - } - return Ok(DecodedTuple { - columns, - partial: true, - }); + let Ok((datum, consumed)) = read_datum(att, buf, abs) else { + absent_from(&mut out, idx); + return Ok(out); }; - columns.push(Some(value)); + out.push(att, datum); cur += consumed; } - Ok(DecodedTuple { - columns, - partial: false, - }) + Ok(out) } /// Resolve a column's fast default ([`RelAttr::missing_default`]) to a value; @@ -1072,14 +1098,6 @@ fn align_for(cur: usize, attalign: char) -> usize { } } -/// Always `rel.attributes`. Replica-identity-only writes (UsingIndex/Default) -/// carry NULL placeholders for non-indexed cols, so wire natts equals rel -/// natts. Post-ALTER-ADD-COLUMN trailing attrs (`natts_in_wal < natts`) -/// surface via PG missing-column handling. -fn effective_attrs(rel: &RelDescriptor, _natts: usize) -> &[RelAttr] { - &rel.attributes -} - /// PG `att_align_nominal` (tupmacs.h). For varlena, PG `att_align_pointer` /// skips alignment when next byte is a 1-byte short-header datum /// (`!VARATT_NOT_PAD_BYTE`); mirror by peeking `buf[abs]` before aligning. @@ -1127,35 +1145,106 @@ fn read_u16(buf: &[u8], cur: &mut usize) -> Result { Ok(v) } -/// Returns `(value, bytes_consumed)`. Varlena reports total on-disk length -/// (header + body, including TOAST pointer's full 18 bytes). +/// Read one attribute at `abs`, returning datum and on-disk length. Varlena +/// reports header + body, including TOAST pointer's full 18 bytes +pub(crate) fn read_datum<'a>( + att: &RelAttr, + buf: &'a [u8], + abs: usize, +) -> Result<(DatumRef<'a>, usize), Truncated> { + let (datum, len) = match att.type_len { + -1 => read_varlena(buf, abs)?, + -2 => read_cstring(buf, abs)?, + len => { + let len = len as usize; + let body = buf.get(abs..abs + len).ok_or(Truncated { + offset: abs, + need: len, + have: buf.len().saturating_sub(abs), + })?; + (DatumRef::Fixed(body), len) + } + }; + // Dropped cols retain attlen/attalign/typbyval per catalog convention, so + // reading still advances cursor + Ok(( + if att.dropped { + DatumRef::Dropped + } else { + datum + }, + len, + )) +} + +/// Returns `(value, bytes_consumed)`, see [`read_datum`] pub(crate) fn decode_one_value( att: &RelAttr, buf: &[u8], abs: usize, ) -> Result<(ColumnValue, usize), Truncated> { - if att.dropped { - // Dropped cols retain attlen/attalign/typbyval per catalog - // convention; still read varlena header to advance cursor - let consumed = consume_dropped(att, buf, abs)?; - return Ok((ColumnValue::Null, consumed)); - } + let (datum, len) = read_datum(att, buf, abs)?; + Ok((datum_value(att, datum).unwrap_or(ColumnValue::Null), len)) +} - if att.type_len == -1 { - return decode_varlena(att, buf, abs); - } - if att.type_len == -2 { - return decode_cstring(buf, abs); - } - let len = att.type_len as usize; - if buf.len() < abs + len { - return Err(Truncated { - offset: abs, - need: len, - have: buf.len().saturating_sub(abs), - }); - } - Ok((fixed_value(att, &buf[abs..abs + len]), len)) +/// Owned cell for `datum`, `None` when absent from WAL image +#[inline(always)] +pub fn datum_value(att: &RelAttr, datum: DatumRef<'_>) -> Option { + Some(match datum { + DatumRef::Absent => return None, + DatumRef::Null | DatumRef::Dropped => ColumnValue::Null, + DatumRef::Missing => missing_value_for(att), + DatumRef::Fixed(body) => fixed_value(att, body), + DatumRef::Varlena(body) => varlena_to_value(att.type_oid, Cow::Borrowed(body)), + DatumRef::Compressed { + method, + raw_len, + body, + datum, + } => decompress_varlena(method, body, raw_len) + .map(|out| varlena_to_value(att.type_oid, Cow::Owned(out))) + .unwrap_or_else(|| ColumnValue::Unsupported { + type_oid: att.type_oid, + raw: datum.to_vec(), + }), + DatumRef::External(p) => ColumnValue::ExternalToast(p), + DatumRef::Cstring(body) => simdutf8::basic::from_utf8(body) + .map(|s| ColumnValue::Text(s.to_owned())) + .unwrap_or_else(|_| ColumnValue::Bytea(body.to_vec())), + DatumRef::BadTag(raw) => ColumnValue::Unsupported { + type_oid: att.type_oid, + raw: raw.to_vec(), + }, + }) +} + +/// Datum whose value only [`datum_value`] builds: fast defaults, compressed +/// bodies, `numeric` and `jsonb` +#[derive(Debug)] +pub struct AllocatingDecode; + +/// Borrowed value for `datum`, `Ok(None)` when absent from WAL image +#[inline] +pub fn datum_view<'a>( + att: &RelAttr, + datum: DatumRef<'a>, +) -> Result>, AllocatingDecode> { + Ok(Some(match datum { + DatumRef::Absent => return Ok(None), + DatumRef::Null | DatumRef::Dropped => ValueRef::Null, + DatumRef::Fixed(body) => fixed_view(att, body), + DatumRef::Varlena(body) => varlena_view(att.type_oid, body)?, + DatumRef::External(p) => ValueRef::ExternalToast(p), + DatumRef::Cstring(body) => match simdutf8::basic::from_utf8(body) { + Ok(s) => ValueRef::Text(s), + Err(_) => ValueRef::Bytea(body), + }, + DatumRef::BadTag(raw) => ValueRef::Unsupported { + type_oid: att.type_oid, + raw, + }, + DatumRef::Missing | DatumRef::Compressed { .. } => return Err(AllocatingDecode), + })) } /// Fixed-width value from its on-disk bytes. Inlined so callers construct @@ -1163,79 +1252,57 @@ pub(crate) fn decode_one_value( /// stalls store forwarding, which dominated fixed-width decode #[inline(always)] fn fixed_value(att: &RelAttr, body: &[u8]) -> ColumnValue { + fixed_view(att, body).to_owned_value() +} + +#[inline(always)] +fn fixed_view<'a>(att: &RelAttr, body: &'a [u8]) -> ValueRef<'a> { + let unsupported = || ValueRef::Unsupported { + type_oid: att.type_oid, + raw: body, + }; match att.type_oid { - BOOLOID => ColumnValue::Bool(body[0] != 0), - CHAROID => ColumnValue::Char(body[0] as i8), - INT2OID => ColumnValue::Int2(i16::from_le_bytes(body.try_into().unwrap())), - INT4OID => ColumnValue::Int4(i32::from_le_bytes(body.try_into().unwrap())), + BOOLOID => ValueRef::Bool(body[0] != 0), + CHAROID => ValueRef::Char(body[0] as i8), + INT2OID => ValueRef::Int2(i16::from_le_bytes(body.try_into().unwrap())), + INT4OID => ValueRef::Int4(i32::from_le_bytes(body.try_into().unwrap())), // `money` counts minor units, scale set by source `lc_monetary` - INT8OID | MONEYOID => ColumnValue::Int8(i64::from_le_bytes(body.try_into().unwrap())), - FLOAT4OID => ColumnValue::Float4(f32::from_le_bytes(body.try_into().unwrap())), - FLOAT8OID => ColumnValue::Float8(f64::from_le_bytes(body.try_into().unwrap())), - OIDOID | XIDOID | CIDOID => ColumnValue::Oid(u32::from_le_bytes(body.try_into().unwrap())), + INT8OID | MONEYOID => ValueRef::Int8(i64::from_le_bytes(body.try_into().unwrap())), + FLOAT4OID => ValueRef::Float4(f32::from_le_bytes(body.try_into().unwrap())), + FLOAT8OID => ValueRef::Float8(f64::from_le_bytes(body.try_into().unwrap())), + OIDOID | XIDOID | CIDOID => ValueRef::Oid(u32::from_le_bytes(body.try_into().unwrap())), OID8OID | XID8OID | PG_LSNOID => { - ColumnValue::Oid8(u64::from_le_bytes(body.try_into().unwrap())) + ValueRef::Oid8(u64::from_le_bytes(body.try_into().unwrap())) } // NUL-padded to NAMEDATALEN NAMEOID => { let end = body.iter().position(|&b| b == 0).unwrap_or(body.len()); match simdutf8::basic::from_utf8(&body[..end]) { - Ok(s) => ColumnValue::Name(s.to_owned()), - Err(_) => ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: body.to_vec(), - }, + Ok(s) => ValueRef::Name(s), + Err(_) => unsupported(), } } - MACADDROID => ColumnValue::MacAddr(body.try_into().unwrap()), - MACADDR8OID => ColumnValue::MacAddr8(body.try_into().unwrap()), + MACADDROID => ValueRef::MacAddr(body.try_into().unwrap()), + MACADDR8OID => ValueRef::MacAddr8(body.try_into().unwrap()), TIDOID => { let (block, offset) = crate::decode::codecs::decode_tid(body.try_into().unwrap()); - ColumnValue::Tid { block, offset } + ValueRef::Tid { block, offset } } - DATEOID => ColumnValue::Date(i32::from_le_bytes(body.try_into().unwrap())), - TIMEOID => ColumnValue::Time(i64::from_le_bytes(body.try_into().unwrap())), - TIMESTAMPOID => ColumnValue::Timestamp(i64::from_le_bytes(body.try_into().unwrap())), - TIMESTAMPTZOID => ColumnValue::TimestampTz(i64::from_le_bytes(body.try_into().unwrap())), + DATEOID => ValueRef::Date(i32::from_le_bytes(body.try_into().unwrap())), + TIMEOID => ValueRef::Time(i64::from_le_bytes(body.try_into().unwrap())), + TIMESTAMPOID => ValueRef::Timestamp(i64::from_le_bytes(body.try_into().unwrap())), + TIMESTAMPTZOID => ValueRef::TimestampTz(i64::from_le_bytes(body.try_into().unwrap())), TIMETZOID => { // 12 bytes: 8-byte time + 4-byte tz offset let micros = i64::from_le_bytes(body[..8].try_into().unwrap()); let tz_seconds = i32::from_le_bytes(body[8..12].try_into().unwrap()); - ColumnValue::TimeTz { micros, tz_seconds } + ValueRef::TimeTz { micros, tz_seconds } } - UUIDOID => ColumnValue::Uuid(body.try_into().unwrap()), - INTERVALOID => match crate::decode::codecs::decode_interval(body) { - Ok(v) => ColumnValue::Interval(v), - Err(_) => ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: body.to_vec(), - }, - }, - _ => ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: body.to_vec(), - }, - } -} - -/// Skip a dropped column; still read varlena header to advance correctly. -fn consume_dropped(att: &RelAttr, buf: &[u8], abs: usize) -> Result { - if att.type_len == -1 { - let (_, consumed) = decode_varlena(att, buf, abs)?; - Ok(consumed) - } else if att.type_len == -2 { - let (_, consumed) = decode_cstring(buf, abs)?; - Ok(consumed) - } else { - let len = att.type_len as usize; - if buf.len() < abs + len { - return Err(Truncated { - offset: abs, - need: len, - have: buf.len().saturating_sub(abs), - }); - } - Ok(len) + UUIDOID => ValueRef::Uuid(body.try_into().unwrap()), + INTERVALOID => crate::decode::codecs::decode_interval(body) + .map(ValueRef::Interval) + .unwrap_or_else(|_| unsupported()), + _ => unsupported(), } } @@ -1259,20 +1326,15 @@ pub(crate) fn decompress_varlena(method: u8, src: &[u8], raw_len: usize) -> Opti } } -/// Decode a varlena attribute (`typlen == -1`). PG varlena formats (`varatt.h`): +/// Read a varlena attribute (`typlen == -1`). PG varlena formats (`varatt.h`): /// /// - `0bxxxxxx00` 4-byte length, uncompressed. -/// - `0bxxxxxx10` 4-byte length, in-line compressed (surfaced `Unsupported`; -/// later codec detoasts pglz/lz4). +/// - `0bxxxxxx10` 4-byte length, in-line compressed. /// - `0b00000001` TOAST pointer (`varattrib_1b_e`, 18 bytes on disk). /// - `0bxxxxxxx1` 1-byte length, uncompressed short header. /// /// Little-endian only (`WORDS_BIGENDIAN` dead on x86/aarch64). -fn decode_varlena( - att: &RelAttr, - buf: &[u8], - abs: usize, -) -> Result<(ColumnValue, usize), Truncated> { +fn read_varlena(buf: &[u8], abs: usize) -> Result<(DatumRef<'_>, usize), Truncated> { if buf.len() <= abs { return Err(Truncated { offset: abs, @@ -1301,29 +1363,17 @@ fn decode_varlena( }); } let body = &buf[abs + 2..abs + 2 + 16]; - let va_rawsize = i32::from_le_bytes(body[0..4].try_into().unwrap()); - let va_extinfo = u32::from_le_bytes(body[4..8].try_into().unwrap()); - let va_valueid = u32::from_le_bytes(body[8..12].try_into().unwrap()); - let va_toastrelid = u32::from_le_bytes(body[12..16].try_into().unwrap()); - return Ok(( - ColumnValue::ExternalToast(ToastPointer { - va_rawsize, - va_extinfo, - va_valueid, - va_toastrelid, - }), - 18, - )); + let pointer = ToastPointer { + va_rawsize: i32::from_le_bytes(body[0..4].try_into().unwrap()), + va_extinfo: u32::from_le_bytes(body[4..8].try_into().unwrap()), + va_valueid: u32::from_le_bytes(body[8..12].try_into().unwrap()), + va_toastrelid: u32::from_le_bytes(body[12..16].try_into().unwrap()), + }; + return Ok((DatumRef::External(pointer), 18)); } // INDIRECT/EXPANDED tags are in-memory only, never on disk (varatt.h); // forward Unsupported so a corrupt stream surfaces - return Ok(( - ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: buf[abs..].to_vec(), - }, - buf.len() - abs, - )); + return Ok((DatumRef::BadTag(&buf[abs..]), buf.len() - abs)); } if first & 0x01 != 0 { // Short header: low bit set, total length (incl header byte) in upper 7; @@ -1336,9 +1386,7 @@ fn decode_varlena( have: buf.len() - abs, }); } - let body = &buf[abs + 1..abs + total]; - let v = varlena_to_value(att.type_oid, Cow::Borrowed(body)); - return Ok((v, total)); + return Ok((DatumRef::Varlena(&buf[abs + 1..abs + total]), total)); } // 4-byte length: bits [1:0] = 00 uncompressed, 10 compressed if buf.len() < abs + 4 { @@ -1375,19 +1423,15 @@ fn decode_varlena( }); } let tcinfo = u32::from_le_bytes(buf[abs + 4..abs + 8].try_into().unwrap()); - let raw_len = (tcinfo & VARLENA_EXTSIZE_MASK) as usize; - let method = (tcinfo >> VARLENA_EXTSIZE_BITS) as u8; - let body = &buf[abs + 8..abs + total]; - let v = decompress_varlena(method, body, raw_len) - .map(|out| varlena_to_value(att.type_oid, Cow::Owned(out))) - .unwrap_or_else(|| ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: buf[abs..abs + total].to_vec(), - }); - return Ok((v, total)); + let datum = DatumRef::Compressed { + method: (tcinfo >> VARLENA_EXTSIZE_BITS) as u8, + raw_len: (tcinfo & VARLENA_EXTSIZE_MASK) as usize, + body: &buf[abs + 8..abs + total], + datum: &buf[abs..abs + total], + }; + return Ok((datum, total)); } - let body = &buf[abs + 4..abs + total]; - Ok((varlena_to_value(att.type_oid, Cow::Borrowed(body)), total)) + Ok((DatumRef::Varlena(&buf[abs + 4..abs + total]), total)) } /// Map a varlena body (header stripped, decompressed) to a typed value by the @@ -1445,6 +1489,29 @@ pub(crate) fn varlena_to_value(type_oid: u32, body: Cow<[u8]>) -> ColumnValue { } } +/// [`varlena_to_value`] borrowing an inline body +fn varlena_view(type_oid: u32, body: &[u8]) -> Result, AllocatingDecode> { + Ok(match type_oid { + BYTEAOID => ValueRef::Bytea(body), + TEXTOID | VARCHAROID | BPCHAROID | JSONOID => match simdutf8::basic::from_utf8(body) { + Ok(s) if type_oid == JSONOID => ValueRef::Json(s), + Ok(s) => ValueRef::Text(s), + Err(_) => ValueRef::Bytea(body), + }, + INETOID | CIDROID => crate::decode::codecs::decode_inet(body, type_oid == CIDROID) + .map(ValueRef::Inet) + .unwrap_or(ValueRef::Unsupported { + type_oid, + raw: body, + }), + NUMERICOID | JSONBOID => return Err(AllocatingDecode), + _ => ValueRef::PgPending { + type_oid, + raw: body, + }, + }) +} + /// Match decoder paths producing typed values pub(crate) fn local_matrix_covers(type_oid: u32, type_len: i16) -> bool { match type_len { @@ -1492,26 +1559,20 @@ pub(crate) fn local_matrix_covers(type_oid: u32, type_len: i16) -> bool { } } -/// Handle `typlen=-2` `\0`-terminated cstring (regtype/regclass typoutput +/// Read `typlen=-2` `\0`-terminated cstring (regtype/regclass typoutput /// intermediates). Not in the type matrix, but tracking it lets dropped /// cstring columns walk correctly. (`name`, typlen=64, goes the fixed path.) -fn decode_cstring(buf: &[u8], abs: usize) -> Result<(ColumnValue, usize), Truncated> { - let mut end = abs; - while end < buf.len() && buf[end] != 0 { - end += 1; - } - if end >= buf.len() { - return Err(Truncated { +fn read_cstring(buf: &[u8], abs: usize) -> Result<(DatumRef<'_>, usize), Truncated> { + let end = buf + .get(abs..) + .and_then(|rest| rest.iter().position(|&b| b == 0)) + .ok_or(Truncated { offset: abs, need: 1, have: 0, - }); - } - let body = &buf[abs..end]; - let value = simdutf8::basic::from_utf8(body) - .map(|s| ColumnValue::Text(s.to_owned())) - .unwrap_or_else(|_| ColumnValue::Bytea(body.to_vec())); - Ok((value, (end - abs) + 1)) // include trailing NUL + })?; + // include trailing NUL + Ok((DatumRef::Cstring(&buf[abs..abs + end]), end + 1)) } /// Take `(chunk_id, chunk_seq, chunk_data)` out of a `pg_toast_*` tuple's @@ -1786,6 +1847,37 @@ mod tests { } } + /// Every decode here also checks borrowed views build equal cells + fn decode_heap_record( + record: &XLogRecord, + source_lsn: u64, + rel: &RelDescriptor, + ) -> Result { + let owned = super::decode_heap_record(record, source_lsn, rel); + let cells = |t: &Option>| t.as_ref().map(|t| t.to_tuple(rel)); + match (&owned, view_heap_record(record, rel)) { + (Ok(heaps), Ok(views)) => assert_eq!( + heaps + .iter() + .map(|h| (h.op, h.new.clone(), h.old.clone())) + .collect::>(), + views + .iter() + .map(|v| (v.op, cells(&v.new), cells(&v.old))) + .collect::>(), + ), + (Err(a), Err(b)) => assert_eq!(a.to_string(), b.to_string()), + (a, b) => panic!("owned {a:?} vs view {b:?}"), + } + owned + } + + fn decode_cstring(buf: &[u8], abs: usize) -> Result<(ColumnValue, usize), Truncated> { + let (datum, n) = read_cstring(buf, abs)?; + let att = rel_attr(1, "c", 0, -2, 'c'); + Ok((datum_value(&att, datum).unwrap(), n)) + } + #[test] fn decode_cstring_text_bytea_and_truncated() { let (v, n) = decode_cstring(b"hello\0", 0).unwrap(); @@ -1906,7 +1998,7 @@ mod tests { assert!(comp.len() < text.len(), "must actually compress"); let buf = inline_compressed_varlena(0, &comp, text.len()); let att = rel_attr(1, "val", TEXTOID, -1, 'i'); - let (v, consumed) = decode_varlena(&att, &buf, 0).unwrap(); + let (v, consumed) = decode_one_value(&att, &buf, 0).unwrap(); assert_eq!(consumed, buf.len()); assert_eq!(v, ColumnValue::Text(text)); } @@ -1917,7 +2009,7 @@ mod tests { let comp = lz4::block::compress(text.as_bytes(), None, false).expect("lz4 compress"); let buf = inline_compressed_varlena(1, &comp, text.len()); let att = rel_attr(1, "val", TEXTOID, -1, 'i'); - let (v, consumed) = decode_varlena(&att, &buf, 0).unwrap(); + let (v, consumed) = decode_one_value(&att, &buf, 0).unwrap(); assert_eq!(consumed, buf.len()); assert_eq!(v, ColumnValue::Text(text)); } @@ -1926,7 +2018,7 @@ mod tests { fn decode_inline_compressed_corrupt_surfaces_unsupported() { let buf = inline_compressed_varlena(0, &[0xff, 0xff, 0xff, 0xff], 4096); let att = rel_attr(1, "val", TEXTOID, -1, 'i'); - let (v, consumed) = decode_varlena(&att, &buf, 0).unwrap(); + let (v, consumed) = decode_one_value(&att, &buf, 0).unwrap(); assert_eq!(consumed, buf.len()); assert!(matches!( v, diff --git a/src/decode/mod.rs b/src/decode/mod.rs index 33c9b9e2..271ab5ef 100644 --- a/src/decode/mod.rs +++ b/src/decode/mod.rs @@ -1,4 +1,5 @@ pub mod codecs; +pub mod datum; pub mod decoder_sink; pub mod fpi; pub mod heap_decoder; diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 0fec3f0b..8b8dac0b 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -35,8 +35,13 @@ use crate::ch::is_retryable; use crate::ch::{CompressionChoice, ConnectionConfig, EmitterError, quote_ident, types}; use crate::column_rules::{ColumnEntry, ColumnRule, ColumnRules, Substitute}; #[cfg(test)] +use crate::decode::datum::DatumRef; +use crate::decode::datum::ValueRef; +#[cfg(test)] use crate::decode::decoder_sink::DecoderSinkError; use crate::decode::heap_decoder::{ColumnValue, CommittedTuple, HeapOp}; +#[cfg(test)] +use crate::decode::heap_decoder::{datum_value, datum_view}; use crate::mapping::{ ColumnMapping, DropTableStrategy, NamespaceMapping, SystemColumnNames, SystemColumns, TableMapping, TableTarget, named_instance, @@ -52,6 +57,7 @@ use crate::source::queueing_record_sink::{ }; use crate::table_rules::{MatchKind, PartitionRoute, TableRule}; use foldhash::{HashMap, HashMapExt}; +use smallvec::SmallVec; /// Microseconds between PG `TimestampTz` epoch (2000-01-01 UTC) and Unix /// epoch. CH `DateTime64(6)` is Unix microseconds; PG commit-record @@ -1429,6 +1435,62 @@ pub(crate) struct ColumnPlan { pub(crate) datetime64_scale: u32, pub(crate) non_finite: NonFinite, pub(crate) encoding: ColumnEncoding, + /// Source attribute feeding this column, `None` when descriptor lacks it + pub(crate) source: Option, +} + +/// Per-row handling of a source attribute, compiled with its target +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SourceColumn { + /// attnum - 1, indexes tuple images + pub(crate) index: usize, + pub(crate) direct: Direct, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Direct { + /// Stored little-endian bytes already match target storage + Copy, + /// Decode through shared value policy + Value, + /// PostGIS point rendered as WKT, typoutput yields HEXEWKB + Wkt, +} + +impl Direct { + fn choose(att: &RelAttr, plan: &ColumnPlan) -> Self { + use crate::schema::{ + CHAROID, CIDOID, FLOAT4OID, FLOAT8OID, INT2OID, INT4OID, INT8OID, MONEYOID, OID8OID, + OIDOID, PG_LSNOID, XID8OID, XIDOID, + }; + if matches!(att.type_name.as_str(), "geography" | "geometry") { + return Self::Wkt; + } + if plan.encoding != ColumnEncoding::Local { + return Self::Value; + } + // Equal width alone is no proof: signedness, float layout, and + // epoch all differ between same-sized types + let storage = match att.type_oid { + CHAROID => Kind::Int8, + INT2OID => Kind::Int16, + INT4OID => Kind::Int32, + INT8OID | MONEYOID => Kind::Int64, + FLOAT4OID => Kind::Float32, + FLOAT8OID => Kind::Float64, + OIDOID | XIDOID | CIDOID => Kind::UInt32, + OID8OID | XID8OID | PG_LSNOID => Kind::UInt64, + _ => return Self::Value, + }; + let inner = types::strip_nullable(plan.ast.view()); + if inner.kind() == Some(storage) + && usize::try_from(att.type_len).is_ok_and(|n| n == inner.elem_size()) + { + Self::Copy + } else { + Self::Value + } + } } /// Values replacing non-finite numerics or temporal infinities, `None` rejects @@ -1494,21 +1556,21 @@ impl NonFinite { }) } - fn substitute<'a>(&'a self, v: &'a ColumnValue) -> &'a ColumnValue { + fn substitute<'a>(&'a self, v: ValueRef<'a>) -> ValueRef<'a> { use crate::decode::codecs::NumericKind; let sub = match v { - ColumnValue::Numeric(NumericKind::NaN) => &self.nan, - ColumnValue::Numeric(NumericKind::PInf) - | ColumnValue::Date(i32::MAX) - | ColumnValue::Timestamp(i64::MAX) - | ColumnValue::TimestampTz(i64::MAX) => &self.pos_inf, - ColumnValue::Numeric(NumericKind::NInf) - | ColumnValue::Date(i32::MIN) - | ColumnValue::Timestamp(i64::MIN) - | ColumnValue::TimestampTz(i64::MIN) => &self.neg_inf, + ValueRef::Numeric(NumericKind::NaN) => &self.nan, + ValueRef::Numeric(NumericKind::PInf) + | ValueRef::Date(i32::MAX) + | ValueRef::Timestamp(i64::MAX) + | ValueRef::TimestampTz(i64::MAX) => &self.pos_inf, + ValueRef::Numeric(NumericKind::NInf) + | ValueRef::Date(i32::MIN) + | ValueRef::Timestamp(i64::MIN) + | ValueRef::TimestampTz(i64::MIN) => &self.neg_inf, _ => return v, }; - sub.as_ref().unwrap_or(v) + sub.as_ref().map_or(v, ColumnValue::view) } } @@ -1668,6 +1730,7 @@ impl TablePlan { ast, decimal, non_finite: NonFinite::default(), + source: None, }; let settings = att .map(|a| column_rules.settings(owner, &a.name)) @@ -1686,6 +1749,7 @@ impl TablePlan { ast: oast, decimal, non_finite: NonFinite::default(), + source: None, }; } else { tracing::warn!( @@ -1731,6 +1795,12 @@ impl TablePlan { if plan.encoding != ColumnEncoding::Local && plan.non_finite != NonFinite::default() { return Err(config_err("substitution needs local encoding")); } + plan.source = att.and_then(|a| { + Some(SourceColumn { + index: usize::try_from(c.src_attnum - 1).ok()?, + direct: Direct::choose(a, &plan), + }) + }); columns.push(plan); col_sql.push(quote_ident(&c.target_name)); } @@ -1742,6 +1812,7 @@ impl TablePlan { datetime64_scale: 0, non_finite: NonFinite::default(), encoding: ColumnEncoding::Local, + source: None, }; let synth_lsn = mk(&system.lsn, "UInt64"); let synth_xid = mk(&system.xid, "UInt32"); @@ -2090,33 +2161,111 @@ impl TableEncoder { HeapOp::Delete => decoded.old.as_ref(), _ => decoded.new.as_ref(), }; - let value_of = |col: &ColumnMapping| { - side.and_then(|t| t.columns.get((col.src_attnum - 1) as usize)) - .and_then(|opt| opt.as_ref()) + let cells: SmallVec<[Cell<'_>; 16]> = mapping + .columns + .iter() + .map(|col| { + side.and_then(|t| t.columns.get((col.src_attnum - 1) as usize)) + .and_then(Option::as_ref) + .map_or(Cell::Absent, |v| Cell::Value(v.view())) + }) + .collect(); + let stamp = RowStamp { + source_lsn: decoded.source_lsn, + xid: decoded.xid, + commit_ts: committed.commit_ts, + op_code, + leaf, + }; + self.append_cells(&cells, &stamp) + } + + /// [`Self::append_routed`] reading one image of `rel` in place. Values + /// whose decode allocates, and PostGIS renders, build in row scratch + #[cfg(test)] + pub(crate) fn append_view( + &mut self, + rel: &RelDescriptor, + datums: &[DatumRef<'_>], + stamp: &RowStamp, + ) -> Result { + enum Slot<'a> { + Cell(Cell<'a>), + Owned(usize), + } + let mut owned: SmallVec<[ColumnValue; 2]> = SmallVec::new(); + let mut own = |v: ColumnValue| { + owned.push(v); + Slot::Owned(owned.len() - 1) }; + let slots: SmallVec<[Slot<'_>; 16]> = self + .plan + .columns + .iter() + .map(|c| { + let Some((src, &datum)) = + c.source.and_then(|src| Some((src, datums.get(src.index)?))) + else { + return Slot::Cell(Cell::Absent); + }; + let att = &rel.attributes[src.index]; + match (src.direct, datum) { + (Direct::Copy, DatumRef::Fixed(le)) => Slot::Cell(Cell::Le(le)), + (Direct::Wkt, _) => match datum_value(att, datum) { + None => Slot::Cell(Cell::Absent), + Some(mut v) => { + crate::ops::oracle::render_ext_value(&mut v); + own(v) + } + }, + _ => match datum_view(att, datum) { + Ok(v) => Slot::Cell(v.map_or(Cell::Absent, Cell::Value)), + Err(_) => { + own(datum_value(att, datum).expect("allocating datum is present")) + } + }, + } + }) + .collect(); + let cells: SmallVec<[Cell<'_>; 16]> = slots + .iter() + .map(|s| match s { + Slot::Cell(c) => *c, + Slot::Owned(i) => Cell::Value(owned[*i].view()), + }) + .collect(); + self.append_cells(&cells, stamp) + } + + /// Append one row of cells in `plan.columns` order + fn append_cells( + &mut self, + cells: &[Cell<'_>], + stamp: &RowStamp, + ) -> Result { // Size oracle cells before mutating column buffers - let mut cells = std::mem::take(&mut self.oracle_cells); - cells.clear(); + let mut oracle = std::mem::take(&mut self.oracle_cells); + oracle.clear(); let mut row_oracle_bytes = 0usize; // Literals bound for a String target never reach the request, so they // owe it nothing until a cell needing conversion strands them let mut stranded_bytes = 0usize; - for (i, col) in mapping.columns.iter().enumerate() { + for (i, cell) in cells.iter().enumerate() { if let ColumnBuf::Oracle(o) = &self.buffers[i] { - let cell = oracle_cell(value_of(col), o.source_type_oid) - .map_err(|e| name_column(e, &col.target_name))?; + let cell = oracle_cell(*cell, o.source_type_oid) + .map_err(|e| name_column(e, &self.plan.columns[i].name))?; if !o.resolves_locally(&cell) { row_oracle_bytes += cell.wire_bytes(); stranded_bytes += o.stranded_bytes(); } - cells.push(cell); + oracle.push(cell); } } if self.rows > 0 && self.oracle_bytes + row_oracle_bytes + stranded_bytes > ORACLE_BATCH_SEAL_BYTES { - cells.clear(); - self.oracle_cells = cells; + oracle.clear(); + self.oracle_cells = oracle; return Ok(Append::Full); } // Enforce hard frame cap even for first row @@ -2125,40 +2274,53 @@ impl TableEncoder { "row needs {row_oracle_bytes} oracle bytes, one request carries {MAX_REQUEST_BYTES}" ))); } - let mut cell_iter = cells.iter_mut(); - for (i, col) in mapping.columns.iter().enumerate() { + let mut oracle_iter = oracle.iter_mut(); + for (i, cell) in cells.iter().enumerate() { let col_plan = &self.plan.columns[i]; let buf = &mut self.buffers[i]; if let ColumnBuf::Oracle(o) = buf { - let cell = cell_iter.next().expect("one cell per oracle column"); + let cell = oracle_iter.next().expect("one cell per oracle column"); o.push(std::mem::replace(cell, OracleCell::Default)); continue; } - match value_of(col).map(|v| col_plan.non_finite.substitute(v)) { + match *cell { // Absent coerces like NULL, covers key-only delete // tombstones under non-FULL replica identity - None => buf.append_default(), - Some(v) => encode_value(buf, v, col_plan.decimal, col_plan.datetime64_scale) - .map_err(|e| name_column(e, &col.target_name))?, + Cell::Absent => { + buf.append_default(); + Ok(()) + } + #[cfg(test)] + Cell::Le(le) => buf.append_fixed_bytes(le), + Cell::Value(v) => encode_value( + buf, + col_plan.non_finite.substitute(v), + col_plan.decimal, + col_plan.datetime64_scale, + ), } + .map_err(|e| name_column(e, &col_plan.name))?; } - cells.clear(); - self.oracle_cells = cells; + oracle.clear(); + self.oracle_cells = oracle; self.oracle_bytes += row_oracle_bytes + stranded_bytes; // Synthetic columns: lsn, xid, commit_ts (unix micros), delete marker - let off = mapping.columns.len(); - push_fixed(&mut self.buffers[off], &decoded.source_lsn.to_le_bytes())?; - push_fixed(&mut self.buffers[off + 1], &decoded.xid.to_le_bytes())?; - let unix_us = committed.commit_ts.saturating_add(DATETIME64_PG_EPOCH_US); + let off = cells.len(); + push_fixed(&mut self.buffers[off], &stamp.source_lsn.to_le_bytes())?; + push_fixed(&mut self.buffers[off + 1], &stamp.xid.to_le_bytes())?; + let unix_us = stamp.commit_ts.saturating_add(DATETIME64_PG_EPOCH_US); push_fixed(&mut self.buffers[off + 2], &unix_us.to_le_bytes())?; let mut next = off + 3; if self.plan.synth_is_deleted.is_some() { - let is_deleted: u8 = (op_code == OP_DELETE).into(); + let is_deleted: u8 = (stamp.op_code == OP_DELETE).into(); push_fixed(&mut self.buffers[next], &is_deleted.to_le_bytes())?; next += 1; } if self.plan.synth_leaf.is_some() { - push_fixed(&mut self.buffers[next], &leaf.unwrap_or(0).to_le_bytes())?; + push_fixed( + &mut self.buffers[next], + &stamp.leaf.unwrap_or(0).to_le_bytes(), + )?; } self.rows += 1; self.approx_bytes = self.buffers.iter().map(ColumnBuf::approx_size).sum(); @@ -2166,6 +2328,26 @@ impl TableEncoder { } } +/// One mapped cell ready to encode +#[derive(Clone, Copy, Debug)] +pub(crate) enum Cell<'a> { + /// Absent from image, coerces like NULL + Absent, + Value(ValueRef<'a>), + /// Little-endian bytes [`Direct::Copy`] proved match column storage + #[cfg(test)] + Le(&'a [u8]), +} + +/// Per-row synthetic column inputs +pub(crate) struct RowStamp { + pub(crate) source_lsn: u64, + pub(crate) xid: u32, + pub(crate) commit_ts: i64, + pub(crate) op_code: i8, + pub(crate) leaf: Option, +} + fn name_column(mut e: EmitterError, target: &str) -> EmitterError { if let EmitterError::UnsupportedValue { ref mut target_column, @@ -2514,55 +2696,55 @@ fn micros_to_ticks(us: i128, scale: u32) -> Result { fn encode_value( buf: &mut ColumnBuf, - v: &ColumnValue, + v: ValueRef<'_>, decimal: Option, timestamp_scale: u32, ) -> Result<(), EmitterError> { match v { // Non-Nullable target takes type default - ColumnValue::Null => { + ValueRef::Null => { buf.append_default(); Ok(()) } - ColumnValue::Bool(b) => buf.append_fixed_bytes(&[*b as u8]), - ColumnValue::Char(c) => buf.append_fixed_bytes(&c.to_le_bytes()), - ColumnValue::Int2(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ColumnValue::Int4(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ColumnValue::Int8(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ColumnValue::Float4(f) => buf.append_fixed_bytes(&f.to_le_bytes()), - ColumnValue::Float8(f) => buf.append_fixed_bytes(&f.to_le_bytes()), - ColumnValue::Oid(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ColumnValue::Oid8(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ColumnValue::Date(i32::MIN | i32::MAX) - | ColumnValue::Timestamp(i64::MIN | i64::MAX) - | ColumnValue::TimestampTz(i64::MIN | i64::MAX) => Err(unsupported( + ValueRef::Bool(b) => buf.append_fixed_bytes(&[b as u8]), + ValueRef::Char(c) => buf.append_fixed_bytes(&c.to_le_bytes()), + ValueRef::Int2(n) => buf.append_fixed_bytes(&n.to_le_bytes()), + ValueRef::Int4(n) => buf.append_fixed_bytes(&n.to_le_bytes()), + ValueRef::Int8(n) => buf.append_fixed_bytes(&n.to_le_bytes()), + ValueRef::Float4(f) => buf.append_fixed_bytes(&f.to_le_bytes()), + ValueRef::Float8(f) => buf.append_fixed_bytes(&f.to_le_bytes()), + ValueRef::Oid(n) => buf.append_fixed_bytes(&n.to_le_bytes()), + ValueRef::Oid8(n) => buf.append_fixed_bytes(&n.to_le_bytes()), + ValueRef::Date(i32::MIN | i32::MAX) + | ValueRef::Timestamp(i64::MIN | i64::MAX) + | ValueRef::TimestampTz(i64::MIN | i64::MAX) => Err(unsupported( "temporal infinity, set pos_inf/neg_inf to null on a Nullable destination", )), - ColumnValue::Date(n) => { + ValueRef::Date(n) => { let days = n .checked_add(DATE32_PG_EPOCH_DAYS) .ok_or_else(|| unsupported("finite date overflows Date32 epoch conversion"))?; buf.append_fixed_bytes(&days.to_le_bytes()) } // PG time counts microseconds since midnight, without an epoch offset - ColumnValue::Time(n) => { - let ticks = micros_to_ticks(i128::from(*n), timestamp_scale)?; + ValueRef::Time(n) => { + let ticks = micros_to_ticks(i128::from(n), timestamp_scale)?; buf.append_fixed_bytes(&ticks.to_le_bytes()) } - ColumnValue::Timestamp(n) | ColumnValue::TimestampTz(n) => { - let ticks = timestamp_ticks(*n, timestamp_scale)?; + ValueRef::Timestamp(n) | ValueRef::TimestampTz(n) => { + let ticks = timestamp_ticks(n, timestamp_scale)?; buf.append_fixed_bytes(&ticks.to_le_bytes()) } // `timetz` → text: CH has no zone-aware time type, text keeps // the offset the old fixed encoding dropped - ColumnValue::TimeTz { micros, tz_seconds } => buf.append_string_bytes( - crate::decode::codecs::timetz_to_text(*micros, *tz_seconds).as_bytes(), + ValueRef::TimeTz { micros, tz_seconds } => buf.append_string_bytes( + crate::decode::codecs::timetz_to_text(micros, tz_seconds).as_bytes(), ), - ColumnValue::Uuid(b) => buf.append_fixed_bytes(&crate::decode::codecs::uuid_to_ch_wire(b)), - ColumnValue::Name(s) | ColumnValue::Text(s) | ColumnValue::Json(s) => { + ValueRef::Uuid(b) => buf.append_fixed_bytes(&crate::decode::codecs::uuid_to_ch_wire(&b)), + ValueRef::Name(s) | ValueRef::Text(s) | ValueRef::Json(s) => { buf.append_string_bytes(s.as_bytes()) } - ColumnValue::Numeric(n) => { + ValueRef::Numeric(n) => { use crate::decode::codecs::NumericKind; match decimal { // Decimal column: non-finite (NaN/±Inf) is unrepresentable, @@ -2581,31 +2763,31 @@ fn encode_value( None => buf.append_string_bytes(n.as_text().as_bytes()), } } - ColumnValue::Inet(v) => buf.append_string_bytes(v.to_text().as_bytes()), - ColumnValue::MacAddr(b) => { - buf.append_string_bytes(crate::decode::codecs::macaddr_to_text(b).as_bytes()) + ValueRef::Inet(v) => buf.append_string_bytes(v.to_text().as_bytes()), + ValueRef::MacAddr(b) => { + buf.append_string_bytes(crate::decode::codecs::macaddr_to_text(&b).as_bytes()) } - ColumnValue::MacAddr8(b) => { - buf.append_string_bytes(crate::decode::codecs::macaddr8_to_text(b).as_bytes()) + ValueRef::MacAddr8(b) => { + buf.append_string_bytes(crate::decode::codecs::macaddr8_to_text(&b).as_bytes()) } - ColumnValue::Tid { block, offset } => { - buf.append_string_bytes(crate::decode::codecs::tid_to_text(*block, *offset).as_bytes()) + ValueRef::Tid { block, offset } => { + buf.append_string_bytes(crate::decode::codecs::tid_to_text(block, offset).as_bytes()) } - ColumnValue::Interval(v) => buf.append_string_bytes(v.to_text().as_bytes()), - ColumnValue::Bytea(b) => buf.append_string_bytes(b), - ColumnValue::ExternalToast(_) => Err(unsupported( + ValueRef::Interval(v) => buf.append_string_bytes(v.to_text().as_bytes()), + ValueRef::Bytea(b) => buf.append_string_bytes(b), + ValueRef::ExternalToast(_) => Err(unsupported( "unresolved TOAST pointer (xact buffer should have reassembled)", )), - ColumnValue::PgPendingText { text, .. } => buf.append_string_bytes(text.as_bytes()), + ValueRef::PgPendingText { text, .. } => buf.append_string_bytes(text.as_bytes()), // Never interpret raw Datum bytes as local String data - ColumnValue::PgPending { .. } | ColumnValue::Unsupported { .. } => Err(unsupported( + ValueRef::PgPending { .. } | ValueRef::Unsupported { .. } => Err(unsupported( "unresolved source value routed to a local column", )), } } /// Map NULL, absent, and unlogged fields to default cells -fn oracle_cell(v: Option<&ColumnValue>, source_type_oid: u32) -> Result { +fn oracle_cell(cell: Cell<'_>, source_type_oid: u32) -> Result { let mismatch = |got: u32| EmitterError::UnsupportedValue { target_column: String::new(), kind: if got == 0 { @@ -2614,28 +2796,31 @@ fn oracle_cell(v: Option<&ColumnValue>, source_type_oid: u32) -> Result return Ok(OracleCell::Default), + Cell::Value(v) => v, + #[cfg(test)] + Cell::Le(_) => return Err(mismatch(0)), + }; Ok(match v { - None | Some(ColumnValue::Null) => OracleCell::Default, - Some(ColumnValue::PgPending { type_oid, raw }) - | Some(ColumnValue::Unsupported { type_oid, raw }) => { - if *type_oid != source_type_oid { - return Err(mismatch(*type_oid)); + ValueRef::Null => OracleCell::Default, + ValueRef::PgPending { type_oid, raw } | ValueRef::Unsupported { type_oid, raw } => { + if type_oid != source_type_oid { + return Err(mismatch(type_oid)); } - OracleCell::DiskRaw(raw.clone()) + OracleCell::DiskRaw(raw.to_vec()) } - Some(ColumnValue::PgPendingText { type_oid, text }) => { - if *type_oid != source_type_oid { - return Err(mismatch(*type_oid)); + ValueRef::PgPendingText { type_oid, text } => { + if type_oid != source_type_oid { + return Err(mismatch(type_oid)); } OracleCell::TextInput(text.as_bytes().to_vec()) } // jsonb needs typinput to reconstruct binary storage from document text - Some(ColumnValue::Json(s)) => OracleCell::TextInput(s.as_bytes().to_vec()), + ValueRef::Json(s) => OracleCell::TextInput(s.as_bytes().to_vec()), // PostGIS WKT must bypass HEXEWKB typoutput - Some(ColumnValue::Text(s)) | Some(ColumnValue::Name(s)) => { - OracleCell::Literal(s.as_bytes().to_vec()) - } - Some(_) => return Err(mismatch(0)), + ValueRef::Text(s) | ValueRef::Name(s) => OracleCell::Literal(s.as_bytes().to_vec()), + _ => return Err(mismatch(0)), }) } @@ -2988,6 +3173,15 @@ pub async fn load_effective( #[cfg(test)] mod tests { use super::*; + + fn encode_value( + buf: &mut ColumnBuf, + v: &ColumnValue, + decimal: Option, + timestamp_scale: u32, + ) -> Result<(), EmitterError> { + super::encode_value(buf, v.view(), decimal, timestamp_scale) + } use crate::backfill::copy_backfill::COPY_CHUNK_BLOCKS; use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple}; use backon::BackoffBuilder; @@ -3708,7 +3902,7 @@ mod tests { #[test] fn oracle_cell_rejects_values_off_the_planned_type() { - let rejects = |v: ColumnValue, want: &str| matches!(oracle_cell(Some(&v), 1007), Err(EmitterError::UnsupportedValue { kind, .. }) if kind == want); + let rejects = |v: ColumnValue, want: &str| matches!(oracle_cell(Cell::Value(v.view()), 1007), Err(EmitterError::UnsupportedValue { kind, .. }) if kind == want); let raw = vec![0u8; 4]; let moved = "source type oid differs from the planned one"; assert!(rejects(ColumnValue::PgPending { type_oid: 23, raw }, moved)); @@ -3726,7 +3920,7 @@ mod tests { text: "{1}".into(), }; assert!( - matches!(oracle_cell(Some(&text), 1007), Ok(OracleCell::TextInput(t)) if t == b"{1}") + matches!(oracle_cell(Cell::Value(text.view()), 1007), Ok(OracleCell::TextInput(t)) if t == b"{1}") ); } @@ -4000,7 +4194,7 @@ mod tests { ColumnValue::Timestamp(i64::MIN + 1), ColumnValue::TimestampTz(i64::MAX - 1), ] { - assert_eq!(nf.substitute(&value), &value); + assert_eq!(nf.substitute(value.view()), value.view()); } } } diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs new file mode 100644 index 00000000..e3980107 --- /dev/null +++ b/src/emit/direct_parity.rs @@ -0,0 +1,833 @@ +//! Direct tuple-view encoding must write the same blocks, and fail with the +//! same errors, as decoding owned cells first + +use clickhouse_c::Allocator; +use walrus::pg::walparser::RelFileNode; + +use crate::column_rules::{ColumnRule, ColumnRules, ColumnRulesBuilder, Substitute}; +use crate::decode::heap_decoder::{ + ColumnValue, CommittedTuple, DecodedHeap, HeapOp, decode_block_data, view_block_data, +}; +use crate::emit::ch_emitter::{ + ColumnBuf, Direct, OP_DELETE, OP_INSERT, RowStamp, TableEncoder, TablePlan, +}; +use crate::mapping::{ColumnMapping, SystemColumns, TableMapping, TableTarget}; +use crate::schema::*; +use crate::table_rules::MatchKind; + +/// One source column, its target, and stored datums to try, `None` for NULL +struct Case { + oid: u32, + len: i16, + align: char, + type_name: &'static str, + target: &'static str, + datums: Vec>>, +} + +fn case(oid: u32, len: i16, align: char, target: &'static str, datums: Vec>) -> Case { + let mut datums: Vec<_> = datums.into_iter().map(Some).collect(); + datums.push(None); + Case { + oid, + len, + align, + type_name: "", + target, + datums, + } +} + +fn attr(attnum: i16, oid: u32, len: i16, align: char, type_name: &str) -> RelAttr { + RelAttr { + attnum, + name: format!("c{attnum}"), + type_oid: oid, + typmod: -1, + not_null: false, + dropped: false, + type_name: type_name.into(), + type_byval: len > 0 && len <= 8, + type_len: len, + type_align: align, + type_storage: if len == -1 { 'x' } else { 'p' }, + missing_default: None, + } +} + +fn rel(attributes: Vec) -> RelDescriptor { + RelDescriptor { + rfn: RelFileNode { + spc_node: 1663, + db_node: 5, + rel_node: 16385, + }, + oid: 16385, + toast_oid: 0, + namespace_oid: 2200, + rel_name: RelName::new("public", "t"), + kind: 'r', + persistence: 'p', + ancestors: Vec::new(), + replident: ReplIdent::Full { pk_attnums: None }, + attributes, + } +} + +fn short(content: &[u8]) -> Vec { + let mut v = vec![(((1 + content.len()) as u8) << 1) | 1]; + v.extend_from_slice(content); + v +} + +fn long(content: &[u8]) -> Vec { + let mut v = (((4 + content.len()) as u32) << 2).to_le_bytes().to_vec(); + v.extend_from_slice(content); + v +} + +fn lz4(content: &[u8]) -> Vec { + let body = lz4::block::compress(content, None, false).unwrap(); + let mut v = ((((8 + body.len()) as u32) << 2) | 0b10) + .to_le_bytes() + .to_vec(); + v.extend_from_slice(&(content.len() as u32 | (1 << 30)).to_le_bytes()); + v.extend_from_slice(&body); + v +} + +fn external() -> Vec { + let mut v = vec![0x01, 18]; + for word in [100i32 as u32, 96, 7, 16400] { + v.extend_from_slice(&word.to_le_bytes()); + } + v +} + +/// Datums as `heap_fill_tuple` lays them out after a WAL `xl_heap_header`, +/// first `natts` attributes stored and later ones left to fast defaults +fn tuple(attrs: &[RelAttr], natts: usize, values: &[Option<&[u8]>]) -> Vec { + let has_null = values[..natts].iter().any(Option::is_none); + let bitmap_len = if has_null { natts.div_ceil(8) } else { 0 }; + let hoff = (23 + bitmap_len).next_multiple_of(8); + let mut bitmap = vec![0u8; bitmap_len]; + let mut data = Vec::new(); + for (i, (att, v)) in attrs.iter().zip(values).take(natts).enumerate() { + let Some(v) = v else { continue }; + if has_null { + bitmap[i / 8] |= 1 << (i % 8); + } + // Short and external varlenas skip alignment + if !(att.type_len == -1 && v[0] & 1 == 1) { + let by = match att.type_align { + 's' => 2, + 'i' => 4, + 'd' => 8, + _ => 1, + }; + data.resize(data.len().next_multiple_of(by), 0); + } + data.extend_from_slice(v); + } + let mut out = (natts as u16).to_le_bytes().to_vec(); + out.extend_from_slice(&u16::from(has_null).to_le_bytes()); + out.push(hoff as u8); + out.extend_from_slice(&bitmap); + out.resize(5 + hoff - 23, 0); + out.extend_from_slice(&data); + out +} + +fn mapping_of(rel: &RelDescriptor, targets: &[&str]) -> TableMapping { + TableMapping { + target: TableTarget::new("default", "t"), + columns: rel + .attributes + .iter() + .zip(targets) + .filter(|(a, _)| !a.dropped) + .map(|(a, t)| ColumnMapping { + src_attnum: a.attnum, + target_name: a.name.clone(), + target_type: (*t).into(), + type_pinned: false, + }) + .collect(), + tees: Vec::new(), + } +} + +fn plan(rel: &RelDescriptor, targets: &[&str], rules: &ColumnRules) -> TablePlan { + TablePlan::build( + Allocator::global(&mimalloc::MiMalloc), + rel, + &mapping_of(rel, targets), + rules, + &SystemColumns::default(), + ) + .unwrap() +} + +/// Encode every row both ways; blocks or first errors must match exactly +fn assert_parity( + rel: &RelDescriptor, + targets: &[&str], + rules: &ColumnRules, + rows: &[(usize, Vec>)], +) { + let mapping = mapping_of(rel, targets); + let mut cells = TableEncoder::new(plan(rel, targets, rules)).unwrap(); + let mut direct = TableEncoder::new(plan(rel, targets, rules)).unwrap(); + for (i, (natts, values)) in rows.iter().enumerate() { + let data = tuple(&rel.attributes, *natts, values); + let op = if i % 3 == 2 { OP_DELETE } else { OP_INSERT }; + let stamp = RowStamp { + source_lsn: 0x1000 + i as u64, + xid: 700 + i as u32, + commit_ts: -86_400_000_000 * i as i64, + op_code: op, + leaf: None, + }; + let mut decoded = decode_block_data(&data, rel).unwrap(); + // Placement renders PostGIS cells before batching + crate::ops::oracle::render_ext_columns(&rel.attributes, &mut decoded.columns); + let committed = CommittedTuple { + decoded: DecodedHeap { + rfn: rel.rfn, + xid: stamp.xid, + source_lsn: stamp.source_lsn, + op: if op == OP_DELETE { + HeapOp::Delete + } else { + HeapOp::Insert + }, + new: (op != OP_DELETE).then(|| decoded.clone()), + old: (op == OP_DELETE).then_some(decoded), + }, + commit_ts: stamp.commit_ts, + commit_lsn: 0, + }; + let view = view_block_data(&data, rel).unwrap(); + assert_eq!(view.to_tuple(rel).columns.len(), rel.attributes.len()); + let a = cells + .append_routed(&committed, &mapping, op, None) + .map_err(|e| e.to_string()); + let b = direct + .append_view(rel, &view.datums, &stamp) + .map_err(|e| e.to_string()); + assert_eq!(a, b, "row {i}: {values:?}"); + if a.is_err() { + return; + } + } + let (a, na) = cells.take_block().unwrap(); + let (b, nb) = direct.take_block().unwrap(); + assert_eq!(na, nb); + let wire = |bufs: &[ColumnBuf]| { + bufs.iter() + .map(|b| format!("{:?}", b.wire())) + .collect::>() + }; + assert_eq!(wire(&a), wire(&b)); +} + +/// Each case in its own column between an int4 and a text column, so +/// alignment of neighbours runs through both walkers +fn run(c: &Case, rules: &ColumnRules) { + let rel = rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, c.oid, c.len, c.align, c.type_name), + attr(3, TEXTOID, -1, 'i', "text"), + ]); + let targets = ["Int32", c.target, "Nullable(String)"]; + let id = 7i32.to_le_bytes(); + let tail = short(b"tail"); + let rows: Vec<_> = c + .datums + .iter() + .map(|d| (3, vec![Some(&id[..]), d.as_deref(), Some(&tail[..])])) + .collect(); + assert_parity(&rel, &targets, rules, &rows); +} + +fn ints(vals: impl IntoIterator) -> Vec> { + vals.into_iter().map(|b| b.to_vec()).collect() +} + +fn numeric_short(neg: bool, dscale: u16, digits: &[i16]) -> Vec { + let header = 0x8000 | if neg { 0x2000 } else { 0 } | ((dscale << 7) & 0x1F80); + let mut v = header.to_le_bytes().to_vec(); + for d in digits { + v.extend_from_slice(&d.to_le_bytes()); + } + v +} + +fn name(s: &[u8]) -> Vec { + let mut v = s.to_vec(); + v.resize(64, 0); + v +} + +fn jsonb_scalar(s: &[u8]) -> Vec { + let mut v = 0x5000_0001u32.to_le_bytes().to_vec(); + v.extend_from_slice(&(s.len() as u32).to_le_bytes()); + v.extend_from_slice(s); + v +} + +fn rules(entries: &[(&str, ColumnRule)]) -> ColumnRules { + let mut b = ColumnRulesBuilder::new(); + for (attname, rule) in entries { + b.add( + &RelName::new("public", "t"), + MatchKind::Exact, + attname, + MatchKind::Exact, + rule.clone(), + ); + } + b.finish().0 +} + +fn target_type(ty: &str) -> ColumnRules { + rules(&[( + "c2", + ColumnRule { + target_type: Some(ty.into()), + ..ColumnRule::default() + }, + )]) +} + +#[test] +fn fixed_width_types_match() { + let f4 = [ + f32::from_bits(0x7fc0_0001), + -0.0, + f32::INFINITY, + f32::MIN_POSITIVE, + ]; + let f8 = [ + f64::from_bits(0x7ff8_0000_0000_0001), + -0.0, + f64::NEG_INFINITY, + 1.5, + ]; + let cases = [ + case(BOOLOID, 1, 'c', "Bool", vec![vec![0], vec![1]]), + case(CHAROID, 1, 'c', "Int8", vec![vec![0x80], vec![65]]), + case( + INT2OID, + 2, + 's', + "Int16", + ints([i16::MIN, -1, 0, i16::MAX].map(i16::to_le_bytes)), + ), + case( + INT4OID, + 4, + 'i', + "Nullable(Int32)", + ints([i32::MIN, -1, 0, i32::MAX].map(i32::to_le_bytes)), + ), + case( + INT8OID, + 8, + 'd', + "Int64", + ints([i64::MIN, -1, 0, i64::MAX].map(i64::to_le_bytes)), + ), + case( + MONEYOID, + 8, + 'd', + "Int64", + ints([-12_345i64, 0, 99].map(i64::to_le_bytes)), + ), + case(FLOAT4OID, 4, 'i', "Float32", ints(f4.map(f32::to_le_bytes))), + case( + FLOAT8OID, + 8, + 'd', + "Nullable(Float64)", + ints(f8.map(f64::to_le_bytes)), + ), + case( + OIDOID, + 4, + 'i', + "UInt32", + ints([0u32, u32::MAX].map(u32::to_le_bytes)), + ), + case(XIDOID, 4, 'i', "UInt32", ints([3u32].map(u32::to_le_bytes))), + case(CIDOID, 4, 'i', "UInt32", ints([9u32].map(u32::to_le_bytes))), + case( + OID8OID, + 8, + 'd', + "UInt64", + ints([u64::MAX].map(u64::to_le_bytes)), + ), + case( + XID8OID, + 8, + 'd', + "UInt64", + ints([1u64 << 40].map(u64::to_le_bytes)), + ), + case( + PG_LSNOID, + 8, + 'd', + "UInt64", + ints([0x16_B374_D848u64].map(u64::to_le_bytes)), + ), + case( + DATEOID, + 4, + 'i', + "Date32", + ints([-1i32, 0, 8_000].map(i32::to_le_bytes)), + ), + case( + TIMEOID, + 8, + 'd', + "Time64(6)", + ints([45_296_000_000i64, 0].map(i64::to_le_bytes)), + ), + case( + TIMESTAMPOID, + 8, + 'd', + "DateTime64(6, 'UTC')", + ints([-1i64, 0, 1_700_000_000_000_000].map(i64::to_le_bytes)), + ), + case( + TIMESTAMPTZOID, + 8, + 'd', + "Nullable(DateTime64(3, 'UTC'))", + ints([-1_999i64, 1_999].map(i64::to_le_bytes)), + ), + case( + TIMETZOID, + 12, + 'd', + "String", + vec![ + [ + 45_296_000_000i64.to_le_bytes().as_slice(), + &(-7200i32).to_le_bytes(), + ] + .concat(), + ], + ), + case( + UUIDOID, + 16, + 'c', + "UUID", + vec![(0u8..16).collect(), vec![0xff; 16]], + ), + case( + INTERVALOID, + 16, + 'd', + "String", + vec![ + [ + 1_000_000i64.to_le_bytes().as_slice(), + &3i32.to_le_bytes(), + &14i32.to_le_bytes(), + ] + .concat(), + ], + ), + case( + NAMEOID, + 64, + 'c', + "String", + vec![name(b"abc"), name(b""), name(b"\xc3\xa9t\xc3\xa9")], + ), + case( + MACADDROID, + 6, + 'i', + "String", + vec![vec![0x08, 0, 0x2b, 1, 2, 0xab]], + ), + case(MACADDR8OID, 8, 'i', "String", vec![vec![0xfe; 8]]), + case(TIDOID, 6, 's', "String", vec![vec![1, 0, 2, 0, 3, 0]]), + ]; + for c in &cases { + run(c, &ColumnRules::default()); + } +} + +#[test] +fn copy_requires_matching_storage() { + let cases = [ + (INT4OID, 4, 'i', "Int32", Direct::Copy), + (INT4OID, 4, 'i', "UInt32", Direct::Value), + (INT4OID, 4, 'i', "Float32", Direct::Value), + (OIDOID, 4, 'i', "Int32", Direct::Value), + (FLOAT8OID, 8, 'd', "Int64", Direct::Value), + (INT8OID, 8, 'd', "Nullable(Int64)", Direct::Copy), + (MONEYOID, 8, 'd', "Int64", Direct::Copy), + (DATEOID, 4, 'i', "Date32", Direct::Value), + (BOOLOID, 1, 'c', "Bool", Direct::Value), + ]; + for (oid, len, align, target, want) in cases { + let rel = rel(vec![attr(1, oid, len, align, "")]); + let plan = plan(&rel, &[target], &ColumnRules::default()); + assert_eq!( + plan.columns[0].source.unwrap().direct, + want, + "{oid} {target}" + ); + } + let rel = rel(vec![attr(1, 999_999, -1, 'd', "geometry")]); + let plan = plan(&rel, &["String"], &ColumnRules::default()); + assert_eq!(plan.columns[0].source.unwrap().direct, Direct::Wkt); +} + +#[test] +fn varlena_types_match() { + let long_text = vec![b'x'; 300]; + let inet4 = [2u8, 24, 0, 4, 10, 1, 2, 0]; + let cases = [ + case( + TEXTOID, + -1, + 'i', + "String", + vec![ + short(b""), + short("héllo".as_bytes()), + long(&long_text), + lz4(&long_text), + short(b"\xff\xfe"), + ], + ), + case( + VARCHAROID, + -1, + 'i', + "Nullable(String)", + vec![long(b"abc"), lz4(&long_text)], + ), + case(BPCHAROID, -1, 'i', "String", vec![short(b"ab ")]), + case( + BYTEAOID, + -1, + 'i', + "String", + vec![short(&[0, 1, 0xff]), long(&[]), lz4(&long_text)], + ), + case(JSONOID, -1, 'i', "String", vec![short(br#"{"a": 1}"#)]), + case( + JSONBOID, + -1, + 'i', + "String", + vec![short(&jsonb_scalar(b"abc")), long(&jsonb_scalar(b""))], + ), + case( + NUMERICOID, + -1, + 'i', + "String", + vec![ + short(&numeric_short(false, 2, &[1, 5000])), + short(&numeric_short(true, 0, &[42])), + short(&0xC000u16.to_le_bytes()), + short(&0xD000u16.to_le_bytes()), + ], + ), + case( + NUMERICOID, + -1, + 'i', + "Decimal(10, 2)", + vec![ + short(&numeric_short(false, 2, &[1, 5000])), + short(&numeric_short(true, 0, &[42])), + ], + ), + case( + INETOID, + -1, + 'i', + "String", + vec![short(&inet4), short(&[9, 9])], + ), + case( + CIDROID, + -1, + 'i', + "String", + vec![short(&[2, 8, 1, 4, 10, 0, 0, 0])], + ), + ]; + for c in &cases { + run(c, &ColumnRules::default()); + } +} + +#[test] +fn errors_match() { + let cases = [ + // Unresolved pointer reaches encoder in both paths + case(TEXTOID, -1, 'i', "String", vec![external()]), + // Non-finite numeric into Decimal rejects + case( + NUMERICOID, + -1, + 'i', + "Decimal(10, 2)", + vec![short(&0xC000u16.to_le_bytes())], + ), + // Temporal infinity rejects without substitute + case( + DATEOID, + 4, + 'i', + "Date32", + vec![i32::MAX.to_le_bytes().to_vec()], + ), + // Invalid name bytes stay unsupported + case(NAMEOID, 64, 'c', "String", vec![name(b"\xff")]), + ]; + for c in &cases { + run(c, &ColumnRules::default()); + } +} + +#[test] +fn overrides_and_substitutes_match() { + let ints4 = ints([i32::MIN, -1, 0, i32::MAX].map(i32::to_le_bytes)); + run( + &case(INT4OID, 4, 'i', "Int32", ints4.clone()), + &target_type("UInt32"), + ); + run( + &case(INT4OID, 4, 'i', "Int32", ints4), + &target_type("Nullable(Int32)"), + ); + run( + &case( + NUMERICOID, + -1, + 'i', + "Decimal(10, 2)", + vec![short(&numeric_short(false, 0, &[7]))], + ), + &target_type("Int64"), + ); + let inf = ColumnRule { + nan: Some(Substitute::Literal("0".into())), + pos_inf: Some(Substitute::Max), + neg_inf: Some(Substitute::Null), + ..ColumnRule::default() + }; + run( + &case( + NUMERICOID, + -1, + 'i', + "Nullable(Decimal(10, 2))", + vec![ + short(&0xC000u16.to_le_bytes()), + short(&0xD000u16.to_le_bytes()), + short(&0xF000u16.to_le_bytes()), + short(&numeric_short(false, 2, &[1, 5000])), + ], + ), + &rules(&[("c2", inf)]), + ); + let temporal = ColumnRule { + pos_inf: Some(Substitute::Null), + neg_inf: Some(Substitute::Null), + ..ColumnRule::default() + }; + run( + &case( + TIMESTAMPTZOID, + 8, + 'd', + "Nullable(DateTime64(6, 'UTC'))", + ints([i64::MAX, i64::MIN, 5].map(i64::to_le_bytes)), + ), + &rules(&[("c2", temporal)]), + ); +} + +#[test] +fn oracle_columns_match() { + // Outside local matrix: raw body reaches oracle request unchanged + run( + &case( + 3614, + -1, + 'i', + "String", + vec![short(b"\x01\x02"), lz4(&[7; 300])], + ), + &ColumnRules::default(), + ); + // Composite target routes even local types to oracle + run( + &case(TEXTOID, -1, 'i', "Array(String)", vec![short(b"{a,b}")]), + &ColumnRules::default(), + ); + run( + &case( + JSONBOID, + -1, + 'i', + "Map(String, String)", + vec![short(&jsonb_scalar(b"x"))], + ), + &ColumnRules::default(), + ); + let mut point = vec![0, 0, 0, 0]; + point.extend_from_slice(&1u32.to_le_bytes()); + point.extend_from_slice(&[0; 8]); + point.extend_from_slice(&1.5f64.to_le_bytes()); + point.extend_from_slice(&(-2.0f64).to_le_bytes()); + let geometry = Case { + type_name: "geometry", + ..case( + 999_999, + -1, + 'd', + "String", + vec![long(&point), short(&[0, 0, 0, 0, 3, 0, 0, 0])], + ) + }; + run(&geometry, &ColumnRules::default()); +} + +#[test] +fn defaults_dropped_and_absent_match() { + let mut attrs = vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, TEXTOID, -1, 'i', "text"), + attr(3, INT8OID, 8, 'd', "int8"), + attr(4, TEXTOID, -1, 'i', "text"), + attr(5, NUMERICOID, -1, 'i', "numeric"), + attr(6, 3614, -1, 'i', "tsvector"), + ]; + attrs[1].dropped = true; + attrs[2].missing_default = Some(MissingDefault::Text("-5".into())); + attrs[3].missing_default = Some(MissingDefault::Raw( + crate::decode::heap_decoder::raw_missing_array(TEXTOID, &short(b"dflt")), + )); + attrs[4].missing_default = Some(MissingDefault::Text("NaN".into())); + attrs[5].missing_default = Some(MissingDefault::Text("'a'".into())); + let rel = rel(attrs); + let targets = [ + "Int32", + "String", + "Int64", + "String", + "Nullable(String)", + "String", + ]; + let one = 1i32.to_le_bytes(); + let gone = short(b"dropped"); + let big = 9i64.to_le_bytes(); + let rows = vec![ + ( + 2, + vec![Some(&one[..]), Some(&gone[..]), None, None, None, None], + ), + (1, vec![Some(&one[..]), None, None, None, None, None]), + ( + 3, + vec![Some(&one[..]), None, Some(&big[..]), None, None, None], + ), + (6, vec![None; 6]), + ]; + assert_parity(&rel, &targets, &ColumnRules::default(), &rows); +} + +#[test] +fn predeclared_columns_match() { + // Mapping names attnum 2, descriptor predates it + let rel = rel(vec![attr(1, INT4OID, 4, 'i', "int4")]); + let mapping_rel = self::rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, TEXTOID, -1, 'i', "text"), + ]); + let targets = ["Int32", "Nullable(String)"]; + let mapping = mapping_of(&mapping_rel, &targets); + let build = || { + TableEncoder::new( + TablePlan::build( + Allocator::global(&mimalloc::MiMalloc), + &rel, + &mapping, + &ColumnRules::default(), + &SystemColumns::default(), + ) + .unwrap(), + ) + .unwrap() + }; + let (mut cells, mut direct) = (build(), build()); + let one = 1i32.to_le_bytes(); + let data = tuple(&rel.attributes, 1, &[Some(&one[..])]); + let decoded = decode_block_data(&data, &rel).unwrap(); + let stamp = RowStamp { + source_lsn: 1, + xid: 2, + commit_ts: 3, + op_code: OP_INSERT, + leaf: None, + }; + let committed = CommittedTuple { + decoded: DecodedHeap { + rfn: rel.rfn, + xid: 2, + source_lsn: 1, + op: HeapOp::Insert, + new: Some(decoded), + old: None, + }, + commit_ts: 3, + commit_lsn: 0, + }; + cells + .append_routed(&committed, &mapping, OP_INSERT, None) + .unwrap(); + let view = view_block_data(&data, &rel).unwrap(); + direct.append_view(&rel, &view.datums, &stamp).unwrap(); + let wire = |e: &mut TableEncoder| { + let (bufs, _) = e.take_block().unwrap(); + bufs.iter() + .map(|b| format!("{:?}", b.wire())) + .collect::>() + }; + assert_eq!(wire(&mut cells), wire(&mut direct)); +} + +#[test] +fn view_matches_owned_decode_for_records() { + let rel = rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, TEXTOID, -1, 'i', "text"), + ]); + let one = 1i32.to_le_bytes(); + let text = long(b"value"); + let data = tuple(&rel.attributes, 2, &[Some(&one[..]), Some(&text[..])]); + let view = view_block_data(&data, &rel).unwrap(); + assert_eq!(view.to_tuple(&rel), decode_block_data(&data, &rel).unwrap()); + assert_eq!( + view.to_tuple(&rel).columns[1], + Some(ColumnValue::Text("value".into())) + ); +} diff --git a/src/emit/mod.rs b/src/emit/mod.rs index de952a3d..d6475e1f 100644 --- a/src/emit/mod.rs +++ b/src/emit/mod.rs @@ -1,4 +1,6 @@ pub mod ch_ddl; pub mod ch_emitter; +#[cfg(test)] +mod direct_parity; pub mod pipeline; pub mod route; diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index c7672eea..54bd4174 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -481,7 +481,7 @@ fn decode_block( pub fn render_ext_columns(attrs: &[RelAttr], columns: &mut [Option]) { // Scan cells first: attribute type names sit behind a pointer per column for (idx, cell) in columns.iter_mut().enumerate() { - let Some(ColumnValue::PgPending { raw, .. }) = cell.as_ref() else { + let Some(ColumnValue::PgPending { .. }) = cell.as_ref() else { continue; }; let is_geo = attrs.iter().any(|att| { @@ -489,12 +489,21 @@ pub fn render_ext_columns(attrs: &[RelAttr], columns: &mut [Option] && usize::try_from(att.attnum - 1) == Ok(idx) && matches!(att.type_name.as_str(), "geography" | "geometry") }); - if is_geo && let Some(text) = gserialized_point_to_wkt(raw) { - *cell = Some(ColumnValue::Text(text)); + if is_geo && let Some(v) = cell.as_mut() { + render_ext_value(v); } } } +/// Render one PostGIS cell, see [`render_ext_columns`] +pub(crate) fn render_ext_value(cell: &mut ColumnValue) { + if let ColumnValue::PgPending { raw, .. } = cell + && let Some(text) = gserialized_point_to_wkt(raw) + { + *cell = ColumnValue::Text(text); + } +} + /// PostGIS on-disk GSERIALIZED → `POINT(x y)` for 2-D points. Layout: /// `[srid(3) + gflags(1)][geomtype u32-le][…]`; POINT has geomtype 1 and its /// two `f64`-LE coordinates are the trailing 16 bytes. `None` for non-points. From 3ca3a29bb803313c5af017bea5c34b125468c629 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:41:41 +0000 Subject: [PATCH 04/20] Buffer ordinary heap records as WAL bytes until commit Covered user-heap records no longer decode into ColumnValue rows at record time. HeapInput keeps writer xid, op, LSN, tuple count, main_data and block 0 registered data in one allocation; images and other blocks drop since wal_level=logical keeps tuple data registered. DescribedInput pins the record-time descriptor version as DescribedHeap does and shares the spill descriptor dictionary under new TAG_INPUT frames (spill v10). Record time still walks each tuple without building cells, so malformed records fail and per-op decoder counts move there as before. Commit drain decodes inputs through the cell adapter into the existing pending-heap queue; unresolved catalog stash keeps its RawRecord path and verdicts. TOAST relations and runtime-config tables still decode at record time. Predecoded row graphs are gone. The dispatcher prescan borrows record bytes to start TOAST read-ahead for kept values and record toast births, and RecordSink::on_records drops its predecoded slice. xact_spill now feeds real insert records: 15.8k buffer allocations per 7812-row transaction against 23.6k for decoded rows; decode cost moves into drain until planning encodes directly. Co-Authored-By: Claude Opus 5.5 --- benches/xact_spill.rs | 68 +++-- src/bin/stream/session.rs | 4 +- src/bin/stream/sinks.rs | 8 +- src/decode/decoder_sink.rs | 10 +- src/decode/heap_decoder.rs | 53 ++-- src/emit/pipeline/planner.rs | 4 +- src/emit/pipeline/reorder.rs | 5 +- src/record.rs | 7 +- src/source/queueing_record_sink.rs | 74 ++--- src/xact/spill.rs | 279 ++++++++++++++++-- src/xact/xact_buffer.rs | 443 ++++++++++++++++++----------- 11 files changed, 659 insertions(+), 296 deletions(-) diff --git a/benches/xact_spill.rs b/benches/xact_spill.rs index 3d1568c9..7b990d96 100644 --- a/benches/xact_spill.rs +++ b/benches/xact_spill.rs @@ -6,9 +6,13 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use clap::Parser; -use walrus::pg::walparser::RelFileNode; -use walshadow::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; -use walshadow::schema::{RelDescriptor, RelName, ReplIdent}; +use walrus::pg::walparser::{ + BlockLocation, RelFileNode, RmId, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, + XLogRecordHeader, +}; +use walshadow::heap_decoder::ColumnValue; +use walshadow::schema::{RelAttr, RelDescriptor, RelName, ReplIdent, TEXTOID}; +use walshadow::spill::{DescribedInput, HeapInput}; use walshadow::xact_buffer::{StashResolved, XactBuffer, XactBufferConfig}; #[path = "common/alloc_meter.rs"] @@ -61,9 +65,49 @@ async fn run(args: Args) -> anyhow::Result<()> { persistence: 'p', ancestors: Vec::new(), replident: ReplIdent::Default { pk_attnums: None }, - attributes: Vec::new(), + attributes: vec![RelAttr { + attnum: 1, + name: "v".into(), + type_oid: TEXTOID, + typmod: -1, + not_null: false, + dropped: false, + type_name: "text".into(), + type_byval: false, + type_len: -1, + type_align: 'i', + type_storage: 'x', + missing_default: None, + }], }); let payload = "x".repeat(args.payload_bytes); + // XLOG_HEAP_INSERT block data: xl_heap_header, then one 4-byte-header text + let mut tuple = 1u16.to_le_bytes().to_vec(); + tuple.extend_from_slice(&0u16.to_le_bytes()); + tuple.push(24); + tuple.push(0); + tuple.extend_from_slice(&(((4 + payload.len()) as u32) << 2).to_le_bytes()); + tuple.extend_from_slice(payload.as_bytes()); + let record = |xid: u32| XLogRecord { + header: XLogRecordHeader { + xact_id: xid, + resource_manager_id: RmId::Heap as u8, + ..Default::default() + }, + blocks: vec![XLogRecordBlock { + header: XLogRecordBlockHeader { + location: BlockLocation { + rel: descriptor.rfn, + block_no: 0, + }, + ..Default::default() + }, + data: std::borrow::Cow::Borrowed(&tuple[..]), + ..Default::default() + }], + main_data: std::borrow::Cow::Borrowed(&[1, 0, 8][..]), + ..Default::default() + }; println!("{args:?}"); for repeat in 0..args.repeats { let dir = tempfile::tempdir_in(&args.dir)?; @@ -78,20 +122,12 @@ async fn run(args: Args) -> anyhow::Result<()> { for xid in 1..=args.transactions { let base = GLOBAL.reset_peak(); let start = Instant::now(); + let record = record(xid); for row in 0..args.rows { + let source_lsn = u64::from(xid) * (args.rows as u64 + 1) + row as u64; buffer - .on_heap(DescribedHeap { - decoded: DecodedHeap { - rfn: descriptor.rfn, - xid, - source_lsn: u64::from(xid) * (args.rows as u64 + 1) + row as u64, - op: HeapOp::Insert, - new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Text(payload.clone()))], - partial: false, - }), - old: None, - }, + .on_input(DescribedInput { + input: HeapInput::from_parsed(&record, source_lsn, 1), descriptor: descriptor.clone(), descriptor_valid_from: 1, }) diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index 814455b6..edc33543 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -1342,7 +1342,7 @@ async fn session_body( ) .await?; let mirror_resolver = reorder_sink.planning_resolver(); - let predecode = Arc::new(decoder.predecoder(reorder_sink.planning_resolver())); + let prescan = Arc::new(decoder.prescan(reorder_sink.planning_resolver())); let decoder_xact = QueueingRecordSink::spawn( DecoderXactPair { decoder, @@ -1352,7 +1352,7 @@ async fn session_body( decoder_queue_capacity, span_registry.clone(), ) - .with_predecode(predecode); + .with_prescan(prescan); let boundary_gate = CatalogBoundaryGate::new( shadow_state.clone(), BoundaryGateConfig { diff --git a/src/bin/stream/sinks.rs b/src/bin/stream/sinks.rs index 413623f9..e9184898 100644 --- a/src/bin/stream/sinks.rs +++ b/src/bin/stream/sinks.rs @@ -6,7 +6,6 @@ use anyhow::Result; use std::future::Future; use std::pin::Pin; use walshadow::boundary_hold::BoundaryHoldSink; -use walshadow::heap_decoder::Predecoded; use walshadow::record::{MetricsRecordSink, Record, RecordSink, SinkError}; use walshadow::xact_buffer::BufferingDecoderSink; @@ -30,7 +29,7 @@ impl RecordSink for DecoderXactPair { &'a mut self, record: &'a Record<'a>, ) -> Pin> + Send + 'a>> { - self.on_records(std::slice::from_ref(record), &mut []) + self.on_records(std::slice::from_ref(record)) } /// Absorbs every heap record before draining any commit in the batch: @@ -38,11 +37,10 @@ impl RecordSink for DecoderXactPair { fn on_records<'a>( &'a mut self, batch: &'a [Record<'a>], - pre: &'a mut [Option], ) -> Pin> + Send + 'a>> { Box::pin(async move { - self.decoder.on_records(batch, pre).await?; - self.xact_drain.on_records(batch, &mut []).await?; + self.decoder.on_records(batch).await?; + self.xact_drain.on_records(batch).await?; Ok(()) }) } diff --git a/src/decode/decoder_sink.rs b/src/decode/decoder_sink.rs index 321d09ac..7708a066 100644 --- a/src/decode/decoder_sink.rs +++ b/src/decode/decoder_sink.rs @@ -207,17 +207,21 @@ impl DecoderStats { /// Single source of truth for the `HeapOp → counter` mapping; all decoder /// dispatch sites route through here. pub fn record(&self, decoded: &crate::decode::heap_decoder::DecodedHeap) { + let partial = decoded.new.as_ref().is_some_and(|t| t.partial) + || decoded.old.as_ref().is_some_and(|t| t.partial); + self.record_op(decoded.op, partial); + } + + pub fn record_op(&self, op: crate::decode::heap_decoder::HeapOp, partial: bool) { use crate::decode::heap_decoder::HeapOp; self.decoded.fetch_add(1, Ordering::Relaxed); - match decoded.op { + match op { HeapOp::Insert => self.inserts.fetch_add(1, Ordering::Relaxed), HeapOp::Update => self.updates.fetch_add(1, Ordering::Relaxed), HeapOp::HotUpdate => self.hot_updates.fetch_add(1, Ordering::Relaxed), HeapOp::Delete => self.deletes.fetch_add(1, Ordering::Relaxed), HeapOp::Truncate => self.truncates.fetch_add(1, Ordering::Relaxed), }; - let partial = decoded.new.as_ref().is_some_and(|t| t.partial) - || decoded.old.as_ref().is_some_and(|t| t.partial); if partial { self.partial.fetch_add(1, Ordering::Relaxed); } diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index a96b5ff9..ca162814 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -65,7 +65,6 @@ use std::borrow::Cow; use std::string::FromUtf8Error; -use std::sync::Arc; use smallvec::{SmallVec, smallvec}; @@ -85,32 +84,6 @@ use crate::schema::{ /// only MULTI_INSERT with ntuples > 1 spills. pub type DecodedHeaps = SmallVec<[DecodedHeap; 1]>; -/// Tuples of one record decoded ahead of the serial worker against the -/// descriptor resolved then. Decoding is pure in (record, descriptor), so -/// they stand in for the worker's own decode when its lookup lands on the -/// same descriptor version -#[derive(Debug)] -pub struct Predecoded { - rel: Arc, - valid_from: u64, - set: DecodedHeaps, -} - -impl Predecoded { - pub fn new(rel: Arc, valid_from: u64, set: DecodedHeaps) -> Self { - Self { - rel, - valid_from, - set, - } - } - - /// Tuples, if decoded against `rel` valid from `valid_from` - pub fn take_if(self, rel: &Arc, valid_from: u64) -> Option { - (Arc::ptr_eq(&self.rel, rel) && self.valid_from == valid_from).then_some(self.set) - } -} - /// PG `SizeOfHeapHeader`. pub const SIZE_OF_HEAP_HEADER: usize = 5; /// PG `SizeofHeapTupleHeader`, stable at 23 since PG 7.x. @@ -520,6 +493,24 @@ impl TupleView<'_> { } } +/// Tuple walked for validity and shape only, building nothing +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TupleShape { + pub partial: bool, +} + +impl TupleBuild<'_> for TupleShape { + fn with_capacity(_: usize) -> Self { + Self::default() + } + + fn push(&mut self, _: &RelAttr, _: DatumRef<'_>) {} + + fn set_partial(&mut self) { + self.partial = true; + } +} + /// Images of one heap tuple change, before identity and position attach #[derive(Clone, Debug, PartialEq)] pub struct HeapImages { @@ -568,6 +559,14 @@ pub fn view_heap_record<'a>( parse_heap_record(record, rel) } +/// Validate `record` as [`decode_heap_record`] would, without building cells +pub fn scan_heap_record( + record: &XLogRecord<'_>, + rel: &RelDescriptor, +) -> Result; 1]>, DecodeError> { + parse_heap_record(record, rel) +} + fn parse_heap_record<'a, T: TupleBuild<'a>>( record: &'a XLogRecord<'_>, rel: &RelDescriptor, diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 57c4cf81..195d634e 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -892,7 +892,7 @@ mod tests { /// Value ids come from a tiny wrapping OID space, as in PostgreSQL only /// once every chunk of the previous value is vacuumed, truncated or /// rewritten away; rewrites assign ids no WAL birth shows. Shadow and - /// predecode run up to a few commits ahead of planning. Whatever the + /// prescan run up to a few commits ahead of planning. Whatever the /// sequence, lead and budget, a row using read-ahead carries the value /// its pointer names at that commit whenever a store read at planning /// does, and NULL at worst @@ -1050,7 +1050,7 @@ mod tests { events.push((None, h)); } for (born, h) in &events { - // Predecode reads ahead the values rows keep + // Prescan reads ahead the values rows keep if born.is_none() && let Some(t) = h.decoded.new.as_ref().or(h.decoded.old.as_ref()) && let Some(ColumnValue::ExternalToast(p)) = t.columns[0] diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index 3b3aa7b7..88c07b32 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -27,7 +27,7 @@ use crate::backfill::visibility_pending::{self, SharedPendingLedger}; use crate::catalog::pending::PendingCatalog; #[cfg(feature = "test-support")] use crate::crash_test; -use crate::decode::heap_decoder::{DecodedHeap, DescribedHeap, Predecoded}; +use crate::decode::heap_decoder::{DecodedHeap, DescribedHeap}; use crate::decode::visibility::{PgXactView, SealedPatch, read_pg_xact}; use crate::emit::ch_ddl::{DdlApplicator, Stamp}; use crate::emit::ch_emitter::EmitterStats; @@ -314,7 +314,7 @@ impl ReorderSink { }) } - /// Resolver shared with predecode for TOAST read-ahead + /// Resolver shared with prescan for TOAST read-ahead pub fn planning_resolver(&self) -> ToastResolver { self.resolver.clone() } @@ -1549,7 +1549,6 @@ impl RecordSink for ReorderSink { fn on_records<'a>( &'a mut self, batch: &'a [Record<'a>], - _pre: &'a mut [Option], ) -> Pin> + Send + 'a>> { Box::pin(async move { for record in batch.iter().filter(|r| Self::wants(r)) { diff --git a/src/record.rs b/src/record.rs index e152859b..55f61f6a 100644 --- a/src/record.rs +++ b/src/record.rs @@ -8,7 +8,6 @@ use thiserror::Error; use walrus::pg::wal::segment::SegmentName; use walrus::pg::walparser::{RmId, XLogRecord}; -use crate::decode::heap_decoder::Predecoded; use crate::source::timeline::TimelineHistory; pub const WAL_SEG_SIZE: u64 = walrus::pg::wal::segment::DEFAULT_WAL_SEG_SIZE; @@ -261,14 +260,10 @@ pub trait RecordSink: Send { true } - /// One boxed future per batch rather than per record; order preserved. - /// `pre` holds tuples a dispatcher decoded ahead, off this sink's task, - /// by batch index; empty when nothing was decoded ahead, and sinks - /// without a decode step drop it + /// One boxed future per batch rather than per record; order preserved fn on_records<'a>( &'a mut self, batch: &'a [Record<'a>], - _pre: &'a mut [Option], ) -> Pin> + Send + 'a>> { Box::pin(async move { for record in batch { diff --git a/src/source/queueing_record_sink.rs b/src/source/queueing_record_sink.rs index 0d9c0a5e..ec0d1144 100644 --- a/src/source/queueing_record_sink.rs +++ b/src/source/queueing_record_sink.rs @@ -32,7 +32,6 @@ use tokio::task::JoinHandle; use tracing::Instrument; -use crate::decode::heap_decoder::Predecoded; use crate::ops::trace::TxnSpanRegistry; use crate::record::{Done, Record, RecordSink, SinkError}; @@ -58,14 +57,14 @@ pub const DEFAULT_QUEUEING_IDLE_INTERVAL: Duration = Duration::from_millis(50); /// unstealable LIFO slot const YIELD_INTERVAL: Duration = Duration::from_millis(1); -/// Pure per-record decode run ahead of the worker; see -/// [`QueueingRecordSink::with_predecode`] -pub type Predecode = Arc) -> Option + Send + Sync>; +/// Per-record scan run ahead of the worker; see +/// [`QueueingRecordSink::with_prescan`] +pub type Prescan = Arc) + Send + Sync>; -/// Batch records, or the blocking-pool job decoding them ahead +/// Batch records, or the blocking-pool job scanning them ahead enum Records { Ready(Vec>), - Decoding(JoinHandle<(Vec>, Vec>)>), + Scanning(JoinHandle>>), } /// One channel message: wanted records plus the position of those the inner @@ -90,7 +89,7 @@ pub struct QueueingRecordSink { skipped_lsn: u64, /// Inner sink's [`RecordSink::wants`], evaluated before the owned clone wants: fn(&Record<'_>) -> bool, - predecode: Option, + prescan: Option, batch_size: usize, err: Arc>>, in_flight: Arc, @@ -178,12 +177,12 @@ impl QueueingRecordSink { skipped, skipped_lsn, })) => { - let (batch, mut pre) = match records { - Records::Ready(batch) => (batch, Vec::new()), - Records::Decoding(job) => match job.await { - Ok(decoded) => decoded, + let batch = match records { + Records::Ready(batch) => batch, + Records::Scanning(job) => match job.await { + Ok(batch) => batch, Err(e) => { - let e = SinkError::Other(format!("predecode job: {e}")); + let e = SinkError::Other(format!("prescan job: {e}")); park_err_and_drain(e, n + skipped, &mut rx).await; break 'outer; } @@ -212,7 +211,7 @@ impl QueueingRecordSink { reg.note_popped(record.parsed.header.xact_id); } } - inner.on_records(&batch, &mut pre).await?; + inner.on_records(&batch).await?; // Advance idle ack past trailing non-commit WAL // (`max_lsn` = dispatched high-water). inner.on_idle_advance(max_lsn).await @@ -261,7 +260,7 @@ impl QueueingRecordSink { skipped: 0, skipped_lsn: 0, wants: S::wants, - predecode: None, + prescan: None, batch_size, err, in_flight, @@ -275,8 +274,8 @@ impl QueueingRecordSink { /// Decode each shipped batch on the blocking pool before the worker takes /// it, overlapping the worker's serial pass over earlier batches. The /// worker hands the results to [`RecordSink::on_records`] - pub fn with_predecode(mut self, predecode: Predecode) -> Self { - self.predecode = Some(predecode); + pub fn with_prescan(mut self, prescan: Prescan) -> Self { + self.prescan = Some(prescan); self } @@ -336,12 +335,12 @@ impl QueueingRecordSink { .tx .as_ref() .expect("close drops the sender only after its final flush"); - let records = match &self.predecode { - Some(predecode) if !batch.is_empty() => { - let predecode = predecode.clone(); - Records::Decoding(tokio::task::spawn_blocking(move || { - let pre = batch.iter().map(|r| predecode(r)).collect(); - (batch, pre) + let records = match &self.prescan { + Some(prescan) if !batch.is_empty() => { + let prescan = prescan.clone(); + Records::Scanning(tokio::task::spawn_blocking(move || { + batch.iter().for_each(|r| prescan(r)); + batch })) } _ => Records::Ready(batch), @@ -756,21 +755,21 @@ mod tests { } #[tokio::test] - async fn predecode_panic_surfaces() { + async fn prescan_panic_surfaces() { let mut q = QueueingRecordSink::spawn(Hooks(|_| Ok(())), 1, 4, None) - .with_predecode(Arc::new(|_| panic!("predecode bug"))); + .with_prescan(Arc::new(|_| panic!("prescan bug"))); let in_flight = q.in_flight.clone(); q.on_record(&synth(1)).await.expect("send"); wait_worker_dead(&q).await; assert_eq!(in_flight.load(Ordering::Relaxed), 0); - let err = q.close().await.expect_err("predecode error"); - assert!(matches!(err, SinkError::Other(s) if s.contains("predecode job"))); + let err = q.close().await.expect_err("prescan error"); + assert!(matches!(err, SinkError::Other(s) if s.contains("prescan job"))); } #[tokio::test] - async fn predecode_results_reach_inner_by_index() { - struct PreLens(Arc>>); - impl RecordSink for PreLens { + async fn prescan_runs_before_inner_batches() { + struct Lens(Arc>>, Arc); + impl RecordSink for Lens { fn on_record<'a>( &'a mut self, _r: &'a Record<'a>, @@ -781,9 +780,9 @@ mod tests { fn on_records<'a>( &'a mut self, batch: &'a [Record<'a>], - pre: &'a mut [Option], ) -> Pin> + Send + 'a>> { - self.0.lock().unwrap().push((batch.len(), pre.len())); + let scanned = self.1.load(Ordering::Relaxed); + self.0.lock().unwrap().push((batch.len(), scanned)); Box::pin(async move { for record in batch { self.on_record(record).await?; @@ -795,17 +794,18 @@ mod tests { let seen = Arc::new(StdMutex::new(Vec::new())); let calls = Arc::new(AtomicU64::new(0)); let calls_p = calls.clone(); - let mut q = QueueingRecordSink::spawn(PreLens(seen.clone()), 2, 8, None).with_predecode( - Arc::new(move |_| { + let mut q = QueueingRecordSink::spawn(Lens(seen.clone(), calls.clone()), 2, 8, None) + .with_prescan(Arc::new(move |_| { calls_p.fetch_add(1, Ordering::Relaxed); - None - }), - ); + })); for lsn in [1, 2, 3] { q.on_record(&synth(lsn)).await.expect("send"); } q.close().await.expect("close"); - assert_eq!(*seen.lock().unwrap(), [(2, 2), (1, 1)]); + // Later batches may scan ahead of earlier ones dispatching + let seen = seen.lock().unwrap(); + assert_eq!(seen.iter().map(|(n, _)| *n).collect::>(), [2, 1]); + assert!(seen[0].1 >= 2 && seen[1].1 == 3); assert_eq!(calls.load(Ordering::Relaxed), 3); } diff --git a/src/xact/spill.rs b/src/xact/spill.rs index 63e9966e..ad343d10 100644 --- a/src/xact/spill.rs +++ b/src/xact/spill.rs @@ -20,6 +20,7 @@ //! TAG_TOAST_DELETE (body = encoded ToastDelete) //! TAG_RAW (body = encoded RawRecord) //! TAG_DESCRIPTOR (body = u64 valid_from + descriptor) +//! TAG_INPUT (body = u32 dict id + encoded HeapInput) //! ``` //! //! `TAG_DESCRIPTOR` never surfaces as an entry: each distinct `(rfn, valid_from)` @@ -76,6 +77,122 @@ pub enum SpillEntry { /// CREATE / TRUNCATE / rewrite generation); decoded at commit once /// `relation_at(rfn, commit_lsn)` resolves the survivor Raw(Box), + /// Ordinary heap record under a descriptor covering it at record time, + /// decoded at commit against that version + Input(Box), +} + +/// Heap record bytes decode reads: `main_data` and block 0 registered data. +/// Block images and other blocks drop; `wal_level=logical` keeps tuple +/// data registered (`REGBUF_KEEP_DATA`) +#[derive(Debug, Clone, PartialEq)] +pub struct HeapInput { + /// Writer xid from record header, a subxact's own under merge + pub xid: u32, + pub rm: u8, + pub info: u8, + pub source_lsn: u64, + /// Tuples a decode yields, MULTI_INSERT fans out + pub tuples: u16, + /// `main_data` then block 0 data + bytes: Box<[u8]>, + main_len: u32, + has_block: bool, +} + +impl HeapInput { + pub fn from_parsed( + parsed: &walrus::pg::walparser::XLogRecord<'_>, + source_lsn: u64, + tuples: u16, + ) -> Self { + let block = parsed.blocks.first().map(|b| &b.data[..]); + let mut bytes = Vec::with_capacity(parsed.main_data.len() + block.map_or(0, <[u8]>::len)); + bytes.extend_from_slice(&parsed.main_data); + bytes.extend_from_slice(block.unwrap_or_default()); + Self { + xid: parsed.header.xact_id, + rm: parsed.header.resource_manager_id, + info: parsed.header.info, + source_lsn, + tuples, + bytes: bytes.into_boxed_slice(), + main_len: parsed.main_data.len() as u32, + has_block: block.is_some(), + } + } + + pub fn approx_bytes(&self) -> usize { + std::mem::size_of::() + self.bytes.len() + } + + /// Borrowed record targeting `rfn` for the shared heap decoder + pub fn to_xlog_record(&self, rfn: RelFileNode) -> walrus::pg::walparser::XLogRecord<'_> { + use walrus::pg::walparser::{ + BlockLocation, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, XLogRecordHeader, + }; + let (main_data, data) = self.bytes.split_at(self.main_len as usize); + XLogRecord { + header: XLogRecordHeader { + xact_id: self.xid, + info: self.info, + resource_manager_id: self.rm, + ..Default::default() + }, + main_data_len: self.main_len, + origin: 0, + toplevel_xid: 0, + blocks: self + .has_block + .then(|| XLogRecordBlock { + header: XLogRecordBlockHeader { + data_length: data.len() as u16, + location: BlockLocation { + rel: rfn, + block_no: 0, + }, + ..Default::default() + }, + image: std::borrow::Cow::Borrowed(&[]), + data: std::borrow::Cow::Borrowed(data), + }) + .into_iter() + .collect(), + main_data: std::borrow::Cow::Borrowed(main_data), + } + } +} + +/// [`HeapInput`] pinned to its record-time descriptor version, as +/// [`DescribedHeap`] pins decoded cells +#[derive(Debug, Clone, PartialEq)] +pub struct DescribedInput { + pub input: HeapInput, + pub descriptor: Arc, + /// Log entry `valid_from` of `descriptor` at attach + pub descriptor_valid_from: u64, +} + +impl DescribedInput { + /// Owned cells per tuple, through the cell adapter + pub fn decode( + &self, + ) -> std::result::Result< + crate::decode::heap_decoder::DecodedHeaps, + crate::decode::heap_decoder::DecodeError, + > { + crate::decode::heap_decoder::decode_heap_record( + &self.input.to_xlog_record(self.descriptor.rfn), + self.input.source_lsn, + &self.descriptor, + ) + } + + pub fn approx_bytes(&self) -> usize { + self.input.approx_bytes() + + std::mem::size_of::>() + + std::mem::size_of::() + } } const _: () = assert!(std::mem::size_of::() <= 16); @@ -288,10 +405,11 @@ pub const SPILL_MAGIC: [u8; 2] = *b"WS"; /// the `TAG_DESCRIPTOR` dictionary (Heap bodies reference descriptors by dict /// id). v6 added writer xid to heap + raw bodies. v7 added database OID to /// chunk and `ToastDelete` bodies. v8 dropped heap rfn, restored from the -/// dict descriptor. v9 stored inet addr as fixed 16 bytes. `DrainEntry` +/// dict descriptor. v9 stored inet addr as fixed 16 bytes. v10 added +/// `TAG_INPUT` undecoded ordinary heap records. `DrainEntry` /// events (Catalog, ToastBarrier, Config/Signal) are drain-time, never /// spilled, so they don't touch this format -pub const SPILL_VERSION: u16 = 9; +pub const SPILL_VERSION: u16 = 10; /// Wire tags, one block per tag space so encode and decode read off the same /// table. Renumbering is a body-encoding change: bump [`SPILL_VERSION`] @@ -302,6 +420,7 @@ const TAG_CHUNK: u8 = 1; const TAG_TOAST_DELETE: u8 = 2; const TAG_RAW: u8 = 3; const TAG_DESCRIPTOR: u8 = 4; +const TAG_INPUT: u8 = 5; /// [`HeapOp`], one byte in every heap body const OP_INSERT: u8 = 0; @@ -552,24 +671,46 @@ impl SpillWriter { self.out.flush().await } + /// Dictionary id for a descriptor version, framing it on first use + fn dict_id(&mut self, descriptor: &RelDescriptor, valid_from: u64) -> u32 { + let next_id = self.dict.len() as u32; + let id = *self + .dict + .entry((descriptor.rfn, valid_from)) + .or_insert(next_id); + if id == next_id { + frame_into(&mut self.out.buf, TAG_DESCRIPTOR, |out| { + push_u64(out, valid_from); + encode_descriptor_bytes(out, descriptor); + }); + } + id + } + fn encode(&mut self, entry: &SpillEntry) { - let body = &mut self.out.buf; match entry { SpillEntry::Heap(h) => { - let key = (h.descriptor.rfn, h.descriptor_valid_from); - let next_id = self.dict.len() as u32; - let id = *self.dict.entry(key).or_insert(next_id); - if id == next_id { - frame_into(body, TAG_DESCRIPTOR, |out| { - push_u64(out, h.descriptor_valid_from); - encode_descriptor_bytes(out, &h.descriptor); - }); - } - frame_into(body, TAG_HEAP, |out| { + let id = self.dict_id(&h.descriptor, h.descriptor_valid_from); + frame_into(&mut self.out.buf, TAG_HEAP, |out| { push_u32(out, id); encode_heap_into(out, &h.decoded); }); } + SpillEntry::Input(i) => { + let id = self.dict_id(&i.descriptor, i.descriptor_valid_from); + frame_into(&mut self.out.buf, TAG_INPUT, |out| { + push_u32(out, id); + encode_input_into(out, &i.input); + }); + } + entry => self.encode_payload(entry), + } + } + + fn encode_payload(&mut self, entry: &SpillEntry) { + let body = &mut self.out.buf; + match entry { + SpillEntry::Heap(_) | SpillEntry::Input(_) => unreachable!("dictionary entries"), SpillEntry::Chunk(c) => frame_into(body, TAG_CHUNK, |out| encode_chunk_into(out, c)), SpillEntry::ToastDelete(d) => frame_into(body, TAG_TOAST_DELETE, |out| { encode_toast_delete_into(out, d) @@ -737,24 +878,36 @@ impl SpillReader { } let body = &self.chunks.consume(5 + len)[5..]; let mut cur = Cursor::new(body); + let described = |cur: &mut Cursor| { + let dict_id = cur.u32()? as usize; + self.dict + .get(dict_id) + .cloned() + .ok_or_else(|| SpillError::Format { + offset: cur.pos, + detail: format!( + "heap references dict id {dict_id}, only {} defined", + self.dict.len(), + ), + }) + }; let entry = match tag { TAG_HEAP => { - let dict_id = cur.u32()? as usize; - let Some((descriptor, valid_from)) = self.dict.get(dict_id).cloned() else { - return Err(SpillError::Format { - offset: cur.pos, - detail: format!( - "heap references dict id {dict_id}, only {} defined", - self.dict.len(), - ), - }); - }; + let (descriptor, valid_from) = described(&mut cur)?; SpillEntry::Heap(Box::new(DescribedHeap { decoded: decode_heap(&mut cur, descriptor.rfn)?, descriptor, descriptor_valid_from: valid_from, })) } + TAG_INPUT => { + let (descriptor, valid_from) = described(&mut cur)?; + SpillEntry::Input(Box::new(DescribedInput { + input: decode_input(&mut cur)?, + descriptor, + descriptor_valid_from: valid_from, + })) + } TAG_CHUNK => SpillEntry::Chunk(Box::new(decode_chunk(&mut cur)?)), TAG_TOAST_DELETE => { SpillEntry::ToastDelete(Box::new(decode_toast_delete(&mut cur)?)) @@ -1144,6 +1297,18 @@ fn encode_raw_into(out: &mut Vec, r: &RawRecord) { } } +fn encode_input_into(out: &mut Vec, i: &HeapInput) { + out.reserve(32 + i.bytes.len()); + push_u32(out, i.xid); + push_u8(out, i.rm); + push_u8(out, i.info); + push_u64(out, i.source_lsn); + push_u16(out, i.tuples); + push_u32(out, i.main_len); + push_u8(out, u8::from(i.has_block)); + push_bytes(out, &i.bytes); +} + // ── decoding ──────────────────────────────────────────────────────── pub(crate) struct Cursor<'a> { @@ -1397,6 +1562,33 @@ fn decode_toast_delete(c: &mut Cursor) -> Result { }) } +fn decode_input(c: &mut Cursor) -> Result { + let xid = c.u32()?; + let rm = c.u8()?; + let info = c.u8()?; + let source_lsn = c.u64()?; + let tuples = c.u16()?; + let main_len = c.u32()?; + let has_block = c.u8()? != 0; + let bytes = c.bytes()?.into_boxed_slice(); + if main_len as usize > bytes.len() { + return Err(SpillError::Format { + offset: c.pos, + detail: format!("input main_data {main_len} exceeds {} bytes", bytes.len()), + }); + } + Ok(HeapInput { + xid, + rm, + info, + source_lsn, + tuples, + bytes, + main_len, + has_block, + }) +} + fn decode_raw(c: &mut Cursor) -> Result { let xid = c.u32()?; let rm = c.u8()?; @@ -1636,6 +1828,47 @@ mod tests { r.unlink().await.unwrap(); } + /// Inputs keep only main data and block 0 data, share the descriptor + /// dictionary with heaps, and decode identically after spill + #[tokio::test(flavor = "current_thread")] + async fn round_trip_heap_input() { + let tmp = tempdir().unwrap(); + let store = SpillStore::new(tmp.path().to_path_buf()).unwrap(); + let mut w = store.writer(1, 0x300).await.unwrap(); + let descriptor = crate::xact::xact_buffer::raw_fixtures::int4_descriptor(16480); + let raw = + crate::xact::xact_buffer::raw_fixtures::multi_insert_raw(9, 0x4242, 16480, &[5, 6]); + let input = DescribedInput { + input: HeapInput::from_parsed(&raw.to_xlog_record(), raw.source_lsn, 2), + descriptor: descriptor.clone(), + descriptor_valid_from: 0x100, + }; + let heap = DescribedHeap { + decoded: input.decode().unwrap().remove(0), + descriptor, + descriptor_valid_from: 0x100, + }; + let entries = vec![ + SpillEntry::Input(Box::new(input.clone())), + SpillEntry::Heap(Box::new(heap.clone())), + SpillEntry::Input(Box::new(input.clone())), + ]; + w.write_batch(entries.clone()).await.unwrap(); + let mut r = w.finish().await.unwrap(); + for want in entries { + let got = r.next().await.unwrap().unwrap(); + if let (SpillEntry::Input(a), SpillEntry::Input(b)) = (&got, &want) { + assert_eq!(a.decode().unwrap(), b.decode().unwrap()); + } + assert_eq!(got, want); + } + assert!(r.next().await.unwrap().is_none()); + assert_eq!( + input.input.approx_bytes(), + std::mem::size_of::() + raw.main_data.len() + raw.blocks[0].data.len() + ); + } + #[tokio::test(flavor = "current_thread")] async fn round_trip_raw_record() { let tmp = tempdir().unwrap(); diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index 6f678087..b1df5259 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -51,9 +51,11 @@ use crate::catalog::desc_log::{ Ambiguity, DescriptorLog, DescriptorLogs, LogEntry, LogValue, LookupMiss, }; use crate::catalog::pending::{PendingCatalog, PendingSlot}; +use crate::decode::datum::DatumRef; use crate::decode::decoder_sink::{DecoderSinkError, DecoderStats, SkipReason}; use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, DescribedHeap, HeapOp, Predecoded, ToastPointer, decode_heap_record, + ColumnValue, DecodedHeap, DescribedHeap, HeapImages, HeapOp, ToastPointer, TupleView, + decode_heap_record, scan_heap_record, view_heap_record, }; #[cfg(test)] use crate::decode::wal_xact::{ @@ -71,10 +73,11 @@ use crate::toast::{ check_value_caps, detoasted_sized, finish_value, pointer_extsize, }; use crate::xact::spill::{ - BodySpoolFile, BodySpoolWriter, RawRecord, SpillEntry, SpillError, SpillReader, SpillStore, - SpillWriter, ToastChunk, ToastDelete, + BodySpoolFile, BodySpoolWriter, DescribedInput, HeapInput, RawRecord, SpillEntry, SpillError, + SpillReader, SpillStore, SpillWriter, ToastChunk, ToastDelete, }; use foldhash::{HashMap, HashMapExt}; +use smallvec::SmallVec; use std::pin::Pin; @@ -151,9 +154,42 @@ fn entry_lsn(e: &SpillEntry) -> u64 { SpillEntry::Chunk(c) => c.source_lsn, SpillEntry::ToastDelete(d) => d.source_lsn, SpillEntry::Raw(r) => r.source_lsn, + SpillEntry::Input(i) => i.input.source_lsn, } } +/// Heaps an entry yields at drain +fn entry_rows(e: &SpillEntry) -> usize { + match e { + SpillEntry::Input(i) => usize::from(i.input.tuples), + _ => 1, + } +} + +/// Cells for every tuple of a buffered input, under its pinned descriptor. +/// Record-time scan validated the bytes, so failure means corrupt storage +fn input_heaps( + i: &DescribedInput, +) -> std::result::Result, XactBufferError> { + let decoded = i + .decode() + .map_err(|e| XactBufferError::OrdinaryFailClosed { + relid: i.descriptor.oid, + lsn: i.input.source_lsn, + rm: i.input.rm, + info: i.input.info, + reason: FailClosedReason::Malformed(e.to_string()), + })?; + Ok(decoded + .into_iter() + .map(|decoded| DescribedHeap { + decoded, + descriptor: i.descriptor.clone(), + descriptor_valid_from: i.descriptor_valid_from, + }) + .collect()) +} + /// Resident cap on one commit drain's memory-held chunk bodies; bodies /// past it spool to disk (`toastbody-*`), refs stay resident pub const TOAST_BODY_SPOOL_MEM_MAX: usize = 16 << 20; @@ -498,6 +534,7 @@ fn approximate_size(entry: &SpillEntry) -> usize { SpillEntry::Chunk(c) => std::mem::size_of::() + c.chunk_data.len(), SpillEntry::ToastDelete(_) => std::mem::size_of::(), SpillEntry::Raw(r) => r.approx_bytes(), + SpillEntry::Input(i) => i.approx_bytes(), } } @@ -917,6 +954,11 @@ impl XactBuffer { last_lsn = last_lsn.max(r.source_lsn); rels.extend(r.rfn().map(|rfn| (rfn.db_node, rfn.rel_node))); } + SpillEntry::Input(i) => { + heap_count += u64::from(i.input.tuples); + last_lsn = last_lsn.max(i.input.source_lsn); + rels.insert((i.descriptor.rfn.db_node, i.descriptor.rfn.rel_node)); + } } } for (lsn, _) in &st.events { @@ -956,6 +998,16 @@ impl XactBuffer { self.absorb(xid, first_lsn, entry).await } + /// Ordinary heap record kept undecoded until commit + pub async fn on_input( + &mut self, + described: DescribedInput, + ) -> std::result::Result<(), XactBufferError> { + let (xid, first_lsn) = (described.input.xid, described.input.source_lsn); + self.absorb(xid, first_lsn, SpillEntry::Input(Box::new(described))) + .await + } + /// Built from `pg_toast.pg_toast_` INSERTs by the decoder sink pub async fn on_toast_chunk( &mut self, @@ -1166,7 +1218,11 @@ impl XactBuffer { ) -> std::result::Result<(), XactBufferError> { let sz = approximate_size(&entry); let raw_sz = matches!(&entry, SpillEntry::Raw(_)).then_some(sz as u64); - let plain = matches!(&entry, SpillEntry::Heap(h) if h.decoded.op != HeapOp::Truncate); + let plain = match &entry { + SpillEntry::Heap(h) => h.decoded.op != HeapOp::Truncate, + SpillEntry::Input(_) => true, + _ => false, + }; let st = self.state_for(xid, first_lsn); st.plain &= plain; st.in_mem.push(entry); @@ -1365,18 +1421,26 @@ impl XactBuffer { && st.plain && st.spill.is_none() && st.events.is_empty() - && st.in_mem.len() <= self.config.drain_batch_rows + && st.in_mem.iter().map(entry_rows).sum::() <= self.config.drain_batch_rows && st.in_mem_bytes <= self.config.drain_batch_bytes { - use SpillEntry::Heap; - let heaps = std::mem::take(&mut st.in_mem) - .into_iter() - .map(|entry| { - // Plain state holds heaps only - let Heap(h) = entry else { unreachable!() }; - *h - }) - .collect(); + let mut heaps = Vec::with_capacity(st.in_mem.len()); + for entry in std::mem::take(&mut st.in_mem) { + match entry { + SpillEntry::Heap(h) => heaps.push(*h), + SpillEntry::Input(i) => { + if !xids.contains(&i.input.xid) { + return Err(XactBufferError::ForeignXid { + xid: i.input.xid, + top: top_xid, + }); + } + heaps.extend(input_heaps(&i)?); + } + // Plain state holds heaps and inputs only + _ => unreachable!(), + } + } self.stats.committed_xacts_total += 1; return Ok(CommittedDrain { commit_ts, @@ -1789,12 +1853,25 @@ impl MergedDrain { SpillEntry::Chunk(c) => self.fold_chunk(*c)?, SpillEntry::ToastDelete(d) => self.fold_delete(*d)?, SpillEntry::Raw(raw) => self.fold_raw(&raw)?, + SpillEntry::Input(i) => self.fold_input(&i)?, } } } } } + /// Queue an input's tuples like a resolved raw record's + fn fold_input(&mut self, i: &DescribedInput) -> std::result::Result<(), XactBufferError> { + self.check_owned(i.input.xid)?; + for heap in input_heaps(i)? { + let sz = heap.approx_bytes(); + self.gauge.add(sz); + self.pending_gauge.add(sz); + self.pending_heaps.push_back(Box::new(heap)); + } + Ok(()) + } + fn check_owned(&self, xid: u32) -> std::result::Result<(), XactBufferError> { if self.allowed_xids.contains(&xid) { return Ok(()); @@ -2866,16 +2943,23 @@ impl RecentToastBirths { fn read_ahead_kept( ahead: &ToastResolver, rel: &RelDescriptor, - set: &crate::decode::heap_decoder::DecodedHeaps, + source_lsn: u64, + images: &[HeapImages>], born: &std::sync::Mutex, ) { - for d in set { + for d in images { if rel.kind == 't' { - if let Some(Some(ColumnValue::Oid(id))) = d.new.as_ref().and_then(|t| t.columns.first()) + // `chunk_id oid` leads every toast tuple + if rel + .attributes + .first() + .is_some_and(|a| a.type_oid == crate::schema::OIDOID) + && let Some(DatumRef::Fixed(id)) = d.new.as_ref().and_then(|t| t.datums.first()) + && let Ok(id) = <[u8; 4]>::try_from(*id) { born.lock() .expect("recent toast births") - .insert((rel.oid, *id)); + .insert((rel.oid, u32::from_le_bytes(id))); } continue; } @@ -2883,15 +2967,15 @@ fn read_ahead_kept( continue; } for t in [&d.new, &d.old].into_iter().flatten() { - for (col, attr) in t.columns.iter().zip(&rel.attributes) { - if let &Some(ColumnValue::ExternalToast(p)) = col { + for (datum, attr) in t.datums.iter().zip(&rel.attributes) { + if let &DatumRef::External(p) = datum { let recent = born .lock() .expect("recent toast births") .counts .contains_key(&(p.va_toastrelid, p.va_valueid)); if !recent { - ahead.read_ahead(d.rfn.db_node, p, attr.type_oid, d.source_lsn); + ahead.read_ahead(rel.rfn.db_node, p, attr.type_oid, source_lsn); } } } @@ -2939,42 +3023,47 @@ impl BufferingDecoderSink { guard } - /// Decode half of [`RecordSink::on_record`], for a dispatcher to run - /// ahead of the worker. `None` wherever the worker would not decode - /// against a live descriptor; the worker re-resolves and keeps the - /// result only on the same descriptor version - /// - /// Starts the store reads of values updated rows keep, through `ahead` - pub fn predecoder( - &self, - ahead: ToastResolver, - ) -> impl Fn(&Record<'_>) -> Option + Send + Sync + 'static { + /// Scan half of [`RecordSink::on_record`], for a dispatcher to run ahead + /// of the worker: starts store reads of values updated rows keep, + /// through `ahead`, borrowing record bytes and building no cells + pub fn prescan(&self, ahead: ToastResolver) -> impl Fn(&Record<'_>) + Send + Sync + 'static { + use crate::decode::heap_decoder::{ + XLOG_HEAP_INSERT, XLOG_HEAP_OPMASK, XLOG_HEAP_TRUNCATE, XLOG_HEAP2_MULTI_INSERT, + }; let logs = self.logs.clone(); let born = std::sync::Mutex::new(RecentToastBirths::default()); move |record| { let rm = record.parsed.header.resource_manager_id; - if rm != RmId::Heap as u8 && rm != RmId::Heap2 as u8 { - return None; - } - if rm == RmId::Heap as u8 - && record.parsed.header.info & crate::decode::heap_decoder::XLOG_HEAP_OPMASK - == crate::decode::heap_decoder::XLOG_HEAP_TRUNCATE - { - return None; - } - if !matches!(record.route, Route::ToDecoder | Route::ToBoth) + let op = record.parsed.header.info & XLOG_HEAP_OPMASK; + let insert = if rm == RmId::Heap as u8 { + op == XLOG_HEAP_INSERT + } else if rm == RmId::Heap2 as u8 { + op == XLOG_HEAP2_MULTI_INSERT + } else { + return; + }; + if rm == RmId::Heap as u8 && op == XLOG_HEAP_TRUNCATE + || !matches!(record.route, Route::ToDecoder | Route::ToBoth) || record.defer_catalog_decode { - return None; + return; + } + let Some(rfn) = record.parsed.blocks.first().map(|b| b.header.location.rel) else { + return; + }; + let Some(Ok((rel, _))) = logs + .get(rfn.db_node) + .map(|log| log.peek_spanned(rfn, record.source_lsn)) + else { + return; + }; + // Ordinary inserts keep no older value; toast inserts are births + if insert && rel.kind != 't' { + return; + } + if let Ok(images) = view_heap_record(&record.parsed, &rel) { + read_ahead_kept(&ahead, &rel, record.source_lsn, &images, &born); } - let rfn = record.parsed.blocks.first()?.header.location.rel; - let (rel, valid_from) = logs - .get(rfn.db_node)? - .peek_spanned(rfn, record.source_lsn) - .ok()?; - let set = decode_heap_record(&record.parsed, record.source_lsn, &rel).ok()?; - read_ahead_kept(&ahead, &rel, &set, &born); - Some(Predecoded::new(rel, valid_from, set)) } } @@ -3129,11 +3218,7 @@ impl BufferingDecoderSink { } impl BufferingDecoderSink { - async fn handle( - &mut self, - record: &Record<'_>, - pre: Option, - ) -> std::result::Result<(), SinkError> { + async fn handle(&mut self, record: &Record<'_>) -> std::result::Result<(), SinkError> { if !Self::wants(record) { return Ok(()); } @@ -3258,12 +3343,53 @@ impl BufferingDecoderSink { return Ok(()); } }; - let decoded_set = if let Some(set) = pre.and_then(|p| p.take_if(&rel, rel_valid_from)) { - set - } else { - let _decode = decode_parent.as_ref().map(|p| { + let decode_span = || { + decode_parent.as_ref().map(|p| { tracing::info_span!(target: "walshadow::trace", parent: p, "decode").entered() - }); + }) + }; + // Runtime-config overlay: config-table heap writes never reach CH. + // Detect by resolved qualified name (rotation-proof — a rewritten + // relfilenode still resolves to the same schema.name), interpret each + // tuple into a ConfigEvent stamped (xid, source_lsn) so it drains in + // WAL order and applies at its commit LSN (plan §2/§6). + let config_kind = self + .config_schema + .as_deref() + .filter(|schema| &*rel.rel_name.namespace == *schema) + .and_then(|_| ConfigTableKind::from_relname(&rel.rel_name.name)); + if rel.kind != 't' && config_kind.is_none() { + // Ordinary rows stay as WAL bytes until commit; scanning now + // keeps malformed-record errors and per-op counts at record time + let shapes = { + let _decode = decode_span(); + scan_heap_record(&record.parsed, &rel).map_err(DecoderSinkError::from)? + }; + if shapes.is_empty() { + self.skip(SkipReason::NoTuple, record, rfn); + return Ok(()); + } + for t in &shapes { + let partial = [&t.new, &t.old].into_iter().flatten().any(|s| s.partial); + self.stats.record_op(t.op, partial); + } + let input = DescribedInput { + input: HeapInput::from_parsed( + &record.parsed, + record.source_lsn, + shapes.len() as u16, + ), + descriptor: rel, + descriptor_valid_from: rel_valid_from, + }; + let buffer_span = trace_span!(sampled, "buffer", rows = shapes.len()); + return async { self.lock_buffer().await.on_input(input).await } + .instrument(buffer_span) + .await + .map_err(SinkError::from); + } + let decoded_set = { + let _decode = decode_span(); match decode_heap_record(&record.parsed, record.source_lsn, &rel) { Ok(set) => set, Err(e) => return Err(DecoderSinkError::from(e).into()), @@ -3273,15 +3399,7 @@ impl BufferingDecoderSink { self.skip(SkipReason::NoTuple, record, rfn); return Ok(()); } - // Runtime-config overlay: config-table heap writes never reach CH. - // Detect by resolved qualified name (rotation-proof — a rewritten - // relfilenode still resolves to the same schema.name), interpret each - // tuple into a ConfigEvent stamped (xid, source_lsn) so it drains in - // WAL order and applies at its commit LSN (plan §2/§6). - if let Some(schema) = self.config_schema.as_deref() - && &*rel.rel_name.namespace == schema - && let Some(kind) = ConfigTableKind::from_relname(&rel.rel_name.name) - { + if let Some(kind) = config_kind { let mut buf = self.lock_buffer().await; for decoded in &decoded_set { if let Some(ev) = crate::runtime_config::interpret(kind, decoded, &rel) { @@ -3295,81 +3413,71 @@ impl BufferingDecoderSink { // TID for the toast branch: single-tuple INSERT/DELETE only // (toast_save_datum never multi-inserts). blkno rides block ref // 0; offnum sits in xl_heap_insert[0..2] / xl_heap_delete[4..6]. - let tid = (rel.kind == 't' && n_decoded == 1) + let tid = (n_decoded == 1) .then(|| toast_record_tid(&record.parsed)) .flatten(); async move { - // Lock once per record (not per tuple); on_heap/on_toast_chunk - // never touch the catalog, so no buffer→catalog inversion. + // Lock once per record (not per tuple); on_toast_chunk never + // touches the catalog, so no buffer→catalog inversion. let mut buf = self.lock_buffer().await; for decoded in decoded_set { self.stats.record(&decoded); - if rel.kind == 't' { - let xid = decoded.xid; - if decoded.op == HeapOp::Delete - && let Some((blkno, offnum)) = tid - { - // heap_toast_delete's DELETE: TID-keyed. Buffered - // as a store tombstone row applied at commit drain - self.stats - .toast_chunk_deletes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - buf.on_toast_delete( - crate::xact::spill::ToastDelete { - db_oid: rel.rfn.db_node, - toast_relid: rel.oid, - blkno, - offnum, - source_lsn: decoded.source_lsn, - }, - xid, - ) - .await - .map_err(SinkError::from)?; - } else if decoded.op != HeapOp::Insert { - // Non-delete non-insert (TRUNCATE fan-out never - // reaches here — kind 't' is filtered there): - // nothing to apply against the store - self.stats - .toast_chunk_deletes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } else if let Some(chunk) = toast_chunk_from_decoded(decoded, &rel, tid) { - if tid.is_none() { - // No TID → no store row key: the chunk still - // serves same-xact resolution, but its birth - // never reaches the mirror (a later referrer - // superseded-fills). toast_save_datum never - // multi-inserts, so this shape is unexpected. - tracing::warn!( - target: "walshadow::xact_buffer", - toast_relid = chunk.toast_relid, - value_id = chunk.value_id, - chunk_seq = chunk.chunk_seq, - "toast chunk without TID; not mirrored", - ); - self.stats - .toast_chunks_malformed - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } - self.stats - .toast_chunks_buffered - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - buf.on_toast_chunk(chunk, xid) - .await - .map_err(SinkError::from)?; - } else { + let xid = decoded.xid; + if decoded.op == HeapOp::Delete + && let Some((blkno, offnum)) = tid + { + // heap_toast_delete's DELETE: TID-keyed. Buffered + // as a store tombstone row applied at commit drain + self.stats + .toast_chunk_deletes + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + buf.on_toast_delete( + crate::xact::spill::ToastDelete { + db_oid: rel.rfn.db_node, + toast_relid: rel.oid, + blkno, + offnum, + source_lsn: decoded.source_lsn, + }, + xid, + ) + .await + .map_err(SinkError::from)?; + } else if decoded.op != HeapOp::Insert { + // Non-delete non-insert (TRUNCATE fan-out never + // reaches here — kind 't' is filtered there): + // nothing to apply against the store + self.stats + .toast_chunk_deletes + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else if let Some(chunk) = toast_chunk_from_decoded(decoded, &rel, tid) { + if tid.is_none() { + // No TID → no store row key: the chunk still + // serves same-xact resolution, but its birth + // never reaches the mirror (a later referrer + // superseded-fills). toast_save_datum never + // multi-inserts, so this shape is unexpected. + tracing::warn!( + target: "walshadow::xact_buffer", + toast_relid = chunk.toast_relid, + value_id = chunk.value_id, + chunk_seq = chunk.chunk_seq, + "toast chunk without TID; not mirrored", + ); self.stats .toast_chunks_malformed .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } + self.stats + .toast_chunks_buffered + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + buf.on_toast_chunk(chunk, xid) + .await + .map_err(SinkError::from)?; } else { - buf.on_heap(DescribedHeap { - decoded, - descriptor: rel.clone(), - descriptor_valid_from: rel_valid_from, - }) - .await - .map_err(SinkError::from)?; + self.stats + .toast_chunks_malformed + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } } Ok::<(), SinkError>(()) @@ -3410,19 +3518,17 @@ impl RecordSink for BufferingDecoderSink { record: &'a Record<'a>, ) -> Pin> + Send + 'a>> { - Box::pin(self.handle(record, None)) + Box::pin(self.handle(record)) } fn on_records<'a>( &'a mut self, batch: &'a [Record<'a>], - pre: &'a mut [Option], ) -> Pin> + Send + 'a>> { Box::pin(async move { - for (i, record) in batch.iter().enumerate().filter(|(_, r)| Self::wants(r)) { - let pre = pre.get_mut(i).and_then(Option::take); - self.handle(record, pre).await?; + for record in batch.iter().filter(|r| Self::wants(r)) { + self.handle(record).await?; } Ok(()) }) @@ -6812,24 +6918,22 @@ mod tests { assert_eq!(born.counts.len(), 1024); } - /// Predecode reads the log off the worker, so a result for another - /// descriptor generation is discarded and the record decodes again + /// Ordinary rows buffer as WAL bytes and decode at commit against the + /// descriptor version covering them at record time #[tokio::test(flavor = "current_thread")] - async fn predecoded_set_used_only_for_same_descriptor() { + async fn ordinary_records_buffer_undecoded() { use crate::record::Route; let log_dir = tempdir().unwrap(); let rel = int4_descriptor(16480); let log = single_rel_log(log_dir.path(), &rel).await; - let (logged, valid_from) = log.descriptor_at_spanned(rel.rfn, 0x200).unwrap(); let spill_dir = tempdir().unwrap(); let buffer = Arc::new(Mutex::new( XactBuffer::new(cfg(spill_dir.path().to_path_buf())).unwrap(), )); let mut sink = BufferingDecoderSink::new(DescriptorLogs::single(log), buffer.clone()); - - let raws: Vec = (1..=4) - .map(|v| multi_insert_raw(1, 0x200 + v as u64, 16480, &[v])) + let raws: Vec = (1..=3) + .map(|v| multi_insert_raw(1, 0x200 + v as u64, 16480, &[v, v * 10])) .collect(); let batch: Vec> = raws .iter() @@ -6840,32 +6944,19 @@ mod tests { ..Default::default() }) .collect(); - let predecode = sink.predecoder(ToastResolver::disabled()); - assert!( - predecode(&batch[0]).is_some(), - "covered heap record predecodes" - ); - let mut truncate = raws[0].to_xlog_record(); - truncate.header.resource_manager_id = RmId::Heap as u8; - truncate.header.info = crate::decode::heap_decoder::XLOG_HEAP_TRUNCATE; - let truncate = Record { - parsed: truncate, - route: Route::ToDecoder, - ..Default::default() - }; - assert!(predecode(&truncate).is_none(), "truncate never predecodes"); - let decoy_raw = multi_insert_raw(1, 0x200, 16480, &[99]); - let decoy = |rel: &Arc, valid_from| { - let set = decode_heap_record(&decoy_raw.to_xlog_record(), 0x200, rel).unwrap(); - Some(Predecoded::new(rel.clone(), valid_from, set)) - }; - let mut pre = vec![ - decoy(&logged, valid_from), - decoy(&Arc::new((*logged).clone()), valid_from), - decoy(&logged, valid_from + 1), - None, - ]; - sink.on_records(&batch, &mut pre).await.unwrap(); + sink.prescan(ToastResolver::disabled())(&batch[0]); + sink.on_records(&batch).await.unwrap(); + assert_eq!(sink.stats().inserts.load(Ordering::Relaxed), 6); + { + let buf = buffer.lock().await; + let st = &buf.inflight[&1]; + assert!( + st.in_mem + .iter() + .all(|e| matches!(e, SpillEntry::Input(i) if i.input.tuples == 2)) + ); + assert_eq!(buf.inflight_snapshot()[0].heap_count, 6); + } let mut drain = buffer .lock() @@ -6884,7 +6975,15 @@ mod tests { .map(|h| h.decoded.new.as_ref().unwrap().columns[0].clone()) .collect(); let int4 = |v| Some(ColumnValue::Int4(v)); - assert_eq!(values, [int4(99), int4(2), int4(3), int4(4)]); + assert_eq!( + values, + [int4(1), int4(10), int4(2), int4(20), int4(3), int4(30)] + ); + assert!( + out.heaps + .iter() + .all(|h| Arc::ptr_eq(&h.descriptor, &out.heaps[0].descriptor)) + ); drain.finish().await.unwrap(); } From d0b37ad40556730377d133c3088dd11a07c57e3a Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:16:35 +0000 Subject: [PATCH 05/20] Encode rows into column chunks while planning commits Commit drain hands planning undecoded inputs (DrainHeap) instead of decoding them into cells. Planner views each input's tuple images in place and encodes them straight into per-table TableEncoder chunks; inputs with external TOAST pointers, truncates, stash-resolved records and directly buffered heaps go through the cell adapter and detoast as before. Placement moves into planning: append-only routes drop deletes, PostGIS cells render, key changes split into a delete marker at lsn-1 plus the new row, with views comparing stored bytes and falling back to owned values so NaN and representation differences decide as before. Plan spool v3 stores encoded chunks: segments seal at the drain batch row/byte limits and before every control or truncate, spill to frames of serialized column buffers, and replay as one seq each. Encoding errors now fail the plan before transaction effects under reason "encode". Batcher takes PlacedSegment messages and merges chunks into open batches, rebasing string offsets and recomputing the oracle frame size; a layout change seals the open batch first. Row placement (decode.rs) is gone; bootstrap still sends cell rows until it encodes directly. Tuple views store up to 32 datums inline and inputs view without building an XLogRecord, so planning allocates nothing per row on the view path. spool_read --planner --mapped --relations 8 --control-every 10000 --rows 200000 plans in ~0.10s with 39.8k allocations either way and checks encoded rows against a cell-encoded reference plan. Co-Authored-By: Claude Opus 5.5 --- benches/spool_read.rs | 315 +++++--- benches/xact_spill.rs | 23 +- src/backfill/wal_replay.rs | 130 +-- src/bin/stream/metrics_publish.rs | 1 + src/decode/datum.rs | 9 +- src/decode/heap_decoder.rs | 101 +-- src/emit/ch_emitter.rs | 311 +++++++- src/emit/direct_parity.rs | 119 +++ src/emit/pipeline/batcher.rs | 237 +++++- src/emit/pipeline/decode.rs | 319 -------- src/emit/pipeline/mod.rs | 1 - src/emit/pipeline/plan_spool.rs | 1220 +++++++++++++++++------------ src/emit/pipeline/planner.rs | 274 +++++-- src/emit/pipeline/reorder.rs | 207 ++--- src/emit/pipeline/tail.rs | 14 +- src/ops/metrics.rs | 7 +- src/ops/oracle.rs | 86 ++ src/xact/spill.rs | 19 +- src/xact/xact_buffer.rs | 212 +++-- tests/xact_buffer.rs | 5 +- 20 files changed, 2299 insertions(+), 1311 deletions(-) delete mode 100644 src/emit/pipeline/decode.rs diff --git a/benches/spool_read.rs b/benches/spool_read.rs index a00c59ec..9ab0d7f6 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -1,7 +1,8 @@ //! Deferred spool and plan spool write and replay, without servers. //! `cargo bench --bench spool_read -- --rows 200000` //! Add `--plan-only --planner --mapped --relations 8 --control-every 10000` -//! to include planning, route caching within benchmark, and row placement. +//! to include planning, route caching within benchmark, and column encoding. +//! `--cells` plans decoded rows instead of WAL record bytes. //! Excludes WAL drain, TOAST I/O, control execution, and destination insertion. use std::path::PathBuf; @@ -11,18 +12,23 @@ use std::time::{Duration, Instant}; use clap::Parser; use foldhash::{HashMap, HashMapExt}; use walrus::pg::walparser::RelFileNode; +use walrus::pg::walparser::{ + BlockLocation, RmId, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, XLogRecordHeader, +}; use walshadow::backup_page_walk::BackfillTuple; -use walshadow::ch_emitter::EmitterStats; +use walshadow::ch_emitter::EncodedChunk; use walshadow::emit::route::RouteSnapshot; use walshadow::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; -use walshadow::mapping::{TableMapping, TableTarget}; -use walshadow::pipeline::decode::place_rows; -use walshadow::pipeline::plan_spool::{PlanItem, PlanWriter}; +use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; +use walshadow::pipeline::plan_spool::{PlanItem, PlanWriter, ViewChange}; use walshadow::pipeline::planner::{PlanRouteView, Planner}; -use walshadow::schema::{RelDescriptor, RelName, ReplIdent, SchemaEvent}; +use walshadow::schema::{ + INT4OID, RelAttr, RelDescriptor, RelName, ReplIdent, SchemaEvent, TEXTOID, +}; +use walshadow::spill::{DescribedInput, HeapInput}; use walshadow::spool::DeferredSpool; use walshadow::toast::ToastResolver; -use walshadow::xact_buffer::{DrainEntry, DrainedBatch, OrderedEvent}; +use walshadow::xact_buffer::{DrainEntry, DrainHeap, DrainedBatch, OrderedEvent}; #[path = "common/alloc_meter.rs"] mod alloc_meter; @@ -43,9 +49,12 @@ struct Args { dir: PathBuf, #[arg(long)] plan_only: bool, - /// Run Planner and row placement with cached routes. + /// Run Planner with cached routes. #[arg(long)] planner: bool, + /// Plan decoded rows rather than WAL record bytes. + #[arg(long)] + cells: bool, #[arg(long)] mapped: bool, #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u16).range(1..))] @@ -147,18 +156,16 @@ struct BenchRoutes { } impl PlanRouteView for BenchRoutes { - fn route_for(&mut self, heap: &DescribedHeap) -> Option> { + fn route_for(&mut self, rel: &Arc) -> Option> { self.calls += 1; - if let Some((desc, route)) = self.memo.get(&heap.descriptor.rel_name) - && Arc::ptr_eq(desc, &heap.descriptor) + if let Some((desc, route)) = self.memo.get(&rel.rel_name) + && Arc::ptr_eq(desc, rel) { return route.clone(); } let route = self.route.clone(); - self.memo.insert( - heap.descriptor.rel_name.clone(), - (heap.descriptor.clone(), route.clone()), - ); + self.memo + .insert(rel.rel_name.clone(), (rel.clone(), route.clone())); route } @@ -170,6 +177,59 @@ impl PlanRouteView for BenchRoutes { } } +fn attr(attnum: i16, type_oid: u32, type_len: i16) -> RelAttr { + RelAttr { + attnum, + name: format!("c{attnum}"), + type_oid, + typmod: -1, + not_null: false, + dropped: false, + type_name: String::new(), + type_byval: type_len > 0, + type_len, + type_align: 'i', + type_storage: if type_len > 0 { 'p' } else { 'x' }, + missing_default: None, + } +} + +/// XLOG_HEAP_INSERT block data: xl_heap_header, int4, 4-byte-header text +fn insert_tuple(id: i32, payload: &[u8]) -> Vec { + let mut t = Vec::with_capacity(14 + payload.len()); + t.extend_from_slice(&2u16.to_le_bytes()); + t.extend_from_slice(&0u16.to_le_bytes()); + t.push(24); + t.push(0); + t.extend_from_slice(&id.to_le_bytes()); + t.extend_from_slice(&(((4 + payload.len()) as u32) << 2).to_le_bytes()); + t.extend_from_slice(payload); + t +} + +fn insert_record<'a>(xid: u32, rel: &RelDescriptor, tuple: &'a [u8]) -> XLogRecord<'a> { + XLogRecord { + header: XLogRecordHeader { + xact_id: xid, + resource_manager_id: RmId::Heap as u8, + ..Default::default() + }, + blocks: vec![XLogRecordBlock { + header: XLogRecordBlockHeader { + location: BlockLocation { + rel: rel.rfn, + block_no: 0, + }, + ..Default::default() + }, + data: std::borrow::Cow::Borrowed(tuple), + ..Default::default() + }], + main_data: std::borrow::Cow::Borrowed(&[1, 0, 8][..]), + ..Default::default() + } +} + async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { let dir = tempfile::tempdir_in(&args.dir)?; let descriptors: Vec<_> = (0..args.relations) @@ -187,15 +247,21 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { persistence: 'p', ancestors: Vec::new(), replident: ReplIdent::Default { pk_attnums: None }, - attributes: Vec::new(), + attributes: vec![attr(1, INT4OID, 4), attr(2, TEXTOID, -1)], }) }) .collect(); let route = args.mapped.then(|| { + let column = |attnum: i16, ty: &str| ColumnMapping { + src_attnum: attnum, + target_name: format!("c{attnum}"), + target_type: ty.into(), + type_pinned: false, + }; RouteSnapshot::freeze( Arc::new(TableMapping { target: TableTarget::new("db", "spool_bench"), - columns: Vec::new(), + columns: vec![column(1, "Int32"), column(2, "String")], tees: Vec::new(), }), Arc::default(), @@ -203,12 +269,26 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { ) }); let payload = "x".repeat(args.payload_bytes); + let xid_of = |i: usize| 7 + (i % 4) as u32; + let make_input = |i: usize| { + let descriptor = &descriptors[i % descriptors.len()]; + let tuple = insert_tuple(i as i32, payload.as_bytes()); + DescribedInput { + input: HeapInput::from_parsed( + &insert_record(xid_of(i), descriptor, &tuple), + i as u64, + 1, + ), + descriptor: descriptor.clone(), + descriptor_valid_from: 1, + } + }; let make_heap = |i: usize| { let descriptor = &descriptors[i % descriptors.len()]; DescribedHeap { decoded: DecodedHeap { rfn: descriptor.rfn, - xid: 7 + (i % 4) as u32, + xid: xid_of(i), source_lsn: i as u64, op: HeapOp::Insert, new: Some(DecodedTuple { @@ -239,123 +319,153 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { calls: 0, }; let resolver = ToastResolver::disabled(); + // Drained slices built ahead so timing covers planning alone + let batches: Vec = if args.planner { + (0..args.rows) + .step_by(4096) + .map(|first| { + let end = (first + 4096).min(args.rows); + let mut heaps = Vec::with_capacity(end - first); + let mut ordered_events = Vec::new(); + for i in first..end { + if let Some(event) = control(i) { + ordered_events.push(OrderedEvent { + heap_idx: heaps.len(), + row_idx: 0, + event, + }); + } + heaps.push(if args.cells { + DrainHeap::Cells(make_heap(i)) + } else { + DrainHeap::Input(Box::new(make_input(i))) + }); + } + DrainedBatch { + heaps, + ordered_events, + chunks: Vec::new(), + new_rows: Default::default(), + truncate_rows: Vec::new(), + is_final: end == args.rows, + } + }) + .collect() + } else { + Vec::new() + }; let base = GLOBAL.reset_peak(); let start = Instant::now(); let sealed = if args.planner { let mut planner = Planner::create( dir.path().join("plan.spool"), u64::MAX, + (4096, 1 << 20), + 0, &mut view, &resolver, ); - for first in (0..args.rows).step_by(4096) { - let end = (first + 4096).min(args.rows); - let mut heaps = Vec::with_capacity(end - first); - let mut ordered_events = Vec::new(); - for i in first..end { - if let Some(event) = control(i) { - ordered_events.push(OrderedEvent { - heap_idx: heaps.len(), - row_idx: 0, - event, - }); - } - heaps.push(make_heap(i)); - } - planner - .plan_batch(DrainedBatch { - heaps, - ordered_events, - chunks: Vec::new(), - new_rows: Default::default(), - truncate_rows: Vec::new(), - is_final: end == args.rows, - }) - .await?; + for batch in batches { + planner.plan_batch(batch).await?; } planner.seal(u64::MAX, 0)? } else { - let mut writer = PlanWriter::create(dir.path().join("plan.spool"), u64::MAX, 1 << 20); + let mut writer = PlanWriter::create(dir.path().join("plan.spool"), u64::MAX, 1 << 20) + .with_segment_limits(4096, 1 << 20); for i in 0..args.rows { if let Some(event) = control(i) { view.apply(&event).await.map_err(anyhow::Error::msg)?; - writer.push_control(event, 0); + writer.push_control(event, 0)?; + } + let rel = &descriptors[i % descriptors.len()]; + let table = writer.table(rel, 1, || view.route_for(rel)); + if !writer.routed(table) { + writer.discard(table, 1); + } else if args.cells { + writer.push_cells(make_heap(i).decoded, table)?; + } else { + let input = make_input(i); + let record = input.input.to_xlog_record(rel.rfn); + let images = walshadow::heap_decoder::view_heap_record(&record, rel)?; + let change = ViewChange { + op: images[0].op, + xid: input.input.xid, + source_lsn: input.input.source_lsn, + new: images[0].new.as_ref(), + old: None, + }; + writer.push_views(change, table)?; } - let heap = make_heap(i); - let table = writer.table(&heap, || view.route_for(&heap)); - writer.push_heap(heap.decoded, table)?; } writer.seal(u64::MAX, 0)? }; let write = start.elapsed(); let planned = GLOBAL.snapshot().since(base); let (size_bytes, spilled) = (sealed.size_bytes, sealed.path().is_some()); - let stats = EmitterStats::default(); - let mut staged = Vec::new(); - let mut pending = Vec::with_capacity(4096); - let mut routed = 0; - let mut controls = 0; + let routed = sealed.routed_rows; let base = GLOBAL.reset_peak(); let start = Instant::now(); let mut rd = sealed.replay()?; let mut n = 0; + let mut controls = 0; + let mut segments = Vec::new(); while let Some(item) = rd.next_item()? { - let control = matches!(item, PlanItem::Control(_)); match item { - PlanItem::Heap(h) => { - assert_eq!(h.decoded.source_lsn, n as u64); - assert_eq!(h.decoded.rfn, descriptors[n % descriptors.len()].rfn); - assert_eq!(h.decoded.xid, 7 + (n % 4) as u32); - let columns = &h.decoded.new.as_ref().expect("insert image").columns; - assert!(matches!( - columns.as_slice(), - [Some(ColumnValue::Int4(v)), Some(ColumnValue::Text(s))] - if *v == n as i32 && *s == payload - )); - n += 1; - if args.planner { - pending.push(h); - } - } - PlanItem::Control(c) => { - assert_eq!(c.heap_idx, n); - controls += 1; + PlanItem::Segment(s) => { + n += s.chunks.iter().map(|c| c.rows()).sum::(); + segments.push(s.chunks); } + PlanItem::Control { .. } => controls += 1, PlanItem::Truncate { .. } => unreachable!(), } - if pending.len() == 4096 || (control && !pending.is_empty()) { - routed += place_rows( - &mut staged, - &stats, - 4096, - 0, - 0, - u64::MAX, - &rd.tables, - std::mem::take(&mut pending), - None, - ); - staged.clear(); - } - } - if args.planner { - routed += place_rows( - &mut staged, - &stats, - 4096, - 0, - 0, - u64::MAX, - &rd.tables, - pending, - None, - ); - staged.clear(); - assert_eq!(routed, if args.mapped { args.rows as u64 } else { 0 }); } let replay = start.elapsed(); let replayed = GLOBAL.snapshot().since(base); + let n_segments = segments.len(); + // Encoded chunks match a reference plan encoding decoded rows + if args.mapped { + let mut reference = PlanWriter::create(dir.path().join("ref.spool"), u64::MAX, u64::MAX) + .with_segment_limits(usize::MAX, usize::MAX); + let mut tables = HashMap::new(); + for i in 0..args.rows { + if control(i).is_some() { + tables.clear(); + } + let rel = &descriptors[i % descriptors.len()]; + let table = *tables + .entry(rel.oid) + .or_insert_with(|| reference.table(rel, 1, || view.route.clone())); + reference.push_cells(make_heap(i).decoded, table)?; + } + // Chunk boundaries differ between plans; compare rows per relation + let merged = |chunks: Vec| { + let mut by_rel: HashMap = HashMap::new(); + for c in chunks { + match by_rel.entry(c.rel_oid()) { + std::collections::hash_map::Entry::Occupied(mut e) => { + assert!(e.get_mut().merge(c), "one layout per relation"); + } + std::collections::hash_map::Entry::Vacant(e) => { + e.insert(c); + } + } + } + let mut out: Vec<_> = by_rel.into_iter().map(|(k, c)| (k, c.to_bytes())).collect(); + out.sort(); + out + }; + let mut rrd = reference.seal(u64::MAX, 0)?.replay()?; + let mut want = Vec::new(); + while let Some(PlanItem::Segment(s)) = rrd.next_item()? { + want.extend(s.chunks); + } + assert!( + merged(want) == merged(segments.into_iter().flatten().collect()), + "encoded rows differ from reference" + ); + } assert_eq!(n, if args.mapped { args.rows } else { 0 }); + assert_eq!(routed, n as u64); assert_eq!( controls, args.rows @@ -365,7 +475,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { ); report( if args.planner { - "plan_replay_place" + "plan_replay_encode" } else { "plan_replay" }, @@ -384,10 +494,11 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { + (args.rows % interval).min(descriptors.len()); assert_eq!(view.calls, expected_calls); println!( - "plan_total repeat={repeat} total_s={:.6} route_calls={} tables={} routed={routed} spilled={} plan {planned} replay {replayed}", + "plan_total repeat={repeat} total_s={:.6} route_calls={} tables={} routed={routed} segments={} spilled={} plan {planned} replay {replayed}", (write + replay).as_secs_f64(), view.calls, rd.tables.len(), + n_segments, spilled ); Ok(()) diff --git a/benches/xact_spill.rs b/benches/xact_spill.rs index 7b990d96..d29b6c68 100644 --- a/benches/xact_spill.rs +++ b/benches/xact_spill.rs @@ -10,10 +10,11 @@ use walrus::pg::walparser::{ BlockLocation, RelFileNode, RmId, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, XLogRecordHeader, }; -use walshadow::heap_decoder::ColumnValue; +use walshadow::decode::datum::DatumRef; +use walshadow::heap_decoder::view_heap_record; use walshadow::schema::{RelAttr, RelDescriptor, RelName, ReplIdent, TEXTOID}; use walshadow::spill::{DescribedInput, HeapInput}; -use walshadow::xact_buffer::{StashResolved, XactBuffer, XactBufferConfig}; +use walshadow::xact_buffer::{DrainHeap, StashResolved, XactBuffer, XactBufferConfig}; #[path = "common/alloc_meter.rs"] mod alloc_meter; @@ -152,17 +153,21 @@ async fn run(args: Args) -> anyhow::Result<()> { let mut rows = 0; while let Some(batch) = drain.next_batch(1024, 1 << 20, None).await? { for heap in batch.heaps { - assert_eq!(heap.decoded.xid, xid); + let DrainHeap::Input(input) = heap else { + panic!("ordinary rows drain as inputs"); + }; + assert_eq!(input.input.xid, xid); assert_eq!( - heap.decoded.source_lsn, + input.input.source_lsn, u64::from(xid) * (args.rows as u64 + 1) + rows ); - assert_eq!(heap.descriptor.as_ref(), descriptor.as_ref()); - assert_eq!(heap.descriptor_valid_from, 1); - let tuple = heap.decoded.new.unwrap(); - assert_eq!(tuple.columns.len(), 1); + assert_eq!(input.descriptor.as_ref(), descriptor.as_ref()); + assert_eq!(input.descriptor_valid_from, 1); + let record = input.input.to_xlog_record(descriptor.rfn); + let images = view_heap_record(&record, &descriptor)?; + let datums = &images[0].new.as_ref().expect("insert image").datums; assert!( - matches!(&tuple.columns[0], Some(ColumnValue::Text(s)) if s == &payload) + matches!(datums[..], [DatumRef::Varlena(b)] if b == payload.as_bytes()) ); rows += 1; } diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index 8e863adb..ea7203a3 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -581,73 +581,79 @@ impl WalReplaySink { .await .map_err(SinkError::ctx("toast rewrite barrier"))?; } - WalkStep::Heap(mut heap) => { - let rfn = heap.decoded.rfn; - // Decode TOAST chunks, route through parent row - let Some(target) = targets.get(&(rfn.db_node, rfn.rel_node)) else { - continue; - }; - if commit_lsn <= self.from_lsn { - // Walked pages cover commits through from_lsn - self.replay.commits_below_from += 1; - continue; - } - if commit_lsn > target.through { - self.replay.commits_past_through += 1; - continue; - } - let Some(route) = &target.route else { - self.stats - .unsupported_relations - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - }; - // Skip deletes for append-only destinations - if route.drops_deletes() - && matches!(heap.decoded.op, crate::decode::heap_decoder::HeapOp::Delete) - { - self.stats - .deletes_discarded - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - } - let rel = &target.rel; - // Make earlier commits available before resolving values. - // Inline-only heaps read nothing: flushing for them - // shreds the batch on a row-heavy segment. Gate stays - // as wide as detoast's leaf acquire: narrowing it to - // store-bound pointers would hold pending leaf permits - // across that acquire, which can wait on them - if heap_reads_toast(&heap) { - self.flush_rows().await?; - } - let value_permit = detoast_heap(&mut heap, spool, &ref_maps, &self.resolver) - .await - .map_err(SinkError::from)?; - let (open, rows) = self - .open - .get_or_insert_with(|| (self.seqs.open(commit_lsn, Publish::Commit), 0)); - *rows += 1; - let seq = open.seq(); - if !route.drops_deletes() - && let Some(tomb) = take_key_change_tombstone(&mut heap.decoded, rel) - { + WalkStep::Heap(drained) => { + for mut heap in drained.into_cells().map_err(SinkError::from)? { + let rfn = heap.decoded.rfn; + // Decode TOAST chunks, route through parent row + let Some(target) = targets.get(&(rfn.db_node, rfn.rel_node)) else { + continue; + }; + if commit_lsn <= self.from_lsn { + // Walked pages cover commits through from_lsn + self.replay.commits_below_from += 1; + continue; + } + if commit_lsn > target.through { + self.replay.commits_past_through += 1; + continue; + } + let Some(route) = &target.route else { + self.stats + .unsupported_relations + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + continue; + }; + // Skip deletes for append-only destinations + if route.drops_deletes() + && matches!( + heap.decoded.op, + crate::decode::heap_decoder::HeapOp::Delete + ) + { + self.stats + .deletes_discarded + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + continue; + } + let rel = &target.rel; + // Make earlier commits available before resolving values. + // Inline-only heaps read nothing: flushing for them + // shreds the batch on a row-heavy segment. Gate stays + // as wide as detoast's leaf acquire: narrowing it to + // store-bound pointers would hold pending leaf permits + // across that acquire, which can wait on them + if heap_reads_toast(&heap) { + self.flush_rows().await?; + } + let value_permit = + detoast_heap(&mut heap, spool, &ref_maps, &self.resolver) + .await + .map_err(SinkError::from)?; + let (open, rows) = self.open.get_or_insert_with(|| { + (self.seqs.open(commit_lsn, Publish::Commit), 0) + }); *rows += 1; - let tomb = CommittedTuple { - decoded: tomb, + let seq = open.seq(); + if !route.drops_deletes() + && let Some(tomb) = take_key_change_tombstone(&mut heap.decoded, rel) + { + *rows += 1; + let tomb = CommittedTuple { + decoded: tomb, + commit_ts, + commit_lsn, + }; + self.emit(seq, (rel, route), tomb, None).await?; + } + let committed = CommittedTuple { + decoded: heap.decoded, commit_ts, commit_lsn, }; - self.emit(seq, (rel, route), tomb, None).await?; + self.emit(seq, (rel, route), committed, value_permit.map(Arc::new)) + .await?; + self.replay.rows_replayed += 1; } - let committed = CommittedTuple { - decoded: heap.decoded, - commit_ts, - commit_lsn, - }; - self.emit(seq, (rel, route), committed, value_permit.map(Arc::new)) - .await?; - self.replay.rows_replayed += 1; } } } diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index ce291798..e34c089d 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -592,6 +592,7 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M |s| &s.plan_failures_detoast, |s| &s.plan_failures_partial_update, |s| &s.plan_failures_view, + |s| &s.plan_failures_encode, |s| &s.plan_failures_drain, ], ), diff --git a/src/decode/datum.rs b/src/decode/datum.rs index 5894590c..8eb98d86 100644 --- a/src/decode/datum.rs +++ b/src/decode/datum.rs @@ -18,13 +18,8 @@ pub enum DatumRef<'a> { Fixed(&'a [u8]), /// Uncompressed varlena body Varlena(&'a [u8]), - /// Inline compressed varlena; `datum` spans header and body - Compressed { - method: u8, - raw_len: usize, - body: &'a [u8], - datum: &'a [u8], - }, + /// Inline compressed varlena: 4-byte header, `va_tcinfo`, codec body + Compressed(&'a [u8]), External(ToastPointer), /// `typlen = -2` body without trailing NUL Cstring(&'a [u8]), diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index ca162814..f7146713 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -454,17 +454,17 @@ impl<'a> TupleBuild<'a> for DecodedTuple { } /// Tuple image borrowing its record. Indexed by attnum-1 like -/// [`DecodedTuple::columns`] +/// [`DecodedTuple::columns`]; inline up to 32 attributes #[derive(Clone, Debug, PartialEq)] pub struct TupleView<'a> { - pub datums: Vec>, + pub datums: SmallVec<[DatumRef<'a>; 32]>, pub partial: bool, } impl<'a> TupleBuild<'a> for TupleView<'a> { fn with_capacity(n: usize) -> Self { Self { - datums: Vec::with_capacity(n), + datums: SmallVec::with_capacity(n), partial: false, } } @@ -538,7 +538,7 @@ pub fn decode_heap_record( .map(|b| b.header.location.rel) .unwrap_or_default(); let xid = record.header.xact_id; - Ok(parse_heap_record::(record, rel)? + Ok(parse_heap_record::(record.into(), rel)? .into_iter() .map(|images| DecodedHeap { rfn, @@ -553,10 +553,31 @@ pub fn decode_heap_record( /// [`decode_heap_record`] borrowing attribute bytes from `record` pub fn view_heap_record<'a>( - record: &'a XLogRecord<'_>, + record: impl Into>, rel: &RelDescriptor, ) -> Result>; 1]>, DecodeError> { - parse_heap_record(record, rel) + parse_heap_record(record.into(), rel) +} + +/// Record fields heap decode reads +#[derive(Clone, Copy, Debug)] +pub struct HeapRecordBytes<'a> { + pub rm: u8, + pub info: u8, + pub main_data: &'a [u8], + /// Block 0 registered data + pub block: Option<&'a [u8]>, +} + +impl<'a> From<&'a XLogRecord<'_>> for HeapRecordBytes<'a> { + fn from(record: &'a XLogRecord<'_>) -> Self { + Self { + rm: record.header.resource_manager_id, + info: record.header.info, + main_data: &record.main_data, + block: record.blocks.first().map(|b| &b.data[..]), + } + } } /// Validate `record` as [`decode_heap_record`] would, without building cells @@ -564,15 +585,15 @@ pub fn scan_heap_record( record: &XLogRecord<'_>, rel: &RelDescriptor, ) -> Result; 1]>, DecodeError> { - parse_heap_record(record, rel) + parse_heap_record(record.into(), rel) } fn parse_heap_record<'a, T: TupleBuild<'a>>( - record: &'a XLogRecord<'_>, + record: HeapRecordBytes<'a>, rel: &RelDescriptor, ) -> Result; 1]>, DecodeError> { - let rm = record.header.resource_manager_id; - let info_op = record.header.info & XLOG_HEAP_OPMASK; + let rm = record.rm; + let info_op = record.info & XLOG_HEAP_OPMASK; if rm == RmId::Heap2 as u8 && info_op == XLOG_HEAP2_MULTI_INSERT { return decode_multi_insert(record, rel); } @@ -601,10 +622,10 @@ pub const SIZE_OF_MULTI_INSERT_TUPLE: usize = 7; pub const XLOG_HEAP_INIT_PAGE: u8 = 0x80; fn decode_multi_insert<'a, T: TupleBuild<'a>>( - record: &'a XLogRecord<'_>, + record: HeapRecordBytes<'a>, rel: &RelDescriptor, ) -> Result; 1]>, DecodeError> { - let md = &record.main_data; + let md = record.main_data; if md.len() < SIZE_OF_HEAP_MULTI_INSERT { return Err(DecodeError::Truncated { offset: 0, @@ -628,7 +649,7 @@ fn decode_multi_insert<'a, T: TupleBuild<'a>>( if flags & XLH_INSERT_CONTAINS_NEW_TUPLE == 0 { return Ok(SmallVec::new()); } - let init_page = record.header.info & XLOG_HEAP_INIT_PAGE != 0; + let init_page = record.info & XLOG_HEAP_INIT_PAGE != 0; // Non-init_page: main_data continues with offsets[ntuples] of u16. // Values unneeded (block 0 carries tuples in order); length-check only let expected_md_min = if init_page { @@ -644,10 +665,9 @@ fn decode_multi_insert<'a, T: TupleBuild<'a>>( }); } - let Some(block) = record.blocks.first() else { + let Some(data) = record.block else { return Ok(SmallVec::new()); }; - let data: &'a [u8] = &block.data; let mut cursor = 0usize; let mut out = SmallVec::with_capacity(ntuples); for _ in 0..ntuples { @@ -690,13 +710,13 @@ fn decode_multi_insert<'a, T: TupleBuild<'a>>( } fn decode_insert<'a, T: TupleBuild<'a>>( - record: &'a XLogRecord<'_>, + record: HeapRecordBytes<'a>, rel: &RelDescriptor, ) -> Result, DecodeError> { // PG always references target page; emit empty new rather than fail // the stream - let new = match record.blocks.first() { - Some(block) => Some(decode_tuple_payload(&block.data, 0, rel, 0)?), + let new = match record.block { + Some(data) => Some(decode_tuple_payload(data, 0, rel, 0)?), None => None, }; Ok(HeapImages { @@ -707,7 +727,7 @@ fn decode_insert<'a, T: TupleBuild<'a>>( } fn decode_update<'a, T: TupleBuild<'a>>( - record: &'a XLogRecord<'_>, + record: HeapRecordBytes<'a>, rel: &RelDescriptor, hot: bool, ) -> Result, DecodeError> { @@ -724,8 +744,7 @@ fn decode_update<'a, T: TupleBuild<'a>>( let has_old_tuple = flags & XLH_UPDATE_CONTAINS_OLD_TUPLE != 0; let has_old_key = flags & XLH_UPDATE_CONTAINS_OLD_KEY != 0; - let new = if let Some(b) = record.blocks.first() { - let data: &'a [u8] = &b.data; + let new = if let Some(data) = record.block { let mut cursor = 0; let prefixlen = if has_prefix { read_u16(data, &mut cursor)? as usize @@ -746,7 +765,7 @@ fn decode_update<'a, T: TupleBuild<'a>>( }; let old = if has_old_tuple || has_old_key { - decode_old_tuple_from_main_data(&record.main_data, SIZE_OF_HEAP_UPDATE, rel)? + decode_old_tuple_from_main_data(record.main_data, SIZE_OF_HEAP_UPDATE, rel)? } else { None }; @@ -760,7 +779,7 @@ fn decode_update<'a, T: TupleBuild<'a>>( } fn decode_delete<'a, T: TupleBuild<'a>>( - record: &'a XLogRecord<'_>, + record: HeapRecordBytes<'a>, rel: &RelDescriptor, ) -> Result, DecodeError> { if record.main_data.len() < SIZE_OF_HEAP_DELETE { @@ -773,7 +792,7 @@ fn decode_delete<'a, T: TupleBuild<'a>>( let flags = record.main_data[7]; // xl_heap_delete.flags (after xmax:4 + offnum:2 + infobits:1) let has_old = flags & (XLH_DELETE_CONTAINS_OLD_TUPLE | XLH_DELETE_CONTAINS_OLD_KEY) != 0; let old = if has_old { - decode_old_tuple_from_main_data(&record.main_data, SIZE_OF_HEAP_DELETE, rel)? + decode_old_tuple_from_main_data(record.main_data, SIZE_OF_HEAP_DELETE, rel)? } else { None }; @@ -1195,17 +1214,18 @@ pub fn datum_value(att: &RelAttr, datum: DatumRef<'_>) -> Option { DatumRef::Missing => missing_value_for(att), DatumRef::Fixed(body) => fixed_value(att, body), DatumRef::Varlena(body) => varlena_to_value(att.type_oid, Cow::Borrowed(body)), - DatumRef::Compressed { - method, - raw_len, - body, - datum, - } => decompress_varlena(method, body, raw_len) - .map(|out| varlena_to_value(att.type_oid, Cow::Owned(out))) - .unwrap_or_else(|| ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: datum.to_vec(), - }), + DatumRef::Compressed(datum) => { + // Reader checked datum spans header and va_tcinfo + let tcinfo = u32::from_le_bytes(datum[4..8].try_into().unwrap()); + let method = (tcinfo >> VARLENA_EXTSIZE_BITS) as u8; + let raw_len = (tcinfo & VARLENA_EXTSIZE_MASK) as usize; + decompress_varlena(method, &datum[8..], raw_len) + .map(|out| varlena_to_value(att.type_oid, Cow::Owned(out))) + .unwrap_or_else(|| ColumnValue::Unsupported { + type_oid: att.type_oid, + raw: datum.to_vec(), + }) + } DatumRef::External(p) => ColumnValue::ExternalToast(p), DatumRef::Cstring(body) => simdutf8::basic::from_utf8(body) .map(|s| ColumnValue::Text(s.to_owned())) @@ -1242,7 +1262,7 @@ pub fn datum_view<'a>( type_oid: att.type_oid, raw, }, - DatumRef::Missing | DatumRef::Compressed { .. } => return Err(AllocatingDecode), + DatumRef::Missing | DatumRef::Compressed(_) => return Err(AllocatingDecode), })) } @@ -1421,14 +1441,7 @@ fn read_varlena(buf: &[u8], abs: usize) -> Result<(DatumRef<'_>, usize), Truncat have: total, }); } - let tcinfo = u32::from_le_bytes(buf[abs + 4..abs + 8].try_into().unwrap()); - let datum = DatumRef::Compressed { - method: (tcinfo >> VARLENA_EXTSIZE_BITS) as u8, - raw_len: (tcinfo & VARLENA_EXTSIZE_MASK) as usize, - body: &buf[abs + 8..abs + total], - datum: &buf[abs..abs + total], - }; - return Ok((datum, total)); + return Ok((DatumRef::Compressed(&buf[abs..abs + total]), total)); } Ok((DatumRef::Varlena(&buf[abs + 4..abs + total]), total)) } diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 8b8dac0b..ddab5434 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -34,13 +34,10 @@ use clickhouse_c::{Allocator, ColumnBuilder, Kind, TypeAst}; use crate::ch::is_retryable; use crate::ch::{CompressionChoice, ConnectionConfig, EmitterError, quote_ident, types}; use crate::column_rules::{ColumnEntry, ColumnRule, ColumnRules, Substitute}; -#[cfg(test)] -use crate::decode::datum::DatumRef; -use crate::decode::datum::ValueRef; +use crate::decode::datum::{DatumRef, ValueRef}; #[cfg(test)] use crate::decode::decoder_sink::DecoderSinkError; use crate::decode::heap_decoder::{ColumnValue, CommittedTuple, HeapOp}; -#[cfg(test)] use crate::decode::heap_decoder::{datum_value, datum_view}; use crate::mapping::{ ColumnMapping, DropTableStrategy, NamespaceMapping, SystemColumnNames, SystemColumns, @@ -2036,6 +2033,189 @@ impl ColumnBuf { } } + /// Same wire layout, so `other` can follow these rows + fn same_shape(&self, other: &Self) -> bool { + match (self, other) { + (Self::Fixed { width: a, .. }, Self::Fixed { width: b, .. }) + | (Self::NullableFixed { width: a, .. }, Self::NullableFixed { width: b, .. }) => { + a == b + } + (Self::String { absent: a, .. }, Self::String { absent: b, .. }) + | (Self::NullableString { absent: a, .. }, Self::NullableString { absent: b, .. }) => { + a == b + } + (Self::Oracle(a), Self::Oracle(b)) => a.same_column(b), + _ => false, + } + } + + /// Append `other`'s rows; caller checked [`Self::same_shape`] + fn extend(&mut self, other: Self) { + let rebase = |offsets: &mut Vec, base: usize, more: Vec| { + offsets.extend(more.into_iter().map(|o| o + base as u64)); + }; + match (self, other) { + (Self::Fixed { bytes, .. }, Self::Fixed { bytes: more, .. }) => { + bytes.extend_from_slice(&more); + } + ( + Self::String { offsets, data, .. }, + Self::String { + offsets: more, + data: more_data, + .. + }, + ) => { + rebase(offsets, data.len(), more); + data.extend_from_slice(&more_data); + } + ( + Self::NullableFixed { + null_map, inner, .. + }, + Self::NullableFixed { + null_map: more_nulls, + inner: more, + .. + }, + ) => { + null_map.extend_from_slice(&more_nulls); + inner.extend_from_slice(&more); + } + ( + Self::NullableString { + offsets, + data, + null_map, + .. + }, + Self::NullableString { + offsets: more, + data: more_data, + null_map: more_nulls, + .. + }, + ) => { + rebase(offsets, data.len(), more); + data.extend_from_slice(&more_data); + null_map.extend_from_slice(&more_nulls); + } + (Self::Oracle(a), Self::Oracle(b)) => a.append(b), + _ => unreachable!("chunk shape checked before merge"), + } + } + + pub(crate) fn encode_into(&self, out: &mut Vec) { + use crate::xact::spill::{push_bytes, push_u8, push_u32}; + let offsets = |out: &mut Vec, offsets: &[u64]| { + push_u32(out, offsets.len() as u32); + for o in offsets { + out.extend_from_slice(&o.to_le_bytes()); + } + }; + match self { + Self::Fixed { width, bytes } => { + push_u8(out, 0); + push_u32(out, *width as u32); + push_bytes(out, bytes); + } + Self::String { + offsets: o, + data, + absent, + } => { + push_u8(out, 1); + push_u8(out, u8::from(!absent.is_empty())); + offsets(out, o); + push_bytes(out, data); + } + Self::NullableFixed { + width, + null_map, + inner, + } => { + push_u8(out, 2); + push_u32(out, *width as u32); + push_bytes(out, null_map); + push_bytes(out, inner); + } + Self::NullableString { + offsets: o, + data, + null_map, + absent, + } => { + push_u8(out, 3); + push_u8(out, u8::from(!absent.is_empty())); + offsets(out, o); + push_bytes(out, data); + push_bytes(out, null_map); + } + Self::Oracle(o) => { + push_u8(out, 4); + o.encode_into(out); + } + } + } + + pub(crate) fn decode( + c: &mut crate::xact::spill::Cursor<'_>, + ) -> Result { + let absent = |c: &mut crate::xact::spill::Cursor<'_>| { + Ok::<&'static [u8], crate::xact::spill::SpillError>(if c.u8()? != 0 { + b"{}" + } else { + b"" + }) + }; + let offsets = |c: &mut crate::xact::spill::Cursor<'_>| { + let n = c.u32()? as usize; + Ok::<_, crate::xact::spill::SpillError>( + c.need(n * 8)? + .as_chunks::<8>() + .0 + .iter() + .map(|b| u64::from_le_bytes(*b)) + .collect::>(), + ) + }; + Ok(match c.u8()? { + 0 => Self::Fixed { + width: c.u32()? as usize, + bytes: c.bytes()?, + }, + 1 => { + let absent = absent(c)?; + Self::String { + offsets: offsets(c)?, + data: c.bytes()?, + absent, + } + } + 2 => Self::NullableFixed { + width: c.u32()? as usize, + null_map: c.bytes()?, + inner: c.bytes()?, + }, + 3 => { + let absent = absent(c)?; + Self::NullableString { + offsets: offsets(c)?, + data: c.bytes()?, + null_map: c.bytes()?, + absent, + } + } + 4 => Self::Oracle(OracleColumnBuf::decode(c)?), + tag => { + return Err(crate::xact::spill::SpillError::Format { + offset: 0, + detail: format!("unknown column buffer tag {tag}"), + }); + } + }) + } + fn append_string_bytes(&mut self, raw: &[u8]) -> Result<(), EmitterError> { match self { Self::String { offsets, data, .. } => { @@ -2137,6 +2317,50 @@ impl TableEncoder { Ok((old, rows)) } + /// Whether `block` lays out the same columns as these buffers + pub(crate) fn accepts(&self, block: &[ColumnBuf]) -> bool { + block.len() == self.buffers.len() + && self.buffers.iter().zip(block).all(|(a, b)| a.same_shape(b)) + } + + /// Append rows another encoder of this shape sealed. Leaves `block` + /// untouched on [`Append::Full`], empties it otherwise + pub(crate) fn append_block( + &mut self, + block: &mut Vec, + rows: usize, + ) -> Result { + if !self.accepts(block) { + return Err(EmitterError::Type( + "encoded chunk layout differs from its batch".into(), + )); + } + let oracle_bytes = self.oracle_overhead + + self + .buffers + .iter() + .zip(block.iter()) + .map(|(a, b)| match (a, b) { + (ColumnBuf::Oracle(a), ColumnBuf::Oracle(b)) => a.request_bytes_with(b), + _ => 0, + }) + .sum::(); + if self.rows > 0 && oracle_bytes > ORACLE_BATCH_SEAL_BYTES { + return Ok(Append::Full); + } + if self.rows == 0 { + std::mem::swap(&mut self.buffers, block); + } else { + for (buf, more) in self.buffers.iter_mut().zip(block.drain(..)) { + buf.extend(more); + } + } + self.rows += rows; + self.oracle_bytes = oracle_bytes; + self.approx_bytes = self.buffers.iter().map(ColumnBuf::approx_size).sum(); + Ok(Append::Done) + } + #[cfg(test)] pub fn append_row( &mut self, @@ -2182,7 +2406,6 @@ impl TableEncoder { /// [`Self::append_routed`] reading one image of `rel` in place. Values /// whose decode allocates, and PostGIS renders, build in row scratch - #[cfg(test)] pub(crate) fn append_view( &mut self, rel: &RelDescriptor, @@ -2290,7 +2513,6 @@ impl TableEncoder { buf.append_default(); Ok(()) } - #[cfg(test)] Cell::Le(le) => buf.append_fixed_bytes(le), Cell::Value(v) => encode_value( buf, @@ -2335,10 +2557,82 @@ pub(crate) enum Cell<'a> { Absent, Value(ValueRef<'a>), /// Little-endian bytes [`Direct::Copy`] proved match column storage - #[cfg(test)] Le(&'a [u8]), } +/// Rows of one plan table encoded at planning. Relation and route travel +/// once per chunk so the batcher keys and plans its destination +pub struct EncodedChunk { + pub(crate) rel: Arc, + pub(crate) route: Arc, + pub(crate) buffers: Vec, + pub(crate) rows: usize, +} + +impl EncodedChunk { + pub fn rows(&self) -> usize { + self.rows + } + + pub fn allocated_bytes(&self) -> usize { + self.buffers.iter().map(ColumnBuf::allocated_bytes).sum() + } + + pub fn rel_oid(&self) -> u32 { + self.rel.oid + } + + /// Append `other`'s rows; `false` leaves both untouched when layouts differ + pub fn merge(&mut self, other: Self) -> bool { + let same = self.buffers.len() == other.buffers.len() + && self + .buffers + .iter() + .zip(&other.buffers) + .all(|(a, b)| a.same_shape(b)); + if same { + for (buf, more) in self.buffers.iter_mut().zip(other.buffers) { + buf.extend(more); + } + self.rows += other.rows; + } + same + } + + /// Wire content as a plan spool frame body + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::new(); + self.encode_into(&mut out); + out + } + + pub(crate) fn encode_into(&self, out: &mut Vec) { + crate::xact::spill::push_u32(out, self.rows as u32); + crate::xact::spill::push_u32(out, self.buffers.len() as u32); + for b in &self.buffers { + b.encode_into(out); + } + } + + pub(crate) fn decode( + rel: Arc, + route: Arc, + c: &mut crate::xact::spill::Cursor<'_>, + ) -> Result { + let rows = c.u32()? as usize; + let n = c.u32()? as usize; + let buffers = (0..n) + .map(|_| ColumnBuf::decode(c)) + .collect::>()?; + Ok(Self { + rel, + route, + buffers, + rows, + }) + } +} + /// Per-row synthetic column inputs pub(crate) struct RowStamp { pub(crate) source_lsn: u64, @@ -2799,7 +3093,6 @@ fn oracle_cell(cell: Cell<'_>, source_type_oid: u32) -> Result return Ok(OracleCell::Default), Cell::Value(v) => v, - #[cfg(test)] Cell::Le(_) => return Err(mismatch(0)), }; Ok(match v { @@ -2931,6 +3224,8 @@ crate::atomic_stats! { pub plan_failures_detoast, pub plan_failures_partial_update, pub plan_failures_view, + /// Row values a mapped column cannot encode + pub plan_failures_encode, pub plan_failures_drain, pub unmapped_rows_by_table: crate::emit::pipeline::row_ledger::TableRowCounts, pub rows_inserted_by_table: crate::emit::pipeline::row_ledger::TableRowCounts, diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs index e3980107..6eb72840 100644 --- a/src/emit/direct_parity.rs +++ b/src/emit/direct_parity.rs @@ -831,3 +831,122 @@ fn view_matches_owned_decode_for_records() { Some(ColumnValue::Text("value".into())) ); } + +/// Operation with new and old tuple bytes +type Change = (HeapOp, Option>, Option>); + +/// Plan row changes both ways; segments must carry identical rows +fn plan_both(rel: &RelDescriptor, targets: &[&str], changes: &[Change]) { + use crate::emit::pipeline::plan_spool::{PlanItem, PlanWriter, ViewChange}; + use crate::emit::route::RouteSnapshot; + let rel = std::sync::Arc::new(rel.clone()); + let route = RouteSnapshot::freeze( + std::sync::Arc::new(mapping_of(&rel, targets)), + std::sync::Arc::default(), + Default::default(), + ); + let tmp = tempfile::tempdir().unwrap(); + let mut out = Vec::new(); + for views in [false, true] { + let mut w = PlanWriter::create(tmp.path().join(format!("{views}.plan")), 1 << 20, 1 << 20) + .for_commit(9); + let table = w.table(&rel, 1, || Some(route.clone())); + for (i, (op, new, old)) in changes.iter().enumerate() { + let lsn = 0x1000 + 0x10 * i as u64; + if views { + let new = new.as_ref().map(|d| view_block_data(d, &rel).unwrap()); + let old = old.as_ref().map(|d| view_block_data(d, &rel).unwrap()); + let change = ViewChange { + op: *op, + xid: 5, + source_lsn: lsn, + new: new.as_ref(), + old: old.as_ref(), + }; + w.push_views(change, table).unwrap(); + } else { + let decoded = DecodedHeap { + rfn: rel.rfn, + xid: 5, + source_lsn: lsn, + op: *op, + new: new.as_ref().map(|d| decode_block_data(d, &rel).unwrap()), + old: old.as_ref().map(|d| decode_block_data(d, &rel).unwrap()), + }; + w.push_cells(decoded, table).unwrap(); + } + } + let mut rd = w.seal(0x2000, 9).unwrap().replay().unwrap(); + let mut bytes = Vec::new(); + while let Some(item) = rd.next_item().unwrap() { + if let PlanItem::Segment(s) = item { + bytes.extend(s.chunks.iter().map(|c| (c.rows(), c.to_bytes()))); + } + } + out.push(bytes); + } + assert_eq!(out[0], out[1]); + assert!(!out[0].is_empty()); +} + +#[test] +fn plan_views_match_cells_for_row_changes() { + let int4 = |v: i32| v.to_le_bytes(); + let f8 = |v: f64| v.to_le_bytes(); + let txt = |s: &str| short(s.as_bytes()); + for replident in [ + ReplIdent::Full { pk_attnums: None }, + ReplIdent::Default { + pk_attnums: Some(vec![1]), + }, + ] { + let mut r = rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, FLOAT8OID, 8, 'd', "float8"), + attr(3, TEXTOID, -1, 'i', "text"), + ]); + r.replident = replident; + let attrs = r.attributes.clone(); + let row = |id: i32, f: f64, t: &str| { + tuple( + &attrs, + 3, + &[Some(&int4(id)[..]), Some(&f8(f)[..]), Some(&txt(t)[..])], + ) + }; + let nan = f64::from_bits(0x7ff8_0000_0000_0001); + plan_both( + &r, + &["Int32", "Float64", "String"], + &[ + (HeapOp::Insert, Some(row(1, 1.0, "a")), None), + // Key change splits into delete marker plus new row + ( + HeapOp::Update, + Some(row(2, 1.0, "a")), + Some(row(1, 1.0, "a")), + ), + // Same key, other columns change + ( + HeapOp::Update, + Some(row(2, 2.0, "b")), + Some(row(2, 1.0, "a")), + ), + // NaN never equals itself + ( + HeapOp::Update, + Some(row(2, nan, "b")), + Some(row(2, nan, "b")), + ), + ( + HeapOp::HotUpdate, + Some(row(2, 3.0, "c")), + Some(row(2, nan, "b")), + ), + (HeapOp::Update, Some(row(3, 3.0, "c")), None), + (HeapOp::Delete, None, Some(row(3, 3.0, "c"))), + (HeapOp::Delete, None, None), + ], + ); + } +} diff --git a/src/emit/pipeline/batcher.rs b/src/emit/pipeline/batcher.rs index 1c58eb69..93613805 100644 --- a/src/emit/pipeline/batcher.rs +++ b/src/emit/pipeline/batcher.rs @@ -32,15 +32,19 @@ use tokio::time::Instant; use crate::config::ResolvedConfig; use crate::decode::heap_decoder::{CommittedTuple, HeapOp}; use crate::emit::ch_emitter::{ - Append, ColumnBuf, ColumnEncoding, EmitterStats, OP_DELETE, OP_INSERT, OP_UPDATE, TableEncoder, - TablePlan, + Append, ColumnBuf, ColumnEncoding, EmitterStats, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, + TableEncoder, TablePlan, }; -use crate::emit::pipeline::decode::DECODE_CHUNK_BYTES; use crate::emit::pipeline::{DEFAULT_PIPELINE_FLUSH, Fatal}; use crate::emit::route::RouteSnapshot; use crate::schema::{RelDescriptor, TableKey}; use foldhash::HashMap; +/// Byte half of dual trigger with configured row cap on one batcher hop. +/// Bounds channel items for fat detoasted rows that would pin many MiB +/// before the row cap fires +pub const DECODE_CHUNK_BYTES: usize = 4 << 20; + /// Source relation and route captured during planning pub struct RowTable { pub rel: Arc, @@ -161,6 +165,14 @@ impl RowBuffer { } } +/// Planned segment placed under its seq +pub struct PlacedSegment { + pub seq: u64, + pub chunks: Vec, + /// Released post-insert-ack, when every covering `InsertBatch` drops + pub permit: Option>, +} + #[derive(Debug, thiserror::Error)] #[error("batcher channel closed")] pub struct BatcherClosed; @@ -243,13 +255,13 @@ pub(crate) struct InsertBatch { /// process ahead of rows enqueued before it — else flush seals a partial set /// and falsely signals "earlier data sealed", pinning the durability wait. pub enum BatcherMsg { - /// Chunk of rows from one placement (see `decode::DECODE_CHUNK_BYTES`), + /// Chunk of rows from one placement (see [`DECODE_CHUNK_BYTES`]), /// amortizing the per-row channel-send + cross-thread wakeup — the /// dominant coordination cost under sustained load. Rows may carry /// different `seq`s; batcher routes each independently. Rows(RowChunk), - /// Placements staged across commits, one hop for the lot - Chunks(Vec), + /// Planned segments staged across commits, one hop for the lot + Segments(Vec), /// Seal every open table, push to inserters, reply. Barrier (drain before /// DDL/TRUNCATE) and shutdown. FlushAll(oneshot::Sender<()>), @@ -292,16 +304,7 @@ impl Tables { Entry::Occupied(e) => return Ok(*e.get()), Entry::Vacant(e) => e, }; - // Read column overrides from route captured during planning. - // Configuration barrier clears cached plans before applying column changes - let plan = TablePlan::for_route(ctx.alloc, &t.rel, &t.route).map_err(|e| e.to_string())?; - let meta = Arc::new(BatchMeta::from_plan( - &plan, - e.key().clone(), - ctx.epoch, - ctx.stats, - )); - let enc = TableEncoder::new(plan).map_err(|e| e.to_string())?; + let (enc, meta) = table_encoder(ctx, e.key().clone(), t)?; self.open.push(Table { enc, allocated_bytes: 0, @@ -315,6 +318,19 @@ impl Tables { } } +/// Read column overrides from route captured during planning. +/// Configuration barrier clears cached plans before applying column changes +fn table_encoder( + ctx: &RowCtx<'_>, + key: TableKey, + t: &RowTable, +) -> Result<(TableEncoder, Arc), String> { + let plan = TablePlan::for_route(ctx.alloc, &t.rel, &t.route).map_err(|e| e.to_string())?; + let meta = Arc::new(BatchMeta::from_plan(&plan, key, ctx.epoch, ctx.stats)); + let enc = TableEncoder::new(plan).map_err(|e| e.to_string())?; + Ok((enc, meta)) +} + struct BatchOutput { sender: async_channel::Sender, budget: crate::budget::MemoryBudget, @@ -376,12 +392,12 @@ pub(crate) fn spawn( break; } } - Some(BatcherMsg::Chunks(chunks)) => { + Some(BatcherMsg::Segments(segments)) => { let live = effective_cfg(&cfg, snapshot(config_rx.as_ref()).as_deref()); let ctx = RowCtx { cfg: live, out: &out, alloc, epoch, stats }; let mut res = Ok(()); - for chunk in chunks { - res = handle_rows(&mut tables, &ctx, chunk).await; + for segment in segments { + res = handle_segment(&mut tables, &ctx, segment).await; if res.is_err() { break; } @@ -503,6 +519,72 @@ async fn handle_row( .append_routed(&row.committed, &route.mapping, op, leaf) .map_err(|e| e.to_string())?; } + if let Some(p) = row.value_permit + && t.value_permits + .last() + .is_none_or(|held| !Arc::ptr_eq(held, &p)) + { + t.value_permits.push(p); + } + appended(tables, ctx, slot, row.seq, 1, slice_permit).await +} + +/// Merge each chunk of a planned segment into its destination's batch +async fn handle_segment( + tables: &mut Tables, + ctx: &RowCtx<'_>, + segment: PlacedSegment, +) -> Result<(), String> { + for mut chunk in segment.chunks { + ctx.stats + .insertbatch_rows_in + .fetch_add(chunk.rows as u64, Ordering::Relaxed); + let rt = RowTable { + rel: chunk.rel.clone(), + route: chunk.route.clone(), + }; + let slot = tables.slot(ctx, &rt)?; + let t = &mut tables.open[slot]; + if !t.enc.accepts(&chunk.buffers) { + // Same destination, other layout: seal what it holds first + emit_batch(t, ctx.out, ctx.stats).await?; + (t.enc, t.meta) = table_encoder(ctx, t.meta.table_key.clone(), &rt)?; + } + let rows = chunk.rows; + if t.enc + .append_block(&mut chunk.buffers, rows) + .map_err(|e| e.to_string())? + == Append::Full + { + emit_batch(t, ctx.out, ctx.stats).await?; + t.enc + .append_block(&mut chunk.buffers, rows) + .map_err(|e| e.to_string())?; + } + appended( + tables, + ctx, + slot, + segment.seq, + rows as u64, + segment.permit.as_ref(), + ) + .await?; + } + Ok(()) +} + +/// Account `rows` of `seq` just appended to `slot`, sealing what limits +/// call for +async fn appended( + tables: &mut Tables, + ctx: &RowCtx<'_>, + slot: usize, + seq: u64, + rows: u64, + slice_permit: Option<&Arc>, +) -> Result<(), String> { + let t = &mut tables.open[slot]; // Rows sharing a permit arrive consecutively, so check only last entry. // Sealing clears permit lists so each batch retains its own permits if let Some(p) = slice_permit @@ -512,16 +594,9 @@ async fn handle_row( { t.slice_permits.push(p.clone()); } - if let Some(p) = row.value_permit - && t.value_permits - .last() - .is_none_or(|held| !Arc::ptr_eq(held, &p)) - { - t.value_permits.push(p); - } match t.seq_counts.last_mut() { - Some((s, c)) if *s == row.seq => *c += 1, - _ => t.seq_counts.push((row.seq, 1)), + Some((s, c)) if *s == seq => *c += rows, + _ => t.seq_counts.push((seq, rows)), } if t.deadline.is_none() { t.deadline = Some(Instant::now() + ctx.cfg.flush_timeout); @@ -1284,6 +1359,109 @@ mod tests { assert!(!fatal.is_set()); } + /// Rows encoded at planning, one chunk per segment + fn segment( + seq: u64, + rel: &Arc, + route: &Arc, + rows: Vec, + ) -> BatcherMsg { + let plan = + TablePlan::for_route(Allocator::global(&mimalloc::MiMalloc), rel, route).unwrap(); + let mut enc = TableEncoder::new(plan).unwrap(); + for r in &rows { + assert_eq!( + enc.append_routed(&r.committed, &route.mapping, OP_INSERT, None) + .unwrap(), + Append::Done + ); + } + let (buffers, rows) = enc.take_block().unwrap(); + BatcherMsg::Segments(vec![PlacedSegment { + seq, + chunks: vec![EncodedChunk { + rel: rel.clone(), + route: route.clone(), + buffers, + rows, + }], + permit: None, + }]) + } + + fn text_row(seq: u64, id: i32, text: &str) -> RoutedRow { + let mut r = tuple(seq, id); + r.committed.decoded.new.as_mut().unwrap().columns = + vec![Some(ColumnValue::Text(text.into()))]; + r + } + + /// Chunks of several transactions merge into one batch with string + /// offsets rebased, matching rows batched one at a time + #[tokio::test] + async fn segments_merge_like_rows() { + let (rel, route) = text_table("Nullable(String)"); + let texts = [(0, "a"), (0, ""), (1, "ccc"), (2, "dd")]; + let mut wires = Vec::new(); + for chunked in [false, true] { + let (tx, rx, fatal, handle) = + spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); + if chunked { + for seq in 0..3 { + let rows = texts + .iter() + .enumerate() + .filter(|(_, (s, _))| *s == seq) + .map(|(i, (s, t))| text_row(*s, i as i32, t)) + .collect(); + tx.send(segment(seq, &rel, &route, rows)).await.unwrap(); + } + } else { + for (i, (s, t)) in texts.iter().enumerate() { + tx.send(one(rel.clone(), route.clone(), text_row(*s, i as i32, t))) + .await + .unwrap(); + } + } + drop(tx); + let b = rx.recv().await.unwrap(); + assert!(rx.recv().await.is_err(), "one batch"); + assert_eq!(b.n_rows, 4); + assert_eq!(b.per_seq, [(0, 2), (1, 1), (2, 1)]); + wires.push( + b.buffers + .iter() + .map(|c| format!("{:?}", c.wire())) + .collect::>(), + ); + handle.await.unwrap(); + assert!(!fatal.is_set()); + } + assert_eq!(wires[0], wires[1]); + } + + /// Chunk that would push oracle cells past the seal threshold opens the + /// next batch whole + #[tokio::test] + async fn oracle_seal_moves_chunk_to_next_batch() { + let (tx, rx, fatal, handle) = + spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); + let big = "x".repeat(crate::ops::oracle::ORACLE_BATCH_SEAL_BYTES * 3 / 5); + let (rel, route) = text_table("Array(Nullable(String))"); + for seq in 0..2 { + tx.send(segment(seq, &rel, &route, vec![text_row(seq, 1, &big)])) + .await + .unwrap(); + } + drop(tx); + let per_seq: Vec<_> = [rx.recv().await.unwrap(), rx.recv().await.unwrap()] + .map(|b| b.per_seq) + .into(); + assert_eq!(per_seq, [vec![(0, 1)], vec![(1, 1)]]); + handle.await.unwrap(); + assert!(!fatal.is_set()); + } + /// Row that would push oracle cells past the seal threshold opens the next batch #[tokio::test] async fn oracle_seal_moves_row_to_next_batch() { @@ -1450,9 +1628,6 @@ mod tests { let truncate = chunk(with_op(HeapOp::Truncate)); let msg = batcher_fatal(vec![BatcherMsg::Rows(truncate)], true, false).await; assert_eq!(msg, "batcher: TRUNCATE routed to batcher"); - let chunks = vec![chunk(tuple(0, 1)), chunk(with_op(HeapOp::Truncate))]; - let msg = batcher_fatal(vec![BatcherMsg::Chunks(chunks)], true, false).await; - assert_eq!(msg, "batcher: TRUNCATE routed to batcher"); let (rel, route) = date_table("Date32"); let mut overflow = tuple(0, 1); diff --git a/src/emit/pipeline/decode.rs b/src/emit/pipeline/decode.rs deleted file mode 100644 index d7778e82..00000000 --- a/src/emit/pipeline/decode.rs +++ /dev/null @@ -1,319 +0,0 @@ -//! Row placement: turn one planned slice into batcher row chunks -//! -//! Planning already detoasted and routed every heap, so reorder places -//! each dispatched seq inline: narrow local extensions render here, -//! deletes drop for destinations without a delete marker, and rows chunk -//! into the caller's staging list, shipped onto the shared batcher channel -//! in dispatch order. Oracle resolution happens later, in the inserter. - -use std::sync::Arc; -use std::sync::atomic::Ordering; - -use crate::decode::heap_decoder::{CommittedTuple, HeapOp, take_key_change_tombstone}; -use crate::emit::ch_emitter::EmitterStats; -use crate::emit::pipeline::batcher::{RoutedRow, RowChunk, RowTable}; -use crate::emit::pipeline::plan_spool::{PlanTable, PlannedHeap}; - -/// Byte half of dual trigger with configured row cap. Bounds channel item for -/// fat detoasted rows that would pin many MiB before row cap fires; above -/// ~100 KiB an ordinary row-cap chunk reaches, so steady-state coalescing is -/// unchanged. -pub const DECODE_CHUNK_BYTES: usize = 4 << 20; - -/// Chunk every planned heap of one seq onto `staged`. Returns rows routed -/// (the `R` the collector compares against). -/// -/// Caller ships `staged` before the seq's `Placed` and any later -/// `FlushAll`. -#[allow(clippy::too_many_arguments)] -pub fn place_rows( - staged: &mut Vec, - stats: &EmitterStats, - chunk_rows: usize, - seq: u64, - commit_ts: i64, - commit_lsn: u64, - tables: &[PlanTable], - heaps: Vec, - permit: Option>, -) -> u64 { - stats.decode_jobs_in.fetch_add(1, Ordering::Relaxed); - let mut routed = 0u64; - let mut heaps = heaps.into_iter(); - let mut chunk = RowChunk { - rows: Vec::with_capacity(chunk_rows.min(heaps.len())), - ..Default::default() - }; - let mut chunk_bytes = 0usize; - let mut slots = vec![u32::MAX; tables.len()]; - while let Some(heap) = heaps.next() { - let PlanTable { - descriptor: rel, - route, - .. - } = &tables[heap.table as usize]; - let route = route.as_ref().expect("planning discards unmapped heaps"); - // No delete-marker column: a DELETE would land as a phantom insert of - // the old image, so drop it (append-only destination) - if route.drops_deletes() && matches!(heap.decoded.op, HeapOp::Delete) { - stats.deletes_discarded.fetch_add(1, Ordering::Relaxed); - continue; - } - // Ext render and tombstone split leave payload roughly unchanged - chunk_bytes += heap.bytes; - let slot = &mut slots[heap.table as usize]; - if *slot == u32::MAX { - *slot = chunk.tables.len() as u32; - chunk.tables.push(RowTable { - rel: rel.clone(), - route: route.clone(), - }); - } - let table = *slot; - let mut committed = CommittedTuple { - decoded: heap.decoded, - commit_ts, - commit_lsn, - }; - // PostGIS WKT differs from typoutput HEXEWKB - if let Some(t) = committed.decoded.new.as_mut() { - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); - } - if let Some(t) = committed.decoded.old.as_mut() { - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); - } - if !route.drops_deletes() - && let Some(tomb) = take_key_change_tombstone(&mut committed.decoded, rel) - { - let tomb = CommittedTuple { - decoded: tomb, - ..committed - }; - chunk.rows.push(RoutedRow { - seq, - table, - committed: tomb, - value_permit: None, - }); - routed += 1; - } - chunk.rows.push(RoutedRow { - seq, - table, - committed, - value_permit: None, - }); - routed += 1; - if chunk.rows.len() >= chunk_rows || chunk_bytes >= DECODE_CHUNK_BYTES { - let next = RowChunk { - rows: Vec::with_capacity(chunk_rows.min(heaps.len())), - ..Default::default() - }; - staged.push(RowChunk { - permit: permit.clone(), - ..std::mem::replace(&mut chunk, next) - }); - slots.fill(u32::MAX); - chunk_bytes = 0; - } - } - if !chunk.rows.is_empty() { - chunk.permit = permit; - staged.push(chunk); - } - stats.decode_rows_out.fetch_add(routed, Ordering::Relaxed); - routed -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple}; - use crate::emit::route::{RouteSnapshot, RowPolicy}; - use crate::mapping::{ColumnMapping, SystemColumns, TableMapping, TableTarget}; - use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; - use walrus::pg::walparser::RelFileNode; - - const RFN: RelFileNode = RelFileNode { - spc_node: 1663, - db_node: 5, - rel_node: 16385, - }; - - fn rel() -> Arc { - Arc::new(RelDescriptor { - rfn: RFN, - oid: 16385, - toast_oid: 0, - namespace_oid: 2200, - rel_name: RelName::new("public", "t"), - kind: 'r', - persistence: 'p', - ancestors: Vec::new(), - replident: ReplIdent::Default { pk_attnums: None }, - attributes: vec![RelAttr { - attnum: 1, - name: "id".into(), - type_oid: 23, - typmod: -1, - not_null: true, - dropped: false, - type_name: "int4".into(), - type_byval: true, - type_len: 4, - type_align: 'i', - type_storage: 'p', - missing_default: None, - }], - }) - } - - fn heap(op: HeapOp) -> PlannedHeap { - let tuple = Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(1))], - partial: false, - }); - let (new, old) = match op { - HeapOp::Delete => (None, tuple), - _ => (tuple, None), - }; - let decoded = DecodedHeap { - rfn: RFN, - xid: 7, - source_lsn: 0x1000, - op, - new, - old, - }; - PlannedHeap { - bytes: decoded.approx_bytes(), - decoded, - table: 0, - } - } - - fn table(descriptor: Arc, route: Arc) -> [PlanTable; 1] { - [PlanTable { - descriptor, - valid_from: 0x40, - route: Some(route), - heaps: 0, - }] - } - - fn route(system: SystemColumns) -> Arc { - RouteSnapshot::freeze( - Arc::new(TableMapping { - target: TableTarget::new("default", "t"), - columns: vec![ColumnMapping { - src_attnum: 1, - target_name: "id".into(), - target_type: "Int32".into(), - type_pinned: false, - }], - tees: Vec::new(), - }), - Arc::default(), - RowPolicy { - soft_delete: false, - system: Arc::new(system), - }, - ) - } - - /// Without a delete-marker column a DELETE has nowhere to land: it must - /// drop here, before the placed count the ack collector reconciles - #[tokio::test] - async fn deletes_drop_when_marker_disabled() { - let mut staged = Vec::new(); - let stats = EmitterStats::default(); - let no_marker = route(SystemColumns { - is_deleted: None, - ..SystemColumns::default() - }); - let routed = place_rows( - &mut staged, - &stats, - 8, - 0, - 0, - 0x2000, - &table(rel(), no_marker), - vec![heap(HeapOp::Insert), heap(HeapOp::Delete)], - None, - ); - assert_eq!(routed, 1, "insert routed, delete dropped"); - assert_eq!(stats.deletes_discarded.load(Ordering::Relaxed), 1); - assert!(matches!(&staged[..], [c] if c.rows.len() == 1)); - - // Default policy keeps the marker, so the DELETE rides through - let marked = route(SystemColumns::default()); - let routed = place_rows( - &mut staged, - &stats, - 8, - 1, - 0, - 0x3000, - &table(rel(), marked), - vec![heap(HeapOp::Delete)], - None, - ); - assert_eq!(routed, 1); - assert_eq!(stats.deletes_discarded.load(Ordering::Relaxed), 1); - } - - /// Emit and count old-key delete marker before new row unless deletes are disabled - #[tokio::test] - async fn key_change_places_tombstone_first() { - let stats = EmitterStats::default(); - let keyed = Arc::new(RelDescriptor { - replident: ReplIdent::Default { - pk_attnums: Some(vec![1]), - }, - ..(*rel()).clone() - }); - let update = || { - let mut h = heap(HeapOp::Update); - h.decoded.old = Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(0))], - partial: false, - }); - h - }; - for (system, placed) in [ - (SystemColumns::default(), 2), - ( - SystemColumns { - is_deleted: None, - ..SystemColumns::default() - }, - 1, - ), - ] { - let mut staged = Vec::new(); - let routed = place_rows( - &mut staged, - &stats, - 8, - 0, - 0, - 0x2000, - &table(keyed.clone(), route(system)), - vec![update()], - None, - ); - assert_eq!(routed, placed); - let [c] = &staged[..] else { - panic!("one rows chunk"); - }; - let ops: Vec<_> = c.rows.iter().map(|r| r.committed.decoded.op).collect(); - let want: &[HeapOp] = if placed == 2 { - &[HeapOp::Delete, HeapOp::Update] - } else { - &[HeapOp::Update] - }; - assert_eq!(ops, want); - } - } -} diff --git a/src/emit/pipeline/mod.rs b/src/emit/pipeline/mod.rs index bc676104..0d9f68d3 100644 --- a/src/emit/pipeline/mod.rs +++ b/src/emit/pipeline/mod.rs @@ -13,7 +13,6 @@ pub mod ack; pub mod batcher; pub mod bootstrap; -pub mod decode; pub mod inserter; pub mod plan_spool; pub mod planner; diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index a135aba1..905c877c 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -1,18 +1,20 @@ //! Plan one committed transaction before executing it. //! -//! Keep heap entries in memory until their size exceeds memory limit, then -//! write them to a spool file. Keep table metadata and control entries in -//! memory, assuming control state stays small as in live transaction buffer. -//! Count and discard heaps without mappings, but keep their truncates. -//! Replay heaps and controls in original order, with controls before heaps -//! at matching positions. On planning failure, delete spool file and emit -//! nothing. Store files in transaction scratch directory, cleared at startup; -//! recover plans from source WAL. Read errors during replay require recovery -//! from source WAL, as with crashes during execution, so frames omit checksums. +//! Encode routed rows into per-table column chunks while planning. Seal open +//! chunks into a segment at row and byte limits and before every control or +//! truncate; each segment dispatches as one seq. Keep chunks in memory until +//! their size exceeds memory limit, then write them to a spool file. Keep +//! table metadata, controls, and truncates in memory, assuming control state +//! stays small as in live transaction buffer. Count and discard rows without +//! mappings, but keep their truncates. On planning failure, delete spool file +//! and emit nothing. Store files in transaction scratch directory, cleared at +//! startup; recover plans from source WAL. Read errors during replay require +//! recovery from source WAL, as with crashes during execution, so frames omit +//! checksums. //! -//! Frame layout after 4-byte `magic + version` header: -//! `[len:u32][table_id:u32][heap payload]`. Encode heap payload with spill -//! codec, omitting relation identity. Read `heap_count` frames from file. +//! Frame layout after 4-byte `magic + version` header, one frame per chunk: +//! `[len:u32][table_id:u32][rows:u32][columns:u32][column buffers]`. +//! Relation and route identity come from plan tables. use std::collections::hash_map::Entry; use std::fs::{self, File}; @@ -20,88 +22,65 @@ use std::io::{BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use clickhouse_c::Allocator; use thiserror::Error; -use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, DescribedHeap, HeapOp}; +use crate::ch::EmitterError; +use crate::decode::datum::DatumRef; +use crate::decode::heap_decoder::{ + DecodedHeap, HeapOp, TupleView, datum_value, take_key_change_tombstone, +}; +use crate::emit::ch_emitter::{ + Append, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, RowStamp, TableEncoder, TablePlan, +}; use crate::emit::route::RouteSnapshot; use crate::schema::RelDescriptor; use crate::xact::spill::{self, Cursor, IO_CHUNK, SpillError}; -use crate::xact::xact_buffer::{DrainEntry, OrderedEvent, ToastRowBatch}; +use crate::xact::xact_buffer::{DrainEntry, ToastRowBatch}; use foldhash::HashMap; /// ASCII for `xxd`-friendly debug pub const PLAN_MAGIC: [u8; 2] = *b"WP"; -pub const PLAN_VERSION: u16 = 2; +/// v3 replaced typed heap frames with encoded column chunks +pub const PLAN_VERSION: u16 = 3; /// Plans at or below this stay memory-resident: the common /// single-statement commit never touches the filesystem or the codec. /// Larger plans spill to the file at `path` (bounded by the disk cap) pub const DEFAULT_PLAN_MEM_MAX: u64 = 1 << 20; -/// Resident cost charged per typed heap on top of its values, standing in -/// for the frame a spilled heap would take -const MEM_FRAME_OVERHEAD: u64 = 64; - -/// Descriptor version and the route its heaps planned under. Heaps index -/// these, so a run of one relation's heaps shares one set of refcounts +/// Descriptor version and the route its rows planned under. Chunks index +/// these, so a run of one relation's rows shares one set of refcounts pub struct PlanTable { pub descriptor: Arc, pub valid_from: u64, - /// Count and discard heaps when `None`; still replay truncates + /// Count and discard rows when `None`; still replay truncates pub route: Option>, - /// Heap count for this table, including discarded heaps + /// Row changes for this table, including discarded ones pub heaps: u64, } impl PlanTable { - fn is(&self, heap: &DescribedHeap) -> bool { - self.key() == Self::key_of(heap) - } - fn key(&self) -> (usize, u64) { (Arc::as_ptr(&self.descriptor) as usize, self.valid_from) } - - fn key_of(heap: &DescribedHeap) -> (usize, u64) { - ( - Arc::as_ptr(&heap.descriptor) as usize, - heap.descriptor_valid_from, - ) - } } -/// Validated heap under its plan's table +/// Truncate under its plan table #[derive(Debug)] pub struct PlannedHeap { pub decoded: DecodedHeap, /// Index into plan's `tables` pub table: u32, - /// [`DecodedHeap::approx_bytes`] measured once, as every stage - /// downstream budgets by it - pub bytes: usize, -} - -impl PlannedHeap { - fn new(decoded: DecodedHeap, table: u32) -> Self { - let bytes = decoded.approx_bytes(); - Self { - decoded, - table, - bytes, - } - } } -/// Resident cost of a typed heap: values count payload only, so add each -/// column slot, else narrow wide rows overrun `mem_max` several times -fn mem_cost(heap: &PlannedHeap) -> u64 { - let d = &heap.decoded; - let slots = [&d.new, &d.old] - .into_iter() - .flatten() - .map(|t| t.columns.len()) - .sum::(); - MEM_FRAME_OVERHEAD + (heap.bytes + slots * std::mem::size_of::>()) as u64 +/// Rows sealed together; executor dispatches one seq per segment +pub struct Segment { + pub chunks: Vec, + /// Destination rows, delete markers for key changes included + pub rows: u64, + /// Allocated column bytes, the admission a dispatched segment holds + pub bytes: usize, } #[derive(Debug, Error)] @@ -112,6 +91,8 @@ pub enum PlanSpoolError { Format { detail: String }, #[error("plan spool {needed} bytes exceeds disk budget {cap}")] DiskBudget { needed: u64, cap: u64 }, + #[error("encode: {0}")] + Encode(#[from] EmitterError), } /// Frames decode from memory, so a decode error is always malformed bytes @@ -163,92 +144,124 @@ impl Drop for PlanFile { } } -/// Streams one transaction's validated heaps, typed in memory or to disk, -/// while accumulating the bounded plan header. Dropped unsealed (any -/// planning error), the file unlinks and nothing replays -enum PlanSink { - Mem(Vec), - File(BufWriter, PlanFile), +/// One plan step in drain order +enum Step { + /// `chunks` stay `None` once written to the spool file + Segment { + count: usize, + rows: u64, + bytes: usize, + chunks: Option>, + }, + Control { + event: DrainEntry, + row_idx: usize, + }, + Truncate { + heap: PlannedHeap, + upto: usize, + }, } +/// Per-table encoder of the open segment, keyed by plan table +struct OpenTable { + table: u32, + enc: TableEncoder, +} + +/// Streams one transaction's validated rows into column chunks, in memory +/// or on disk, while accumulating the bounded plan header. Dropped unsealed +/// (any planning error), the file unlinks and nothing replays pub struct PlanWriter { - sink: PlanSink, path: PlanPath, + file: Option<(BufWriter, PlanFile)>, mem_max: u64, cap: u64, + /// Resident chunk bytes while in memory, file bytes once spilled bytes: u64, - /// Number of heaps to replay, including truncates - heap_count: u64, + alloc: Allocator, tables: Vec, /// Tables since the last control entry, which may change routes live_from: usize, - /// Table of the previous heap, checked before `table_ids` + /// Table of the previous row, checked before `table_ids` last_table: u32, /// Live table ids by `(descriptor Arc identity, valid_from)`; the held /// Arc in `tables` keeps each pointer live and unique. Built once a /// second live table appears, so single-relation plans never hash table_ids: Option>, - controls: Vec, + open: Vec, + /// Chunks of the open segment sealed early at an oracle frame limit + sealed: Vec<(u32, EncodedChunk)>, + commit_ts: i64, + open_rows: usize, + segment_rows: usize, + segment_bytes: usize, + steps: Vec, row_batches: Vec, - truncate_rows: Vec, + routed_rows: u64, + deletes_discarded: u64, frame: Vec, } -/// Append one length-prefixed heap frame, charging `bytes` against `cap` -fn encode_frame( - frame: &mut Vec, - bytes: &mut u64, - cap: u64, - out: &mut BufWriter, - heap: &PlannedHeap, -) -> Result<()> { - frame.clear(); - frame.extend_from_slice(&[0; 4]); - spill::push_u32(frame, heap.table); - spill::encode_heap_into(frame, &heap.decoded); - let needed = *bytes + frame.len() as u64; - if needed > cap { - return Err(PlanSpoolError::DiskBudget { needed, cap }); - } - let len = (frame.len() - 4) as u32; - frame[..4].copy_from_slice(&len.to_le_bytes()); - out.write_all(frame)?; - *bytes = needed; - Ok(()) -} - impl PlanWriter { /// Starts memory-resident; the file at `path` is only created if the /// plan outgrows `mem_max` pub fn create(path: impl Into, cap: u64, mem_max: u64) -> Self { Self { - sink: PlanSink::Mem(Vec::new()), path: path.into(), + file: None, mem_max, cap, bytes: 4, - heap_count: 0, + alloc: Allocator::global(&mimalloc::MiMalloc), tables: Vec::new(), live_from: 0, last_table: 0, table_ids: None, - controls: Vec::new(), + open: Vec::new(), + sealed: Vec::new(), + commit_ts: 0, + open_rows: 0, + segment_rows: usize::MAX, + segment_bytes: usize::MAX, + steps: Vec::new(), row_batches: Vec::new(), - truncate_rows: Vec::new(), + routed_rows: 0, + deletes_discarded: 0, frame: Vec::new(), } } - /// Table `heap` plans under. `route` resolves only for a descriptor - /// version without a table since the last control entry + /// Seal a segment once its open rows or bytes reach these + pub fn with_segment_limits(mut self, rows: usize, bytes: usize) -> Self { + self.segment_rows = rows.max(1); + self.segment_bytes = bytes.max(1); + self + } + + /// Commit timestamp stamped on every planned row + pub fn for_commit(mut self, commit_ts: i64) -> Self { + self.commit_ts = commit_ts; + self + } + + /// Disk bytes written so far, header included + pub fn bytes(&self) -> u64 { + self.bytes + } + + /// Table rows of `descriptor` plan under. `route` resolves only for a + /// descriptor version without a table since the last control entry pub fn table( &mut self, - heap: &DescribedHeap, + descriptor: &Arc, + valid_from: u64, route: impl FnOnce() -> Option>, ) -> u32 { + let key = (Arc::as_ptr(descriptor) as usize, valid_from); let live = self.live_from..self.tables.len(); if live.contains(&(self.last_table as usize)) - && self.tables[self.last_table as usize].is(heap) + && self.tables[self.last_table as usize].key() == key { return self.last_table; } @@ -257,7 +270,7 @@ impl PlanWriter { let ids = self .table_ids .get_or_insert_with(|| live.map(|id| (self.tables[id].key(), id as u32)).collect()); - match ids.entry(PlanTable::key_of(heap)) { + match ids.entry(key) { Entry::Occupied(e) => { self.last_table = *e.get(); return self.last_table; @@ -268,8 +281,8 @@ impl PlanWriter { } } self.tables.push(PlanTable { - descriptor: heap.descriptor.clone(), - valid_from: heap.descriptor_valid_from, + descriptor: descriptor.clone(), + valid_from, route: route(), heaps: 0, }); @@ -281,73 +294,240 @@ impl PlanWriter { self.tables[table as usize].route.is_some() } - /// Append validated heap with a [`Self::table`] id; count and discard - /// it if table has no mapping - pub fn push_heap(&mut self, decoded: DecodedHeap, table: u32) -> Result<()> { - if self.routed(table) { - self.store(decoded, table)?; + /// Count `rows` changes of an unmapped table without planning them + pub fn discard(&mut self, table: u32, rows: u64) { + debug_assert!(!self.routed(table)); + self.tables[table as usize].heaps += rows; + } + + /// Encode one owned row change: drop deletes an append-only route has + /// nowhere to land, render PostGIS cells, split key changes into a + /// delete marker plus the new row + pub fn push_cells(&mut self, mut decoded: DecodedHeap, table: u32) -> Result<()> { + let t = &mut self.tables[table as usize]; + t.heaps += 1; + let (rel, route) = (t.descriptor.clone(), t.route.clone().expect("routed table")); + if route.drops_deletes() && decoded.op == HeapOp::Delete { + self.deletes_discarded += 1; + return Ok(()); + } + for t in [decoded.new.as_mut(), decoded.old.as_mut()] + .into_iter() + .flatten() + { + crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); + } + let tomb = (!route.drops_deletes()) + .then(|| take_key_change_tombstone(&mut decoded, &rel)) + .flatten(); + let leaf = route.partition.as_ref().map(|p| p.leaf); + for decoded in tomb.into_iter().chain([decoded]) { + let op = op_code(decoded.op)?; + let committed = crate::decode::heap_decoder::CommittedTuple { + decoded, + commit_ts: self.commit_ts, + commit_lsn: 0, + }; + self.append(table, |enc| { + enc.append_routed(&committed, &route.mapping, op, leaf) + })?; } - self.tables[table as usize].heaps += 1; Ok(()) } - /// Append `HeapOp::Truncate`; executor stores mirror rows with indices - /// below `upto`, then applies truncate - pub fn push_truncate(&mut self, decoded: DecodedHeap, table: u32, upto: usize) -> Result<()> { - self.store(decoded, table)?; - self.tables[table as usize].heaps += 1; - self.truncate_rows.push(upto); - Ok(()) + /// [`Self::push_cells`] reading tuple images in place + pub fn push_views(&mut self, change: ViewChange<'_, '_>, table: u32) -> Result<()> { + let t = &mut self.tables[table as usize]; + t.heaps += 1; + let (rel, route) = (t.descriptor.clone(), t.route.clone().expect("routed table")); + if route.drops_deletes() && change.op == HeapOp::Delete { + self.deletes_discarded += 1; + return Ok(()); + } + let leaf = route.partition.as_ref().map(|p| p.leaf); + let commit_ts = self.commit_ts; + let stamp = |source_lsn, op_code| RowStamp { + source_lsn, + xid: change.xid, + commit_ts, + op_code, + leaf, + }; + if !route.drops_deletes() + && let (Some(old), Some(new)) = (change.old, change.new) + && key_moved(&rel, change.op, old, new) + { + let stamp = stamp(change.source_lsn - 1, OP_DELETE); + self.append(table, |enc| enc.append_view(&rel, &old.datums, &stamp))?; + } + let side = match change.op { + HeapOp::Delete => change.old, + _ => change.new, + }; + let stamp = stamp(change.source_lsn, op_code(change.op)?); + let datums = side.map_or(&[][..], |t| &t.datums); + self.append(table, |enc| enc.append_view(&rel, datums, &stamp)) } - fn store(&mut self, decoded: DecodedHeap, table: u32) -> Result<()> { - let heap = PlannedHeap::new(decoded, table); - self.heap_count += 1; - // Resident estimate overstates encoded size, so enforce disk cap - // only against encoded frames - match &mut self.sink { - PlanSink::Mem(heaps) => { - let cost = mem_cost(&heap); - if self.bytes + cost <= self.mem_max.min(self.cap) { - self.bytes += cost; - heaps.push(heap); - return Ok(()); - } - let mut heaps = std::mem::take(heaps); - heaps.push(heap); - self.spill(&heaps) + /// Encode into `table`'s open chunk, sealing it first when an oracle + /// frame fills + fn append( + &mut self, + table: u32, + mut row: impl FnMut(&mut TableEncoder) -> std::result::Result, + ) -> Result<()> { + let i = match self.open.iter().position(|o| o.table == table) { + Some(i) => i, + None => { + let t = &self.tables[table as usize]; + let route = t.route.as_ref().expect("routed table"); + let plan = TablePlan::for_route(self.alloc, &t.descriptor, route)?; + self.open.push(OpenTable { + table, + enc: TableEncoder::new(plan)?, + }); + self.open.len() - 1 } - PlanSink::File(out, _) => { - encode_frame(&mut self.frame, &mut self.bytes, self.cap, out, &heap) + }; + if row(&mut self.open[i].enc)? == Append::Full { + let chunk = self.take_chunk(i)?; + self.sealed.push(chunk); + row(&mut self.open[i].enc)?; + } + self.open_rows += 1; + self.routed_rows += 1; + let open_bytes = self.open.iter().map(|o| o.enc.approx_bytes).sum::() + + self + .sealed + .iter() + .map(|(_, c)| c.allocated_bytes()) + .sum::(); + if self.open_rows >= self.segment_rows || open_bytes >= self.segment_bytes { + self.seal_segment()?; + } + Ok(()) + } + + fn take_chunk(&mut self, i: usize) -> Result<(u32, EncodedChunk)> { + let o = &mut self.open[i]; + let t = &self.tables[o.table as usize]; + let (buffers, rows) = o.enc.take_block()?; + let chunk = EncodedChunk { + rel: t.descriptor.clone(), + route: t.route.clone().expect("routed table"), + buffers, + rows, + }; + Ok((o.table, chunk)) + } + + /// Close the open segment into a step, spilling once memory runs out + fn seal_segment(&mut self) -> Result<()> { + let mut chunks = std::mem::take(&mut self.sealed); + for i in 0..self.open.len() { + if self.open[i].enc.rows > 0 { + chunks.push(self.take_chunk(i)?); } } + self.open_rows = 0; + if chunks.is_empty() { + return Ok(()); + } + let rows = chunks.iter().map(|(_, c)| c.rows as u64).sum(); + let bytes = chunks + .iter() + .map(|(_, c)| c.allocated_bytes()) + .sum::(); + let count = chunks.len(); + let chunks = + if self.file.is_none() && self.bytes + bytes as u64 <= self.mem_max.min(self.cap) { + self.bytes += bytes as u64; + Some(chunks) + } else { + self.spill()?; + for (table, c) in &chunks { + self.write_chunk(*table, c)?; + } + None + }; + self.steps.push(Step::Segment { + count, + rows, + bytes, + chunks, + }); + Ok(()) } - /// Move resident heaps into a fresh spool file; later heaps append there - fn spill(&mut self, heaps: &[PlannedHeap]) -> Result<()> { - let file = PlanFile(self.path.resolve()); - let mut out = BufWriter::with_capacity(IO_CHUNK, File::create(&file.0)?); + /// Move resident chunks into a fresh spool file, in step order + fn spill(&mut self) -> Result<()> { + if self.file.is_some() { + return Ok(()); + } + let path = self.path.resolve(); + let mut out = BufWriter::with_capacity(IO_CHUNK, File::create(&path)?); out.write_all(&PLAN_MAGIC)?; out.write_all(&PLAN_VERSION.to_le_bytes())?; + self.file = Some((out, PlanFile(path))); self.bytes = 4; - for h in heaps { - encode_frame(&mut self.frame, &mut self.bytes, self.cap, &mut out, h)?; + let mut steps = std::mem::take(&mut self.steps); + let written = steps.iter_mut().try_for_each(|step| { + if let Step::Segment { chunks, .. } = step + && let Some(chunks) = chunks.take() + { + chunks + .iter() + .try_for_each(|(table, c)| self.write_chunk(*table, c))?; + } + Ok::<_, PlanSpoolError>(()) + }); + self.steps = steps; + written + } + + fn write_chunk(&mut self, table: u32, chunk: &EncodedChunk) -> Result<()> { + let frame = &mut self.frame; + frame.clear(); + frame.extend_from_slice(&[0; 4]); + spill::push_u32(frame, table); + chunk.encode_into(frame); + let needed = self.bytes + frame.len() as u64; + if needed > self.cap { + return Err(PlanSpoolError::DiskBudget { + needed, + cap: self.cap, + }); } - self.sink = PlanSink::File(out, file); + let len = (frame.len() - 4) as u32; + frame[..4].copy_from_slice(&len.to_le_bytes()); + let (out, _) = self.file.as_mut().expect("frames require a spool file"); + out.write_all(frame)?; + self.bytes = needed; + Ok(()) + } + + /// Append `HeapOp::Truncate`; executor stores mirror rows with indices + /// below `upto`, then applies truncate + pub fn push_truncate(&mut self, decoded: DecodedHeap, table: u32, upto: usize) -> Result<()> { + self.seal_segment()?; + self.tables[table as usize].heaps += 1; + self.steps.push(Step::Truncate { + heap: PlannedHeap { decoded, table }, + upto, + }); Ok(()) } - /// Pin a control entry before the next heap: replay yields it ahead of - /// heap `heap_count`, preserving the drain's event-before-heap tie break. - /// `row_idx` is the plan-global mirror-row position it fences - pub fn push_control(&mut self, event: DrainEntry, row_idx: usize) { + /// Pin a control entry after every row planned so far, preserving the + /// drain's event-before-heap tie break. `row_idx` is the plan-global + /// mirror-row position it fences + pub fn push_control(&mut self, event: DrainEntry, row_idx: usize) -> Result<()> { + self.seal_segment()?; + self.open.clear(); self.live_from = self.tables.len(); self.table_ids = None; - self.controls.push(OrderedEvent { - heap_idx: self.heap_count as usize, - row_idx, - event, - }); + self.steps.push(Step::Control { event, row_idx }); + Ok(()) } /// Carry one batch's mirror-row refs; already gauged, released when the @@ -360,21 +540,22 @@ impl PlanWriter { } /// Flush spool file and finalize plan metadata - pub fn seal(self, commit_lsn: u64, commit_ts: i64) -> Result { - let store = match self.sink { - PlanSink::Mem(heaps) => PlanStore::Mem(heaps), - PlanSink::File(mut out, file) => { + pub fn seal(mut self, commit_lsn: u64, commit_ts: i64) -> Result { + self.seal_segment()?; + let file = match self.file.take() { + None => None, + Some((mut out, file)) => { out.flush()?; - PlanStore::File(file) + Some(file) } }; Ok(SealedPlan { - store, + file, + steps: self.steps, tables: self.tables, - controls: self.controls, row_batches: self.row_batches, - truncate_rows: self.truncate_rows, - heap_count: self.heap_count, + routed_rows: self.routed_rows, + deletes_discarded: self.deletes_discarded, size_bytes: self.bytes, commit_lsn, commit_ts, @@ -382,28 +563,90 @@ impl PlanWriter { } } -/// Store sealed plan heaps in memory or a spool file -enum PlanStore { - Mem(Vec), - File(PlanFile), +/// One row change read from tuple images in place +#[derive(Clone, Copy)] +pub struct ViewChange<'v, 'a> { + pub op: HeapOp, + pub xid: u32, + pub source_lsn: u64, + pub new: Option<&'v TupleView<'a>>, + pub old: Option<&'v TupleView<'a>>, +} + +/// [`take_key_change_tombstone`]'s test over tuple images in place +fn key_moved(rel: &RelDescriptor, op: HeapOp, old: &TupleView<'_>, new: &TupleView<'_>) -> bool { + let hot = match op { + HeapOp::Update => false, + HeapOp::HotUpdate => true, + _ => return false, + }; + let differs = |a: usize| match (old.datums.get(a), new.datums.get(a), rel.attributes.get(a)) { + (Some(&o), Some(&n), Some(att)) => datums_differ(att, o, n), + _ => false, + }; + let keys = crate::schema::replident_key_attnums(rel); + if keys.is_empty() { + matches!( + rel.replident, + crate::schema::ReplIdent::Full { pk_attnums: None } + ) && (0..new.datums.len()).any(differs) + } else { + // PostgreSQL HOT updates leave indexed columns unchanged + !hot && keys.iter().any(|&a| differs(a as usize - 1)) + } +} + +/// Cell inequality as owned decode compares it. Equal stored bytes decode +/// equal except float NaN, which never equals itself +fn datums_differ(att: &crate::schema::RelAttr, o: DatumRef<'_>, n: DatumRef<'_>) -> bool { + use crate::schema::{FLOAT4OID, FLOAT8OID}; + match (o, n) { + (DatumRef::Absent, _) | (_, DatumRef::Absent) => false, + (DatumRef::Fixed(a), DatumRef::Fixed(b)) + if a == b && !matches!(att.type_oid, FLOAT4OID | FLOAT8OID) => + { + false + } + (DatumRef::Varlena(a), DatumRef::Varlena(b)) if a == b => false, + (DatumRef::Null | DatumRef::Dropped, DatumRef::Null | DatumRef::Dropped) => false, + _ => { + let mut values = [o, n].map(|d| datum_value(att, d)); + if !att.dropped && matches!(att.type_name.as_str(), "geography" | "geometry") { + for v in values.iter_mut().flatten() { + crate::ops::oracle::render_ext_value(v); + } + } + values[0] != values[1] + } + } +} + +fn op_code(op: HeapOp) -> std::result::Result { + Ok(match op { + HeapOp::Insert => OP_INSERT, + HeapOp::Update | HeapOp::HotUpdate => OP_UPDATE, + HeapOp::Delete => OP_DELETE, + HeapOp::Truncate => { + return Err(EmitterError::Type("TRUNCATE planned as a row".into())); + } + }) } /// Validated plan, consumed by [`Self::replay`] pub struct SealedPlan { - store: PlanStore, - /// Tables in first-use order, indexed by [`PlannedHeap::table`] + file: Option, + steps: Vec, + /// Tables in first-use order, indexed by chunk frames and truncates pub tables: Vec, - /// Plan-global positions: entry at `heap_idx` replays before that heap - pub controls: Vec, - /// Mirror-row refs in plan order, still gauged; `controls[..].row_idx` - /// and `truncate_rows` index the concatenation + /// Mirror-row refs in plan order, still gauged; control `row_idx` and + /// truncate fences index the concatenation pub row_batches: Vec, - /// Plan-global mirror-row fence per `HeapOp::Truncate` heap, heap order - pub truncate_rows: Vec, - /// Number of heaps to replay, including truncates - pub heap_count: u64, - /// File size including header, or estimated memory use for plans in - /// memory; reported as `xact_plan_bytes` + /// Destination rows planned, delete markers included + pub routed_rows: u64, + /// Deletes dropped for append-only routes + pub deletes_discarded: u64, + /// File size including header, or column bytes for plans in memory; + /// reported as `xact_plan_bytes` pub size_bytes: u64, pub commit_lsn: u64, pub commit_ts: i64, @@ -412,13 +655,10 @@ pub struct SealedPlan { impl SealedPlan { /// Spool file path; `None` while memory-resident pub fn path(&self) -> Option<&Path> { - match &self.store { - PlanStore::Mem(_) => None, - PlanStore::File(f) => Some(&f.0), - } + self.file.as_ref().map(|f| f.0.as_path()) } - /// Count heaps with routes for `xact_plan_rows` + /// Count row changes with routes for `xact_plan_rows` pub fn routed_count(&self) -> u64 { self.tables .iter() @@ -428,9 +668,9 @@ impl SealedPlan { } pub fn replay(self) -> Result { - let input = match self.store { - PlanStore::Mem(heaps) => PlanInput::Mem(heaps.into_iter()), - PlanStore::File(file) => { + let input = match self.file { + None => None, + Some(file) => { let mut input = BufReader::with_capacity(IO_CHUNK, File::open(&file.0)?); let mut header = [0u8; 4]; input.read_exact(&mut header)?; @@ -448,30 +688,31 @@ impl SealedPlan { ), }); } - PlanInput::File { + Some(FileInput { input, frame: Vec::new(), - heaps: self.heap_count, bytes: self.size_bytes - 4, _file: file, - } + }) } }; Ok(PlanReader { tables: self.tables, row_batches: self.row_batches, - controls: self.controls.into_iter().peekable(), - truncate_rows: self.truncate_rows.into_iter(), + deletes_discarded: self.deletes_discarded, + steps: self.steps.into_iter(), input, - next_heap: 0, }) } } /// One replayed plan step, original drain order pub enum PlanItem { - Control(OrderedEvent), - Heap(PlannedHeap), + Control { + event: DrainEntry, + row_idx: usize, + }, + Segment(Segment), /// Mirror rows below `upto` put before the truncate applies Truncate { heap: PlannedHeap, @@ -479,110 +720,107 @@ pub enum PlanItem { }, } -enum PlanInput { - Mem(std::vec::IntoIter), - File { - input: BufReader, - frame: Vec, - /// Frames and bytes left before the sealed end - heaps: u64, - bytes: u64, - _file: PlanFile, - }, +struct FileInput { + input: BufReader, + frame: Vec, + /// Bytes left before the sealed end + bytes: u64, + _file: PlanFile, } /// Replay [`SealedPlan`] in order, resolving table ids from plan metadata -/// and inserting controls at recorded positions pub struct PlanReader { /// [`SealedPlan::tables`] pub tables: Vec, /// [`SealedPlan::row_batches`] pub row_batches: Vec, - controls: std::iter::Peekable>, - truncate_rows: std::vec::IntoIter, - input: PlanInput, - next_heap: u64, + /// [`SealedPlan::deletes_discarded`] + pub deletes_discarded: u64, + steps: std::vec::IntoIter, + input: Option, } impl PlanReader { pub fn next_item(&mut self) -> Result> { - let next_heap = self.next_heap; - if let Some(c) = self.controls.next_if(|c| c.heap_idx as u64 <= next_heap) { - return Ok(Some(PlanItem::Control(c))); - } - let heap = match &mut self.input { - PlanInput::Mem(heaps) => heaps.next(), - PlanInput::File { heaps: 0, .. } => None, - PlanInput::File { - input, - frame, - heaps, + let Some(step) = self.steps.next() else { + return Ok(None); + }; + Ok(Some(match step { + Step::Control { event, row_idx } => PlanItem::Control { event, row_idx }, + Step::Truncate { heap, upto } => PlanItem::Truncate { heap, upto }, + Step::Segment { + count, + rows, bytes, - .. + chunks, } => { - *heaps -= 1; - Some(read_frame(input, frame, bytes, &self.tables)?) + let chunks = match chunks { + Some(chunks) => chunks.into_iter().map(|(_, c)| c).collect(), + None => { + let Some(f) = self.input.as_mut() else { + return Err(PlanSpoolError::Format { + detail: "spilled segment without spool file".into(), + }); + }; + (0..count) + .map(|_| read_frame(f, &self.tables)) + .collect::>()? + } + }; + PlanItem::Segment(Segment { + chunks, + rows, + bytes, + }) } - }; - let Some(heap) = heap else { - return Ok(None); - }; - self.next_heap += 1; - if heap.decoded.op != HeapOp::Truncate { - return Ok(Some(PlanItem::Heap(heap))); - } - let Some(upto) = self.truncate_rows.next() else { - return Err(PlanSpoolError::Format { - detail: format!( - "truncate heap {} has no mirror-row fence", - self.next_heap - 1 - ), - }); - }; - Ok(Some(PlanItem::Truncate { heap, upto })) + })) } } -/// Read next `[len][table_id][heap payload]` frame -fn read_frame( - input: &mut impl Read, - frame: &mut Vec, - bytes: &mut u64, - tables: &[PlanTable], -) -> Result { +/// Read next `[len][table_id][chunk]` frame +fn read_frame(f: &mut FileInput, tables: &[PlanTable]) -> Result { let mut len = [0u8; 4]; - input.read_exact(&mut len)?; + f.input.read_exact(&mut len)?; let len = u32::from_le_bytes(len); // Bound against sealed size before allocating, so a garbage length // reads as format error instead of a multi-gigabyte buffer - if 4 + u64::from(len) > *bytes { + if 4 + u64::from(len) > f.bytes { return Err(PlanSpoolError::Format { - detail: format!("frame claims {len} bytes, {} left in plan", *bytes - 4), + detail: format!("frame claims {len} bytes, {} left in plan", f.bytes - 4), }); } - *bytes -= 4 + u64::from(len); - frame.resize(len as usize, 0); - input.read_exact(frame)?; - let mut c = Cursor::new(frame); + f.bytes -= 4 + u64::from(len); + f.frame.resize(len as usize, 0); + f.input.read_exact(&mut f.frame)?; + let mut c = Cursor::new(&f.frame); let table = c.u32()?; - let Some(t) = tables.get(table as usize) else { + let Some(PlanTable { + descriptor, + route: Some(route), + .. + }) = tables.get(table as usize) + else { return Err(PlanSpoolError::Format { detail: format!( - "heap references table id {table}, header has {}", + "chunk references table id {table}, header has {} routed", tables.len() ), }); }; - let decoded = spill::decode_heap(&mut c, t.descriptor.rfn)?; - Ok(PlannedHeap::new(decoded, table)) + Ok(EncodedChunk::decode( + descriptor.clone(), + route.clone(), + &mut c, + )?) } #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple, HeapOp}; - use crate::mapping::{TableMapping, TableTarget}; - use crate::schema::{INT4OID, RelAttr, RelDescriptor, RelName, SchemaEvent}; + use crate::decode::heap_decoder::{ColumnValue, DecodedTuple}; + use crate::emit::ch_emitter::Wire; + use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; + use crate::schema::{INT4OID, RelAttr, RelName, SchemaEvent}; use walrus::pg::walparser::RelFileNode; fn descriptor(rel_node: u32) -> Arc { @@ -617,39 +855,31 @@ mod tests { }) } - fn heap(rel: &Arc, lsn: u64, v: i32) -> DescribedHeap { - DescribedHeap { - decoded: DecodedHeap { - rfn: rel.rfn, - xid: 7, - source_lsn: lsn, - op: HeapOp::Insert, - new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(v))], - partial: false, - }), - old: None, - }, - descriptor: rel.clone(), - descriptor_valid_from: 0x40, + fn heap(rel: &Arc, lsn: u64, v: i32) -> DecodedHeap { + DecodedHeap { + rfn: rel.rfn, + xid: 7, + source_lsn: lsn, + op: HeapOp::Insert, + new: Some(DecodedTuple { + columns: vec![Some(ColumnValue::Int4(v))], + partial: false, + }), + old: None, } } - /// Push `heap` under the table it plans to, `route` deciding a new one - fn push( - w: &mut PlanWriter, - heap: DescribedHeap, - route: Option>, - ) -> Result<()> { - let table = w.table(&heap, || route); - w.push_heap(heap.decoded, table) - } - - fn route() -> Arc { + /// Route mapping `id` to `target` + fn route_to(target: &str) -> Arc { RouteSnapshot::freeze( Arc::new(TableMapping { target: TableTarget::new("db", "t"), - columns: Vec::new(), + columns: vec![ColumnMapping { + src_attnum: 1, + target_name: "id".into(), + target_type: target.into(), + type_pinned: false, + }], tees: Vec::new(), }), Arc::default(), @@ -657,6 +887,26 @@ mod tests { ) } + fn route() -> Arc { + route_to("Nullable(Int32)") + } + + /// Push `heap` under the table it plans to, `route` deciding a new one + fn push( + w: &mut PlanWriter, + rel: &Arc, + heap: DecodedHeap, + route: Option>, + ) -> Result<()> { + let table = w.table(rel, 0x40, || route); + if w.routed(table) { + w.push_cells(heap, table) + } else { + w.discard(table, 1); + Ok(()) + } + } + fn dropped(oid: u32) -> DrainEntry { DrainEntry::Catalog(SchemaEvent::Dropped { oid, @@ -665,27 +915,64 @@ mod tests { }) } - /// Preserve write order, placing controls before their heap or after - /// all heaps. Reuse one table per descriptor between controls. Count - /// and discard unmapped heaps, but replay unmapped truncates + /// `(id, _lsn, _xid, _is_deleted)` per row of every chunk + fn rows(chunks: &[EncodedChunk]) -> Vec<(Option, u64, u32, u8)> { + let mut out = Vec::new(); + for c in chunks { + let Wire::NullableFixed { + null_map, inner, .. + } = c.buffers[0].wire() + else { + panic!("nullable int4"); + }; + let fixed = |i: usize, w: usize| match c.buffers[i].wire() { + Wire::Fixed { bytes, .. } => bytes + .chunks_exact(w) + .map(|b| { + let mut v = [0u8; 8]; + v[..w].copy_from_slice(b); + u64::from_le_bytes(v) + }) + .collect::>(), + w => panic!("{w:?}"), + }; + let (lsn, xid) = (fixed(1, 8), fixed(2, 4)); + // Append-only routes carry no delete marker + let deleted = if c.buffers.len() > 4 { + fixed(4, 1) + } else { + vec![0; c.rows] + }; + for r in 0..c.rows { + let id = (null_map[r] == 0) + .then(|| i32::from_le_bytes(inner[r * 4..r * 4 + 4].try_into().unwrap())); + out.push((id, lsn[r], xid[r] as u32, deleted[r] as u8)); + } + } + out + } + + /// Preserve write order, sealing segments before controls and + /// truncates. Reuse one table per descriptor between controls. Count + /// and discard unmapped rows, but replay unmapped truncates #[test] - fn round_trip_orders_heaps_and_controls() { + fn round_trip_orders_segments_and_controls() { for mem_max in [DEFAULT_PLAN_MEM_MAX, 0] { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("1.plan"); - let mut w = PlanWriter::create(path.clone(), 1 << 20, mem_max); + let mut w = PlanWriter::create(path.clone(), 1 << 20, mem_max).for_commit(42); let (d1, d2) = (descriptor(16500), descriptor(16501)); let r = route(); - w.push_control(dropped(1), 0); - push(&mut w, heap(&d1, 100, 10), Some(r.clone())).unwrap(); - push(&mut w, heap(&d2, 110, 20), None).unwrap(); + w.push_control(dropped(1), 0).unwrap(); + push(&mut w, &d1, heap(&d1, 100, 10), Some(r.clone())).unwrap(); + push(&mut w, &d2, heap(&d2, 110, 20), None).unwrap(); let mut truncate = heap(&d2, 115, 25); - truncate.decoded.op = HeapOp::Truncate; - let table = w.table(&truncate, || unreachable!("table live")); - w.push_truncate(truncate.decoded, table, 0).unwrap(); - w.push_control(dropped(2), 0); - push(&mut w, heap(&d1, 120, 30), Some(r.clone())).unwrap(); - w.push_control(dropped(3), 0); + truncate.op = HeapOp::Truncate; + let table = w.table(&d2, 0x40, || unreachable!("table live")); + w.push_truncate(truncate, table, 0).unwrap(); + w.push_control(dropped(2), 0).unwrap(); + push(&mut w, &d1, heap(&d1, 120, 30), Some(r.clone())).unwrap(); + w.push_control(dropped(3), 0).unwrap(); let plan = w.seal(0x2000, 42).unwrap(); let tables: Vec<_> = plan .tables @@ -697,7 +984,7 @@ mod tests { [(16500, true, 1), (16501, false, 2), (16500, true, 1)], "control re-resolves routes" ); - assert_eq!(plan.heap_count, 3, "unmapped insert discarded"); + assert_eq!(plan.routed_rows, 2, "unmapped insert discarded"); assert_eq!(plan.routed_count(), 2); assert!(plan.size_bytes > 0); assert_eq!((plan.commit_lsn, plan.commit_ts), (0x2000, 42)); @@ -707,33 +994,30 @@ mod tests { let mut order = Vec::new(); while let Some(item) = rd.next_item().unwrap() { order.push(match item { - PlanItem::Control(c) => format!("{:?}", c.event), - PlanItem::Heap(h) | PlanItem::Truncate { heap: h, .. } => { - let t = &rd.tables[h.table as usize]; - assert!( - Arc::ptr_eq(&t.descriptor, &d1) || Arc::ptr_eq(&t.descriptor, &d2), - "descriptor identity survives the plan" - ); - if let Some(route) = &t.route { - assert!(Arc::ptr_eq(route, &r)); + PlanItem::Control { event, .. } => format!("{event:?}"), + PlanItem::Segment(s) => { + for c in &s.chunks { + assert!(Arc::ptr_eq(&c.rel, &d1), "descriptor survives plan"); + assert!(Arc::ptr_eq(&c.route, &r)); } - assert_eq!(h.decoded.rfn, t.descriptor.rfn); - let new = h.decoded.new.as_ref().unwrap(); - let routed = if t.route.is_some() { "r" } else { "-" }; - format!("{:?}{routed}", new.columns[0]) + format!("{:?}", rows(&s.chunks)) + } + PlanItem::Truncate { heap: h, .. } => { + assert!(rd.tables[h.table as usize].route.is_none()); + format!("t{}", h.decoded.source_lsn) } }); } let ev = |oid| format!("{:?}", dropped(oid)); - let heap = |v, routed| format!("{:?}{routed}", Some(ColumnValue::Int4(v))); + let seg = |v, lsn| format!("{:?}", [(Some(v), lsn, 7u32, 0u8)]); assert_eq!( order, [ ev(1), - heap(10, "r"), - heap(25, "-"), + seg(10, 100), + "t115".into(), ev(2), - heap(30, "r"), + seg(30, 120), ev(3) ] ); @@ -742,158 +1026,147 @@ mod tests { } } + /// Updates, deletes and key changes encode as destination rows; + /// append-only routes drop deletes and count them #[test] - fn compact_heaps_preserve_subxids_and_images() { - for mem_max in [DEFAULT_PLAN_MEM_MAX, 0] { + fn row_changes_encode_markers_and_drops() { + let d = descriptor(16500); + let image = |v| { + Some(DecodedTuple { + columns: vec![Some(ColumnValue::Int4(v))], + partial: false, + }) + }; + let change = |op, lsn, new, old| DecodedHeap { + op, + new, + old, + xid: 8, + ..heap(&d, lsn, 0) + }; + let append_only = RouteSnapshot::freeze( + route().mapping.clone(), + Arc::default(), + crate::emit::route::RowPolicy { + system: Arc::new(crate::mapping::SystemColumns { + is_deleted: None, + ..Default::default() + }), + ..Default::default() + }, + ); + for (route, want, discarded) in [ + ( + route(), + vec![ + (Some(1), 100, 8, 0), + (Some(1), 199, 8, 1), + (Some(2), 200, 8, 0), + (Some(2), 300, 8, 0), + (Some(2), 400, 8, 1), + ], + 0, + ), + ( + append_only, + vec![ + (Some(1), 100, 8, 0), + (Some(2), 200, 8, 0), + (Some(2), 300, 8, 0), + ], + 1, + ), + ] { let tmp = tempfile::tempdir().unwrap(); - let mut w = PlanWriter::create(tmp.path().join("1.plan"), 1 << 20, mem_max); - let descriptors = [descriptor(16500), descriptor(16501)]; - let mut expected = Vec::new(); - let mut encoded_bytes = 4; - for (i, op) in [ - HeapOp::Insert, - HeapOp::Update, - HeapOp::HotUpdate, - HeapOp::Delete, - HeapOp::Truncate, - ] - .into_iter() - .enumerate() - { - let mut h = heap(&descriptors[i % 2], 100 + i as u64, i as i32); - h.decoded.xid += i as u32; - h.decoded.op = op; - if op != HeapOp::Insert { - h.decoded.old = h.decoded.new.clone(); - } - if matches!(op, HeapOp::Delete | HeapOp::Truncate) { - h.decoded.new = None; - } - if op == HeapOp::Truncate { - h.decoded.old = None; - } - let mut full = Vec::new(); - spill::encode_heap_into(&mut full, &h.decoded); - encoded_bytes += (4 + 4 + full.len()) as u64; - expected.push(h.decoded.clone()); - let table = w.table(&h, || Some(route())); - if op == HeapOp::Truncate { - w.push_truncate(h.decoded, table, 0).unwrap(); - } else { - w.push_heap(h.decoded, table).unwrap(); - } + let mut w = PlanWriter::create(tmp.path().join("1.plan"), 1 << 20, 0); + let table = w.table(&d, 0x40, || Some(route)); + for h in [ + change(HeapOp::Insert, 100, image(1), None), + // FULL identity without key: any change moves the row + change(HeapOp::Update, 200, image(2), image(1)), + change(HeapOp::HotUpdate, 300, image(2), image(2)), + change(HeapOp::Delete, 400, None, image(2)), + ] { + w.push_cells(h, table).unwrap(); } let plan = w.seal(0x2000, 42).unwrap(); - if mem_max == 0 { - assert_eq!(plan.size_bytes, encoded_bytes); - } + assert_eq!(plan.deletes_discarded, discarded); let mut rd = plan.replay().unwrap(); - for decoded in expected { - let item = rd.next_item().unwrap().unwrap(); - assert!( - matches!(item, PlanItem::Heap(h) | PlanItem::Truncate { heap: h, .. } if h.decoded == decoded) - ); - } - assert!(rd.next_item().unwrap().is_none()); + let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() else { + panic!("one segment"); + }; + assert_eq!(rows(&s.chunks), want); + assert_eq!(s.rows, want.len() as u64); + assert_eq!(rd.deletes_discarded, discarded); } } - /// Plan outgrowing its memory cap moves resident heaps into the file in - /// order, ahead of later heaps + /// Segment limits split rows into seqs; a plan outgrowing its memory + /// cap moves resident chunks into the file in order, ahead of later #[test] - fn spill_preserves_resident_heaps() { + fn segments_split_and_spill_in_order() { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("1.plan"); let d = descriptor(16500); - let resident_max = 2 * mem_cost(&PlannedHeap::new(heap(&d, 0, 0).decoded, 0)); - let mut w = PlanWriter::create(path.clone(), 1 << 20, resident_max); - for v in 0..4 { - push(&mut w, heap(&d, 100 + v as u64, v), Some(route())).unwrap(); + let mut w = + PlanWriter::create(path.clone(), 1 << 20, 1 << 10).with_segment_limits(2, usize::MAX); + for v in 0..64 { + push(&mut w, &d, heap(&d, 100 + v as u64, v), Some(route())).unwrap(); } let plan = w.seal(0x2000, 42).unwrap(); assert!(path.exists(), "spilled past the memory cap"); assert_eq!(plan.size_bytes, fs::read(&path).unwrap().len() as u64); let mut rd = plan.replay().unwrap(); let mut seen = Vec::new(); - while let Some(PlanItem::Heap(h)) = rd.next_item().unwrap() { - seen.push(h.decoded.new.unwrap().columns[0].clone()); + while let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() { + assert_eq!(s.rows, 2); + seen.extend(rows(&s.chunks).into_iter().map(|r| r.0.unwrap())); } - let want: Vec<_> = (0..4).map(|v| Some(ColumnValue::Int4(v))).collect(); - assert_eq!(seen, want); + assert_eq!(seen, (0..64).collect::>()); } - /// Wide narrow-typed rows charge per column slot, so a plan whose - /// payload alone fits the memory cap still spills + /// Oracle cells and strings survive the spool byte-identical #[test] - fn wide_rows_charge_column_slots() { + fn oracle_and_string_chunks_round_trip() { + let mut d = (*descriptor(16500)).clone(); + d.attributes[0].type_oid = 3614; + d.attributes[0].type_len = -1; + let d = Arc::new(d); + let cells = |path: PathBuf, mem_max| { + let mut w = PlanWriter::create(path, 1 << 20, mem_max); + let table = w.table(&d, 0x40, || Some(route_to("String"))); + for (lsn, v) in [ + ( + 100, + Some(ColumnValue::PgPending { + type_oid: 3614, + raw: vec![1, 2, 3], + }), + ), + (110, Some(ColumnValue::Null)), + (120, None), + ] { + let mut h = heap(&d, lsn, 0); + h.new.as_mut().unwrap().columns = vec![v]; + w.push_cells(h, table).unwrap(); + } + let mut rd = w.seal(0x2000, 42).unwrap().replay().unwrap(); + let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() else { + panic!("one segment"); + }; + s.chunks + .iter() + .flat_map(|c| c.buffers.iter().map(|b| format!("{:?}", b.wire()))) + .collect::>() + }; let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("1.plan"); - let mut w = PlanWriter::create(path.clone(), 1 << 30, DEFAULT_PLAN_MEM_MAX); - let d = descriptor(16500); - let mut payload = 0; - for v in 0..64 { - let mut h = heap(&d, 100 + v as u64, v); - h.decoded.new.as_mut().unwrap().columns = vec![Some(ColumnValue::Int4(v)); 1000]; - payload += MEM_FRAME_OVERHEAD + h.decoded.approx_bytes() as u64; - push(&mut w, h, Some(route())).unwrap(); - } - assert!(payload < DEFAULT_PLAN_MEM_MAX); - let plan = w.seal(0x2000, 42).unwrap(); + let mem = cells(tmp.path().join("a.plan"), DEFAULT_PLAN_MEM_MAX); + let file = cells(tmp.path().join("b.plan"), 0); assert!( - plan.path().is_some(), - "column slots pushed plan past mem cap" + mem[0].starts_with("Oracle([DiskRaw([1, 2, 3]), Default, Default])"), + "{mem:?}" ); - assert!(path.exists()); - } - - /// Disk cap applies to encoded bytes: a wide row whose resident - /// estimate exceeds the cap still plans when its frame fits - #[test] - fn disk_cap_measures_encoded_bytes() { - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("1.plan"); - let cap = 16 << 10; - let mut w = PlanWriter::create(path.clone(), cap, DEFAULT_PLAN_MEM_MAX); - let d = descriptor(16500); - let mut h = heap(&d, 100, 0); - h.decoded.new.as_mut().unwrap().columns = vec![Some(ColumnValue::Int4(7)); 1000]; - assert!(mem_cost(&PlannedHeap::new(h.decoded.clone(), 0)) > cap); - push(&mut w, h, Some(route())).unwrap(); - let plan = w.seal(0x2000, 42).unwrap(); - assert!(plan.size_bytes <= cap); - assert!(path.exists()); - } - - /// Oracle-pending values (`PgPending` / `Unsupported`) round-trip - /// byte-identical so post-plan decode resolves them exactly as - /// live-decoded rows. Best-effort policy: no plan-time codec rejection, - /// shadow-PG oracle resolves after replay, may lag row's catalog state - #[test] - fn pending_values_round_trip() { - use PlanItem::Heap; - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("1.plan"); - let mut w = PlanWriter::create(path, 1 << 20, DEFAULT_PLAN_MEM_MAX); - let d = descriptor(16500); - let cols = vec![ - Some(ColumnValue::PgPending { - type_oid: 3802, - raw: br#"{"k":1}"#.to_vec(), - }), - Some(ColumnValue::Unsupported { - type_oid: 600, - raw: vec![1, 2, 3, 4], - }), - ]; - let mut h = heap(&d, 100, 0); - h.decoded.new.as_mut().unwrap().columns = cols.clone(); - push(&mut w, h, Some(route())).unwrap(); - let plan = w.seal(0x2000, 42).unwrap(); - let mut rd = plan.replay().unwrap(); - let item = rd.next_item().unwrap(); - let Some(Heap(out)) = item else { panic!() }; - assert_eq!(out.decoded.new.unwrap().columns, cols); - assert!(rd.next_item().unwrap().is_none()); + assert_eq!(mem, file); } /// Delete sealed plan file on drop even without replay @@ -903,7 +1176,7 @@ mod tests { let path = tmp.path().join("1.plan"); let mut w = PlanWriter::create(path.clone(), 1 << 20, 0); let d = descriptor(16500); - push(&mut w, heap(&d, 100, 10), Some(route())).unwrap(); + push(&mut w, &d, heap(&d, 100, 10), Some(route())).unwrap(); let plan = w.seal(0x2000, 42).unwrap(); assert!(path.exists(), "mem_max 0 forced file mode"); drop(plan); @@ -911,20 +1184,13 @@ mod tests { } /// Replay a spilled plan after `tamper`, returning the first error - fn replay_err( - heaps: Vec, - tamper: impl FnOnce(&mut SealedPlan, &mut Vec), - ) -> PlanSpoolError { + fn replay_err(tamper: impl FnOnce(&mut SealedPlan, &mut Vec)) -> PlanSpoolError { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("1.plan"); - let mut w = PlanWriter::create(path.clone(), 1 << 20, 0); - for h in heaps { - let table = w.table(&h, || Some(route())); - if h.decoded.op == HeapOp::Truncate { - w.push_truncate(h.decoded, table, 0).unwrap(); - } else { - w.push_heap(h.decoded, table).unwrap(); - } + let d = descriptor(16500); + let mut w = PlanWriter::create(path.clone(), 1 << 20, 0).with_segment_limits(1, usize::MAX); + for (lsn, v) in [(100, 10), (110, 20)] { + push(&mut w, &d, heap(&d, lsn, v), Some(route())).unwrap(); } let mut plan = w.seal(0x2000, 42).unwrap(); let mut buf = fs::read(&path).unwrap(); @@ -945,32 +1211,16 @@ mod tests { /// Reject invalid headers, frames, and truncated files during replay #[test] fn inconsistent_plan_fails_replay() { - let d = descriptor(16500); - let one = || vec![heap(&d, 100, 10)]; - let mut truncate = heap(&d, 100, 10); - truncate.decoded.op = HeapOp::Truncate; - // header 4, len 4, table 4, xid 4, lsn 8 - let op_byte = 24; + // header 4, len 4, table 4, rows 4, columns 4, then a buffer tag + let tag_byte = 20; let cases: Vec<(&str, PlanSpoolError)> = vec![ - ("bad plan magic", replay_err(one(), |_, b| b[0] ^= 0xFF)), - ( - "unsupported plan version", - replay_err(one(), |_, b| b[2] ^= 0xFF), - ), - ("table id", replay_err(one(), |p, _| p.tables.clear())), - ( - "unknown HeapOp", - replay_err(one(), |_, b| b[op_byte] = 0xFF), - ), + ("bad plan magic", replay_err(|_, b| b[0] ^= 0xFF)), + ("unsupported plan version", replay_err(|_, b| b[2] ^= 0xFF)), + ("table id", replay_err(|p, _| p.tables.clear())), + ("column buffer tag", replay_err(|_, b| b[tag_byte] = 0xFF)), ( "left in plan", - replay_err(one(), |_, b| { - b[4..8].copy_from_slice(&u32::MAX.to_le_bytes()) - }), - ), - ( - "no mirror-row fence", - replay_err(vec![truncate], |p, _| p.truncate_rows.clear()), + replay_err(|_, b| b[4..8].copy_from_slice(&u32::MAX.to_le_bytes())), ), ]; for (want, err) in cases { @@ -980,7 +1230,7 @@ mod tests { ); assert!(err.to_string().contains(want), "{want}: {err}"); } - let tail = replay_err(vec![heap(&d, 100, 10), heap(&d, 110, 20)], |_, b| { + let tail = replay_err(|_, b| { b.pop(); }); assert!( @@ -994,9 +1244,9 @@ mod tests { fn budget_bounds_writes_and_drop_unlinks() { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("1.plan"); - let mut w = PlanWriter::create(path.clone(), 16, 0); + let mut w = PlanWriter::create(path.clone(), 16, 0).with_segment_limits(1, usize::MAX); let d = descriptor(16500); - let err = push(&mut w, heap(&d, 100, 10), Some(route())).unwrap_err(); + let err = push(&mut w, &d, heap(&d, 100, 10), Some(route())).unwrap_err(); assert!(matches!(err, PlanSpoolError::DiskBudget { .. }), "{err}"); drop(w); assert!(!path.exists(), "abandoned plan unlinks"); diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 195d634e..8b92109a 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -39,16 +39,18 @@ use std::sync::Arc; -use crate::decode::heap_decoder::DescribedHeap; +use crate::decode::datum::DatumRef; +use crate::decode::heap_decoder::view_heap_record; use crate::emit::pipeline::plan_spool::{ - DEFAULT_PLAN_MEM_MAX, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, + DEFAULT_PLAN_MEM_MAX, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, }; use crate::emit::route::RouteSnapshot; use crate::schema::RelDescriptor; use crate::toast::{ChunkRefMap, ToastResolver}; +use crate::xact::spill::DescribedInput; use crate::xact::xact_buffer::{ - DrainEntry, DrainWalk, DrainedBatch, FailClosedReason, WalkStep, XactBufferError, detoast_heap, - heap_reads_toast, + DrainEntry, DrainHeap, DrainWalk, DrainedBatch, FailClosedReason, WalkStep, XactBufferError, + detoast_heap, heap_reads_toast, }; /// Planner's window onto route state. Implementations resolve from frozen @@ -58,7 +60,7 @@ use crate::xact::xact_buffer::{ /// discard. A plan asks once per descriptor version between control /// entries, and reports what it discarded through `unmapped` once sealed pub trait PlanRouteView { - fn route_for(&mut self, heap: &DescribedHeap) -> Option>; + fn route_for(&mut self, rel: &Arc) -> Option>; /// `heaps` rows of `rel` a sealed plan discards as unmapped fn unmapped(&mut self, rel: &RelDescriptor, heaps: u64); /// Fold control entry into local view without mutating shared route state @@ -86,6 +88,7 @@ impl PlanError { /// `reason=` label for `xact_plan_failures_total` pub fn reason(&self) -> &'static str { match self { + Self::Spool(PlanSpoolError::Encode(_)) => "encode", Self::Spool(_) => "spool", Self::Detoast(e) => drain_reason(e), Self::PartialUpdate { .. } => "partial_update", @@ -128,14 +131,19 @@ pub struct Planner<'a, V> { } impl<'a, V: PlanRouteView> Planner<'a, V> { + /// `segment` bounds rows and bytes dispatched as one seq pub fn create( path: impl Into, disk_cap: u64, + segment: (usize, usize), + commit_ts: i64, view: &'a mut V, resolver: &'a ToastResolver, ) -> Self { Self { - writer: PlanWriter::create(path, disk_cap, DEFAULT_PLAN_MEM_MAX), + writer: PlanWriter::create(path, disk_cap, DEFAULT_PLAN_MEM_MAX) + .with_segment_limits(segment.0, segment.1) + .for_commit(commit_ts), view, resolver, rows_base: 0, @@ -171,32 +179,53 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Box::pin(self.view.apply(&e)) .await .map_err(PlanError::View)?; - self.writer.push_control(e, self.rows_base + cursor); + self.writer.push_control(e, self.rows_base + cursor)?; } WalkStep::Truncate { heap, upto } => { self.resolver.clear_reads(); - let table = self.writer.table(&heap, || self.view.route_for(&heap)); + let table = + self.writer + .table(&heap.descriptor, heap.descriptor_valid_from, || { + self.view.route_for(&heap.descriptor) + }); self.writer .push_truncate(heap.decoded, table, self.rows_base + upto)?; } - WalkStep::Heap(mut heap) => { + WalkStep::Heap(drained) => { // Route before validation and detoast: unmapped rows - // discard without touching - // the resolver or codec checks. The value permit drops - // once bytes land in the plan file - let table = self.writer.table(&heap, || self.view.route_for(&heap)); - if self.writer.routed(table) { + // discard without touching the resolver or codec checks + let rel = drained.descriptor().clone(); + let table = self + .writer + .table(&rel, drained.descriptor_valid_from(), || { + self.view.route_for(&rel) + }); + if !self.writer.routed(table) { + self.writer.discard(table, drained.rows() as u64); + continue; + } + let drained = match drained { + DrainHeap::Input(input) => { + if self.plan_views(&input, &rel, table)? { + continue; + } + DrainHeap::Input(input) + } + cells => cells, + }; + for mut heap in drained.into_cells()? { if heap.decoded.new.as_ref().is_some_and(|t| t.partial) { return Err(PlanError::PartialUpdate { lsn: heap.decoded.source_lsn, }); } if heap_reads_toast(&heap) { + // Value permit drops once bytes land in the plan Box::pin(detoast_heap(&mut heap, spool, &ref_maps, self.resolver)) .await?; } + self.writer.push_cells(heap.decoded, table)?; } - self.writer.push_heap(heap.decoded, table)?; } } } @@ -205,6 +234,50 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Ok(()) } + /// Encode an input from tuple images in place; `false` when an image + /// holds external values, which only owned cells detoast + fn plan_views( + &mut self, + input: &DescribedInput, + rel: &Arc, + table: u32, + ) -> Result { + let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { + XactBufferError::OrdinaryFailClosed { + relid: rel.oid, + lsn: input.input.source_lsn, + rm: input.input.rm, + info: input.input.info, + reason: FailClosedReason::Malformed(e.to_string()), + } + })?; + let external = images.iter().any(|h| { + [&h.new, &h.old] + .into_iter() + .flatten() + .any(|t| t.datums.iter().any(|d| matches!(d, DatumRef::External(_)))) + }); + if external { + return Ok(false); + } + for h in &images { + if h.new.as_ref().is_some_and(|t| t.partial) { + return Err(PlanError::PartialUpdate { + lsn: input.input.source_lsn, + }); + } + let change = ViewChange { + op: h.op, + xid: input.input.xid, + source_lsn: input.input.source_lsn, + new: h.new.as_ref(), + old: h.old.as_ref(), + }; + self.writer.push_views(change, table)?; + } + Ok(true) + } + /// Freeze the validated plan for the executor pub fn seal(self, commit_lsn: u64, commit_ts: i64) -> Result { let plan = self.writer.seal(commit_lsn, commit_ts)?; @@ -221,7 +294,7 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { mod tests { use super::*; use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, DecodedTuple, HeapOp, ToastPointer, + ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, ToastPointer, }; use crate::emit::pipeline::plan_spool::PlanItem; use crate::mapping::{TableMapping, TableTarget}; @@ -361,6 +434,48 @@ mod tests { ) } + /// Route carrying source column 1 as `Nullable(String)` + fn text_route() -> Arc { + RouteSnapshot::freeze( + Arc::new(TableMapping { + target: TableTarget::new("db", "t"), + columns: vec![crate::mapping::ColumnMapping { + src_attnum: 1, + target_name: "v".into(), + target_type: "Nullable(String)".into(), + type_pinned: false, + }], + tees: Vec::new(), + }), + Arc::default(), + Default::default(), + ) + } + + fn planner<'a>( + path: impl Into, + cap: u64, + segment_rows: usize, + view: &'a mut MapView, + resolver: &'a ToastResolver, + ) -> Planner<'a, MapView> { + Planner::create(path, cap, (segment_rows, usize::MAX), 42, view, resolver) + } + + /// `_lsn` of every row in `chunk`, after `mapped` columns + fn lsns(chunk: &crate::emit::ch_emitter::EncodedChunk, mapped: usize) -> Vec { + let crate::emit::ch_emitter::Wire::Fixed { bytes, .. } = chunk.buffers[mapped].wire() + else { + panic!("lsn column is fixed width"); + }; + bytes + .as_chunks::<8>() + .0 + .iter() + .map(|b| u64::from_le_bytes(*b)) + .collect() + } + /// Fixed-map view: routes by rel name until a Dropped event for the oid /// folds in, then unmapped. Records applied entries struct MapView { @@ -371,11 +486,11 @@ mod tests { } impl PlanRouteView for MapView { - fn route_for(&mut self, heap: &DescribedHeap) -> Option> { - if self.dropped.contains(&heap.descriptor.oid) { + fn route_for(&mut self, rel: &Arc) -> Option> { + if self.dropped.contains(&rel.oid) { return None; } - self.routes.get(&heap.descriptor.rel_name).cloned() + self.routes.get(&rel.rel_name).cloned() } fn unmapped(&mut self, rel: &RelDescriptor, heaps: u64) { @@ -432,7 +547,7 @@ mod tests { unmapped: Vec::new(), }; let resolver = ToastResolver::disabled(); - let mut planner = Planner::create(tmp.path().join("1.plan"), 1 << 20, &mut view, &resolver); + let mut planner = planner(tmp.path().join("1.plan"), 1 << 20, 1, &mut view, &resolver); let mut drain = b .drain_committed(StashResolved::nothing_stashed(1), 42, 0x2000, &[], false) .await @@ -454,9 +569,14 @@ mod tests { let mut rd = plan.replay().unwrap(); let mut order = Vec::new(); while let Some(item) = rd.next_item().unwrap() { - order.push(match item { - PlanItem::Control(_) => "e".to_string(), - PlanItem::Heap(h) | PlanItem::Truncate { heap: h, .. } => format!( + match item { + PlanItem::Control { .. } => order.push("e".to_string()), + PlanItem::Segment(s) => { + for c in &s.chunks { + order.extend(lsns(c, 0).iter().map(|lsn| format!("h{lsn}r"))); + } + } + PlanItem::Truncate { heap: h, .. } => order.push(format!( "h{}{}", h.decoded.source_lsn, if rd.tables[h.table as usize].route.is_some() { @@ -464,8 +584,8 @@ mod tests { } else { "-" } - ), - }); + )), + } } assert_eq!( order, @@ -514,7 +634,7 @@ mod tests { Arc::new(crate::emit::ch_emitter::EmitterStats::default()), ); let path = tmp.path().join("1.plan"); - let mut planner = Planner::create(path.clone(), 1 << 20, &mut view, &resolver); + let mut planner = planner(path.clone(), 1 << 20, usize::MAX, &mut view, &resolver); let mut drain = b .drain_committed(StashResolved::nothing_stashed(1), 42, 0x2000, &[], false) .await @@ -563,7 +683,7 @@ mod tests { }; let resolver = ToastResolver::disabled(); let path = tmp.path().join("1.plan"); - let mut planner = Planner::create(path.clone(), 1 << 20, &mut view, &resolver); + let mut planner = planner(path.clone(), 1 << 20, usize::MAX, &mut view, &resolver); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -597,7 +717,8 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = descriptor(16612, INT4OID); - const ROWS: usize = 32 * 1024; + // Encoded rows of empty mappings cost about 21 bytes each + const ROWS: usize = 64 * 1024; for i in 0..ROWS { b.on_heap(heap(&rel, 100 + i as u64, ColumnValue::Int4(i as i32))) .await @@ -611,7 +732,13 @@ mod tests { unmapped: Vec::new(), }; let resolver = ToastResolver::disabled(); - let mut planner = Planner::create(tmp.path().join("1.plan"), 1 << 30, &mut view, &resolver); + let mut planner = planner( + tmp.path().join("1.plan"), + 1 << 30, + 4096, + &mut view, + &resolver, + ); let mut drain = b .drain_committed(StashResolved::nothing_stashed(1), 42, 0x2000, &[], false) .await @@ -630,16 +757,20 @@ mod tests { "plan crossed mem threshold into file spool" ); let mut rd = plan.replay().unwrap(); - let mut rows = 0usize; + let mut rows = 0u64; while let Some(item) = rd.next_item().unwrap() { - // No controls planned, every heap routed - assert!( - matches!(item, PlanItem::Heap(h) if rd.tables[h.table as usize].route.is_some()) + // No controls planned, every row routed + let PlanItem::Segment(s) = item else { + panic!("segments only"); + }; + assert_eq!( + s.rows, + s.chunks.iter().map(|c| c.rows() as u64).sum::() ); - rows += 1; + rows += s.rows; } drop(rd); - assert_eq!(rows, ROWS); + assert_eq!(rows, ROWS as u64); } /// Mirror-row refs and fences survive multi-batch planning re-based to @@ -692,7 +823,13 @@ mod tests { unmapped: Vec::new(), }; let resolver = ToastResolver::disabled(); - let mut planner = Planner::create(tmp.path().join("1.plan"), 1 << 20, &mut view, &resolver); + let mut planner = planner( + tmp.path().join("1.plan"), + 1 << 20, + usize::MAX, + &mut view, + &resolver, + ); let mut drain = b .drain_committed(StashResolved::nothing_stashed(1), 42, 0x2000, &[], true) .await @@ -708,16 +845,20 @@ mod tests { let plan = planner.seal(0x2000, 42).unwrap(); let total_rows: usize = plan.row_batches.iter().map(|rb| rb.len()).sum(); assert_eq!(total_rows, 2, "both chunk births carried"); - assert_eq!(plan.heap_count, 3, "2 inserts + truncate"); - assert_eq!(plan.controls.len(), 1); - assert_eq!( - plan.controls[0].row_idx, 2, - "event fence re-based to plan-global rows" - ); + assert_eq!(plan.routed_count(), 3, "2 inserts + truncate"); + let mut rd = plan.replay().unwrap(); + let mut fences = Vec::new(); + while let Some(item) = rd.next_item().unwrap() { + match item { + PlanItem::Control { row_idx, .. } => fences.push(("event", row_idx)), + PlanItem::Truncate { upto, .. } => fences.push(("truncate", upto)), + PlanItem::Segment(_) => {} + } + } assert_eq!( - plan.truncate_rows, - vec![2], - "truncate fence re-based to plan-global rows" + fences, + [("event", 2), ("truncate", 2)], + "fences re-based to plan-global rows" ); } @@ -747,7 +888,7 @@ mod tests { }; let resolver = ToastResolver::disabled(); let path = tmp.path().join("1.plan"); - let mut planner = Planner::create(path.clone(), 1 << 20, &mut view, &resolver); + let mut planner = planner(path.clone(), 1 << 20, usize::MAX, &mut view, &resolver); let mut drain = b .drain_committed(StashResolved::nothing_stashed(1), 42, 0x2000, &[], false) .await @@ -769,7 +910,7 @@ mod tests { } else { res.unwrap(); let plan = planner.seal(0x2000, 42).unwrap(); - assert_eq!(plan.heap_count, 0, "unmapped partial heap is discarded"); + assert_eq!(plan.routed_rows, 0, "unmapped partial heap is discarded"); assert_eq!(plan.tables[0].heaps, 1); } } @@ -802,7 +943,13 @@ mod tests { unmapped: Vec::new(), }; let resolver = ToastResolver::disabled(); - let mut planner = Planner::create(tmp.path().join("1.plan"), 1 << 20, &mut view, &resolver); + let mut planner = planner( + tmp.path().join("1.plan"), + 1 << 20, + usize::MAX, + &mut view, + &resolver, + ); let mut drain = b .drain_committed(StashResolved::nothing_stashed(1), 42, 0x2000, &[], false) .await @@ -813,7 +960,7 @@ mod tests { drain.finish().await.unwrap(); let plan = planner.seal(0x2000, 42).unwrap(); assert_eq!( - plan.heap_count, 0, + plan.routed_rows, 0, "unmapped heap is counted without being stored" ); assert_eq!(plan.tables[0].heaps, 1); @@ -857,13 +1004,13 @@ mod tests { dir: &std::path::Path, ) -> Vec { let mut view = MapView { - routes: HashMap::from_iter([(rel.rel_name.clone(), route())]), + routes: HashMap::from_iter([(rel.rel_name.clone(), text_route())]), dropped: HashSet::new(), applied: Vec::new(), unmapped: Vec::new(), }; let path = dir.join(format!("{xid}.plan")); - let mut planner = Planner::create(path, 1 << 20, &mut view, resolver); + let mut planner = planner(path, 1 << 20, usize::MAX, &mut view, resolver); let commit = u64::from(xid) << 12 | 0x100; let stash = StashResolved::nothing_stashed(xid); let mut drain = b @@ -880,10 +1027,27 @@ mod tests { let mut rd = plan.replay().unwrap(); let mut out = Vec::new(); while let Some(item) = rd.next_item().unwrap() { - if let PlanItem::Heap(h) = item { - let d = h.decoded; - let t = d.new.or(d.old).expect("row image"); - out.push(t.columns[0].clone().expect("column")); + let PlanItem::Segment(s) = item else { continue }; + for c in &s.chunks { + let crate::emit::ch_emitter::Wire::NullableString { + offsets, + data, + null_map, + .. + } = c.buffers[0].wire() + else { + panic!("text column"); + }; + let mut start = 0; + for (end, null) in offsets.iter().zip(null_map) { + let body = &data[start..*end as usize]; + start = *end as usize; + out.push(if *null == 1 { + ColumnValue::Null + } else { + ColumnValue::Text(String::from_utf8(body.to_vec()).unwrap()) + }); + } } } out diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index 88c07b32..1ada5aa1 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -27,7 +27,7 @@ use crate::backfill::visibility_pending::{self, SharedPendingLedger}; use crate::catalog::pending::PendingCatalog; #[cfg(feature = "test-support")] use crate::crash_test; -use crate::decode::heap_decoder::{DecodedHeap, DescribedHeap}; +use crate::decode::heap_decoder::DecodedHeap; use crate::decode::visibility::{PgXactView, SealedPatch, read_pg_xact}; use crate::emit::ch_ddl::{DdlApplicator, Stamp}; use crate::emit::ch_emitter::EmitterStats; @@ -48,9 +48,8 @@ use crate::xact::xact_buffer::{ use crate::config::ResolvedConfig; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::{AckHandle, BufferedRowSender, Publish, SeqAlloc}; -use crate::emit::pipeline::batcher::{BatcherMsg, RowChunk}; -use crate::emit::pipeline::decode; -use crate::emit::pipeline::plan_spool::{PlanItem, PlanPath, PlanTable, PlannedHeap, SealedPlan}; +use crate::emit::pipeline::batcher::{BatcherMsg, PlacedSegment}; +use crate::emit::pipeline::plan_spool::{PlanItem, PlanPath, SealedPlan, Segment}; use crate::emit::pipeline::planner::{PlanRouteView, Planner, drain_reason}; use crate::emit::route::{RouteSnapshot, RowPolicy, route_via}; use crate::mapping::{MappingSnapshot, TableMapping}; @@ -162,7 +161,7 @@ struct Outbox { seqs: SeqAlloc, /// Placed rows not yet on the batcher channel, shipped per worker batch /// so small commits share one channel hop and wakeup - staged: Vec, + staged: Vec, staged_rows: usize, staged_bytes: usize, /// Global resident-payload pool; slice and chunk generation admission @@ -207,7 +206,7 @@ impl BufferedRowSender for Outbox { let sent = tokio::select! { biased; msg = self.fatal.wait() => return Err(SinkError::Other(msg)), - r = self.msg_tx.send(BatcherMsg::Chunks(chunks)) => r, + r = self.msg_tx.send(BatcherMsg::Segments(chunks)) => r, }; sent.map_err(|_| SinkError::Other("batcher channel closed".into())) } @@ -884,47 +883,36 @@ impl ReorderSink { Ok(()) } - /// Place accumulated planned heaps as one seq under a fresh admission - /// permit; values detoasted at planning. `publish` marks the commit's - /// final data segment so its seq carries the LSN publication (no - /// trailing marker needed) - /// - /// Takes `&mut self` so the borrow across awaits is `&mut Self` (Send): - /// owned `DdlApplicator` is Send but not Sync - async fn dispatch_planned( + /// Place one planned segment as one seq under a fresh admission permit; + /// values detoasted and encoded at planning. `publish` marks the + /// commit's final data segment so its seq carries the LSN publication + /// (no trailing marker needed) + async fn dispatch_segment( &mut self, - tables: &[PlanTable], - pending: &mut Vec, - pending_bytes: &mut usize, - commit_ts: i64, + segment: Segment, commit_lsn: u64, publish: Publish, ) -> Result<(), SinkError> { - if pending.is_empty() { - return Ok(()); - } - let heaps = std::mem::take(pending); - let bytes = std::mem::take(pending_bytes); - let permit = self.out.admit(bytes).await?.map(Arc::new); + let permit = self.out.admit(segment.bytes).await?.map(Arc::new); let seq = self.out.seqs.open(commit_lsn, publish); self.stats.queue_jobs_out.fetch_add(1, Ordering::Relaxed); + self.stats.decode_jobs_in.fetch_add(1, Ordering::Relaxed); + self.stats + .decode_rows_out + .fetch_add(segment.rows, Ordering::Relaxed); let chunk_rows = self.dest.current().decode_chunk_rows; - let rows = decode::place_rows( - &mut self.out.staged, - &self.stats, - chunk_rows, - seq.seq(), - commit_ts, - commit_lsn, - tables, - heaps, - permit, - ); let out = &mut self.out; - out.seqs.place(seq, rows); - out.staged_rows += rows as usize; - out.staged_bytes += bytes; - if out.staged_rows >= chunk_rows || out.staged_bytes >= decode::DECODE_CHUNK_BYTES { + out.staged.push(PlacedSegment { + seq: seq.seq(), + chunks: segment.chunks, + permit, + }); + out.seqs.place(seq, segment.rows); + out.staged_rows += segment.rows as usize; + out.staged_bytes += segment.bytes; + if out.staged_rows >= chunk_rows + || out.staged_bytes >= crate::emit::pipeline::batcher::DECODE_CHUNK_BYTES + { out.flush_pending().await?; } Ok(()) @@ -950,28 +938,26 @@ impl ReorderSink { commit_ts, }; let mut rd = plan.replay().map_err(SinkError::ctx("plan replay"))?; - let mut pending: Vec = Vec::new(); - let mut pending_bytes = 0usize; + self.stats + .deletes_discarded + .fetch_add(rd.deletes_discarded, Ordering::Relaxed); + // Segment held until the next step shows whether it is final + let mut pending: Option = None; let mut rows_cursor = 0usize; let total_rows: usize = rd.row_batches.iter().map(|rb| rb.len()).sum(); let mut rows_total = 0u64; while let Some(item) = rd.next_item().map_err(SinkError::ctx("plan replay"))? { match item { - PlanItem::Control(c) => { + PlanItem::Control { event, row_idx } => { Box::pin(async { - self.put_plan_rows(&rd.row_batches, &mut rows_cursor, c.row_idx) + self.put_plan_rows(&rd.row_batches, &mut rows_cursor, row_idx) .await?; - self.dispatch_planned( - &rd.tables, - &mut pending, - &mut pending_bytes, - commit_ts, - commit_lsn, - Publish::Partial, - ) - .await?; + if let Some(segment) = pending.take() { + self.dispatch_segment(segment, commit_lsn, Publish::Partial) + .await?; + } let fenced = self.barrier_fence().await?; - self.apply_drain_entry(fenced, db, &c.event, at).await + self.apply_drain_entry(fenced, db, &event, at).await }) .await?; } @@ -979,15 +965,10 @@ impl ReorderSink { Box::pin(async { self.put_plan_rows(&rd.row_batches, &mut rows_cursor, upto) .await?; - self.dispatch_planned( - &rd.tables, - &mut pending, - &mut pending_bytes, - commit_ts, - commit_lsn, - Publish::Partial, - ) - .await?; + if let Some(segment) = pending.take() { + self.dispatch_segment(segment, commit_lsn, Publish::Partial) + .await?; + } let fenced = self.barrier_fence().await?; let rel = &rd.tables[h.table as usize].descriptor; self.apply_truncate(fenced, rel, &h.decoded, commit_ts) @@ -995,20 +976,11 @@ impl ReorderSink { }) .await?; } - PlanItem::Heap(h) => { - rows_total += 1; - pending_bytes += h.bytes; - pending.push(h); - if pending.len() >= self.batch_rows || pending_bytes >= self.batch_bytes { - self.dispatch_planned( - &rd.tables, - &mut pending, - &mut pending_bytes, - commit_ts, - commit_lsn, - Publish::Partial, - ) - .await?; + PlanItem::Segment(segment) => { + rows_total += segment.rows; + if let Some(prev) = pending.replace(segment) { + self.dispatch_segment(prev, commit_lsn, Publish::Partial) + .await?; } } } @@ -1016,16 +988,11 @@ impl ReorderSink { if rows_cursor < total_rows { Box::pin(self.put_plan_rows(&rd.row_batches, &mut rows_cursor, total_rows)).await?; } - let published = !pending.is_empty(); - self.dispatch_planned( - &rd.tables, - &mut pending, - &mut pending_bytes, - commit_ts, - commit_lsn, - Publish::Commit, - ) - .await?; + let published = pending.is_some(); + if let Some(segment) = pending { + self.dispatch_segment(segment, commit_lsn, Publish::Commit) + .await?; + } Ok((rows_total, published)) } @@ -1164,8 +1131,14 @@ impl ReorderSink { let (batch_rows, batch_bytes) = (self.batch_rows, self.batch_bytes); let out = &mut self.out; let stats = self.stats.clone(); - let mut planner = - Planner::create(plan_path, self.plan_disk_max, &mut view, &resolver); + let mut planner = Planner::create( + plan_path, + self.plan_disk_max, + (batch_rows, batch_bytes), + commit_ts, + &mut view, + &resolver, + ); loop { let Some(batch) = drain .next_batch_admitting(batch_rows, batch_bytes, async |bytes| { @@ -1322,6 +1295,7 @@ fn bump_plan_failure(stats: &EmitterStats, reason: &'static str) { "detoast" => &stats.plan_failures_detoast, "partial_update" => &stats.plan_failures_partial_update, "view" => &stats.plan_failures_view, + "encode" => &stats.plan_failures_encode, _ => &stats.plan_failures_drain, }; counter.fetch_add(1, Ordering::Relaxed); @@ -1442,20 +1416,18 @@ impl ReorderRouteView<'_> { } impl PlanRouteView for ReorderRouteView<'_> { - fn route_for(&mut self, heap: &DescribedHeap) -> Option> { - let rel_name = &heap.descriptor.rel_name; + fn route_for(&mut self, rel: &Arc) -> Option> { + let rel_name = &rel.rel_name; if let Some((desc, r)) = self.maps.memo.get(rel_name) - && Arc::ptr_eq(desc, &heap.descriptor) + && Arc::ptr_eq(desc, rel) { r.clone() } else { let rules = self.config.as_ref().map(|rc| rc.rules.clone()); - let route = route_via(&heap.descriptor, rules.as_deref(), |owner| { - self.owned_route(owner) - }); + let route = route_via(rel, rules.as_deref(), |owner| self.owned_route(owner)); self.maps .memo - .insert(rel_name.clone(), (heap.descriptor.clone(), route.clone())); + .insert(rel_name.clone(), (rel.clone(), route.clone())); route } } @@ -1583,7 +1555,6 @@ impl RecordSink for ReorderSink { #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{DecodedHeap, HeapOp}; use crate::emit::pipeline::tail::{self, TailParts}; use crate::mapping::TableTarget; use crate::xact::xact_buffer::raw_fixtures::int4_descriptor; @@ -1597,21 +1568,10 @@ mod tests { } } - fn heap_of(rel: &RelName) -> DescribedHeap { + fn rel_of(rel: &RelName) -> Arc { let mut desc = (*int4_descriptor(16400)).clone(); desc.rel_name = rel.clone(); - DescribedHeap { - decoded: DecodedHeap { - rfn: desc.rfn, - xid: 1, - source_lsn: 0x100, - op: HeapOp::Insert, - new: None, - old: None, - }, - descriptor: Arc::new(desc), - descriptor_valid_from: 0x40, - } + Arc::new(desc) } #[tokio::test] @@ -1632,8 +1592,8 @@ mod tests { stats, RouteMaps::default(), ); - assert!(view.route_for(&heap_of(&planned)).is_some()); - assert!(view.route_for(&heap_of(&added)).is_none()); + assert!(view.route_for(&rel_of(&planned)).is_some()); + assert!(view.route_for(&rel_of(&added)).is_none()); handle.publish(Arc::new(HashMap::from_iter([( added.clone(), @@ -1641,11 +1601,11 @@ mod tests { )]))); assert!( - view.route_for(&heap_of(&planned)).is_some(), + view.route_for(&rel_of(&planned)).is_some(), "republish can't unroute rows already planned" ); assert!( - view.route_for(&heap_of(&added)).is_none(), + view.route_for(&rel_of(&added)).is_none(), "republish can't route rows this transaction planned unmapped" ); } @@ -1711,7 +1671,7 @@ mod tests { crate::mapping::mapping_handle(HashMap::from_iter([(rel.clone(), mapping_for(&rel))])); let snapshot = handle.snapshot(); let stats = Arc::new(EmitterStats::default()); - let heap = heap_of(&rel); + let heap = rel_of(&rel); let memoized = |view: &ReorderRouteView, desc: &Arc| { view.maps .memo @@ -1743,20 +1703,16 @@ mod tests { Arc::ptr_eq(&first, &view.route_for(&heap).unwrap()), "next commit reuses the carried route" ); - assert!(memoized(&view, &heap.descriptor)); + assert!(memoized(&view, &heap)); - let mut altered = heap.clone(); - altered.descriptor = Arc::new((*heap.descriptor).clone()); + let altered = Arc::new((*heap).clone()); view.route_for(&altered).expect("mapped"); - assert!( - memoized(&view, &altered.descriptor), - "new descriptor version rebuilds" - ); + assert!(memoized(&view, &altered), "new descriptor version rebuilds"); // Catalog event folded mid-plan drops the memo even without an // applicator to predict its mapping view.apply(&DrainEntry::Catalog(SchemaEvent::Dropped { - oid: altered.descriptor.oid, + oid: altered.oid, rel_name: rel.clone(), partition: None, })) @@ -1771,7 +1727,7 @@ mod tests { .await .unwrap(); assert!( - memoized(&view, &altered.descriptor), + memoized(&view, &altered), "non-catalog entries keep the memo" ); } @@ -1798,9 +1754,10 @@ mod tests { let fatal = Fatal::new(); let (mut out, _tail) = outbox(&budget, &fatal); let staged = budget.try_admit(768 << 10).unwrap(); - out.staged.push(RowChunk { + out.staged.push(PlacedSegment { + seq: 0, + chunks: Vec::new(), permit: Some(Arc::new(staged)), - ..Default::default() }); let permit = tokio::time::timeout(Duration::from_secs(5), out.admit(512 << 10)) .await diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 12c3ef51..50d4a881 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -217,6 +217,18 @@ fn swallow(ack: &AckHandle, chunks: impl IntoIterator) ack.swallowed(counts.into_iter().collect()); } +/// [`swallow`] for planned segments +fn swallow_segments(ack: &AckHandle, segments: Vec) { + let mut counts: HashMap = HashMap::new(); + for s in segments { + let rows = s.chunks.iter().map(|c| c.rows() as u64).sum::(); + if rows > 0 { + *counts.entry(s.seq).or_insert(0) += rows; + } + } + ack.swallowed(counts.into_iter().collect()); +} + /// Metrics-only tail: ack collector + one swallow task, zero CH /// connections. Every routed row acks at swallow (permits release on /// drop) so nothing can pin the watermark; `FlushAll` replies @@ -234,7 +246,7 @@ pub fn spawn_null( while let Some(msg) = msg_rx.recv().await { match msg { BatcherMsg::Rows(chunk) => swallow(&swallow_ack, [chunk]), - BatcherMsg::Chunks(chunks) => swallow(&swallow_ack, chunks), + BatcherMsg::Segments(segments) => swallow_segments(&swallow_ack, segments), BatcherMsg::FlushAll(reply) => { let _ = reply.send(()); } diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index b4de878b..bb42b4bb 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -377,7 +377,7 @@ snapshot! { "Sealed transaction-plan bytes by final backing.", /// Planning failures `[spool, fail_closed, detoast, partial_update, /// view, drain]`, rendered `reason=` labelled - counter xact_plan_failures_by_reason: [u64; 11] ["reason" = PLAN_FAILURE_REASONS] = + counter xact_plan_failures_by_reason: [u64; 12] ["reason" = PLAN_FAILURE_REASONS] = "Planning-stage failures; the whole transaction emits nothing.", counter raw_stash_bytes_by_storage: [u64; 2] ["storage" = ["mem", "spill"]] = "Cumulative raw-stash payload bytes by first landing.", @@ -739,7 +739,7 @@ impl Default for RateEstimator { /// `reason=` label order of `MetricsSnapshot::xact_plan_failures_by_reason`, /// matching [`crate::emit::pipeline::planner::drain_reason`] -const PLAN_FAILURE_REASONS: [&str; 11] = [ +const PLAN_FAILURE_REASONS: [&str; 12] = [ "spool", "fail_closed_image_only", "fail_closed_malformed", @@ -750,6 +750,7 @@ const PLAN_FAILURE_REASONS: [&str; 11] = [ "detoast", "partial_update", "view", + "encode", "drain", ]; @@ -1225,7 +1226,7 @@ mod tests { let snap = MetricsSnapshot { xact_plan_rows: 9, xact_plan_bytes_by_storage: [100, 200], - xact_plan_failures_by_reason: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + xact_plan_failures_by_reason: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], ..MetricsSnapshot::default() }; let body = render(snap); diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 54bd4174..495c45e2 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -174,6 +174,92 @@ impl OracleColumnBuf { pub fn approx_size(&self) -> usize { self.wire_bytes } + + /// Request bytes this column adds: none while it resolves locally, + /// every cell once one needs conversion + pub fn request_bytes(&self) -> usize { + if self.remote_cells == 0 { + 0 + } else { + self.wire_bytes + } + } + + /// Request bytes once `other`'s cells follow these + pub fn request_bytes_with(&self, other: &Self) -> usize { + if self.remote_cells + other.remote_cells == 0 { + 0 + } else { + self.wire_bytes + other.wire_bytes + } + } + + /// Same source type and target, so cells of both share one request column + pub fn same_column(&self, other: &Self) -> bool { + ( + self.source_type_oid, + self.source_typmod, + self.string_target, + self.nullable_target, + ) == ( + other.source_type_oid, + other.source_typmod, + other.string_target, + other.nullable_target, + ) + } + + /// Append `other`'s cells, which must target the same column + pub fn append(&mut self, other: Self) { + self.cells.reserve(other.cells.len()); + for cell in other.cells { + self.push(cell); + } + } + + pub(crate) fn encode_into(&self, out: &mut Vec) { + use crate::xact::spill::{push_bytes, push_i32, push_u8, push_u32}; + push_u32(out, self.source_type_oid); + push_i32(out, self.source_typmod); + push_u8(out, u8::from(self.string_target)); + push_u8(out, u8::from(self.nullable_target)); + push_u32(out, self.cells.len() as u32); + for cell in &self.cells { + let (tag, body) = match cell { + OracleCell::Default => (0, &[][..]), + OracleCell::DiskRaw(b) => (1, &b[..]), + OracleCell::TextInput(b) => (2, &b[..]), + OracleCell::Literal(b) => (3, &b[..]), + }; + push_u8(out, tag); + if tag != 0 { + push_bytes(out, body); + } + } + } + + pub(crate) fn decode( + c: &mut crate::xact::spill::Cursor<'_>, + ) -> Result { + let mut buf = Self::with_target(c.u32()?, c.i32()?, c.u8()? != 0, c.u8()? != 0); + let n = c.u32()? as usize; + buf.cells.reserve(n.min(1 << 20)); + for _ in 0..n { + buf.push(match c.u8()? { + 0 => OracleCell::Default, + 1 => OracleCell::DiskRaw(c.bytes()?), + 2 => OracleCell::TextInput(c.bytes()?), + 3 => OracleCell::Literal(c.bytes()?), + tag => { + return Err(crate::xact::spill::SpillError::Format { + offset: 0, + detail: format!("unknown oracle cell tag {tag}"), + }); + } + }); + } + Ok(buf) + } } pub struct OracleRequestColumn<'a> { diff --git a/src/xact/spill.rs b/src/xact/spill.rs index ad343d10..b67b12fe 100644 --- a/src/xact/spill.rs +++ b/src/xact/spill.rs @@ -126,6 +126,17 @@ impl HeapInput { std::mem::size_of::() + self.bytes.len() } + /// Fields heap decode reads, borrowed without building a record + pub fn record_bytes(&self) -> crate::decode::heap_decoder::HeapRecordBytes<'_> { + let (main_data, data) = self.bytes.split_at(self.main_len as usize); + crate::decode::heap_decoder::HeapRecordBytes { + rm: self.rm, + info: self.info, + main_data, + block: self.has_block.then_some(data), + } + } + /// Borrowed record targeting `rfn` for the shared heap decoder pub fn to_xlog_record(&self, rfn: RelFileNode) -> walrus::pg::walparser::XLogRecord<'_> { use walrus::pg::walparser::{ @@ -1053,7 +1064,7 @@ pub(crate) fn push_u16(out: &mut Vec, v: u16) { pub(crate) fn push_u32(out: &mut Vec, v: u32) { out.extend_from_slice(&v.to_le_bytes()); } -fn push_i32(out: &mut Vec, v: i32) { +pub(crate) fn push_i32(out: &mut Vec, v: i32) { out.extend_from_slice(&v.to_le_bytes()); } pub(crate) fn push_u64(out: &mut Vec, v: u64) { @@ -1062,7 +1073,7 @@ pub(crate) fn push_u64(out: &mut Vec, v: u64) { fn push_i64(out: &mut Vec, v: i64) { out.extend_from_slice(&v.to_le_bytes()); } -fn push_bytes(out: &mut Vec, b: &[u8]) { +pub(crate) fn push_bytes(out: &mut Vec, b: &[u8]) { push_u32(out, b.len() as u32); out.extend_from_slice(b); } @@ -1321,7 +1332,7 @@ impl<'a> Cursor<'a> { Self { buf, pos: 0 } } - fn need(&mut self, n: usize) -> Result<&'a [u8]> { + pub(crate) fn need(&mut self, n: usize) -> Result<&'a [u8]> { if self.pos + n > self.buf.len() { return Err(SpillError::Format { offset: self.pos, @@ -1346,7 +1357,7 @@ impl<'a> Cursor<'a> { pub(crate) fn u32(&mut self) -> Result { self.take().map(u32::from_le_bytes) } - fn i32(&mut self) -> Result { + pub(crate) fn i32(&mut self) -> Result { self.take().map(i32::from_le_bytes) } pub(crate) fn u64(&mut self) -> Result { diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index b1df5259..ad9e1f27 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -158,6 +158,65 @@ fn entry_lsn(e: &SpillEntry) -> u64 { } } +/// One drained row change: cells for truncates, stash-resolved records and +/// directly buffered heaps; record bytes for covered inputs, which planning +/// encodes without building cells +#[derive(Debug, Clone, PartialEq)] +pub enum DrainHeap { + Cells(DescribedHeap), + Input(Box), +} + +impl DrainHeap { + pub fn descriptor(&self) -> &Arc { + match self { + Self::Cells(h) => &h.descriptor, + Self::Input(i) => &i.descriptor, + } + } + + pub fn descriptor_valid_from(&self) -> u64 { + match self { + Self::Cells(h) => h.descriptor_valid_from, + Self::Input(i) => i.descriptor_valid_from, + } + } + + pub fn source_lsn(&self) -> u64 { + match self { + Self::Cells(h) => h.decoded.source_lsn, + Self::Input(i) => i.input.source_lsn, + } + } + + pub fn is_truncate(&self) -> bool { + matches!(self, Self::Cells(h) if h.decoded.op == HeapOp::Truncate) + } + + pub fn approx_bytes(&self) -> usize { + match self { + Self::Cells(h) => h.approx_bytes(), + Self::Input(i) => i.approx_bytes(), + } + } + + /// Row changes this entry yields + pub fn rows(&self) -> usize { + match self { + Self::Cells(_) => 1, + Self::Input(i) => usize::from(i.input.tuples), + } + } + + /// Owned cells per tuple, through the cell adapter + pub fn into_cells(self) -> std::result::Result, XactBufferError> { + match self { + Self::Cells(h) => Ok(smallvec::smallvec![h]), + Self::Input(i) => input_heaps(&i), + } + } +} + /// Heaps an entry yields at drain fn entry_rows(e: &SpillEntry) -> usize { match e { @@ -1426,8 +1485,8 @@ impl XactBuffer { { let mut heaps = Vec::with_capacity(st.in_mem.len()); for entry in std::mem::take(&mut st.in_mem) { - match entry { - SpillEntry::Heap(h) => heaps.push(*h), + heaps.push(match entry { + SpillEntry::Heap(h) => DrainHeap::Cells(*h), SpillEntry::Input(i) => { if !xids.contains(&i.input.xid) { return Err(XactBufferError::ForeignXid { @@ -1435,11 +1494,11 @@ impl XactBuffer { top: top_xid, }); } - heaps.extend(input_heaps(&i)?); + DrainHeap::Input(i) } // Plain state holds heaps and inputs only _ => unreachable!(), - } + }); } self.stats.committed_xacts_total += 1; return Ok(CommittedDrain { @@ -1729,6 +1788,7 @@ impl MergeSource { enum MergeItem { Heap(Box), + Input(Box), /// Boxed: a config row carries every per-relation setting, so the variant /// dwarfs the heap pointer the merge yields per row Event(Box), @@ -1853,25 +1913,16 @@ impl MergedDrain { SpillEntry::Chunk(c) => self.fold_chunk(*c)?, SpillEntry::ToastDelete(d) => self.fold_delete(*d)?, SpillEntry::Raw(raw) => self.fold_raw(&raw)?, - SpillEntry::Input(i) => self.fold_input(&i)?, + SpillEntry::Input(i) => { + self.check_owned(i.input.xid)?; + return Ok(Some(MergeItem::Input(i))); + } } } } } } - /// Queue an input's tuples like a resolved raw record's - fn fold_input(&mut self, i: &DescribedInput) -> std::result::Result<(), XactBufferError> { - self.check_owned(i.input.xid)?; - for heap in input_heaps(i)? { - let sz = heap.approx_bytes(); - self.gauge.add(sz); - self.pending_gauge.add(sz); - self.pending_heaps.push_back(Box::new(heap)); - } - Ok(()) - } - fn check_owned(&self, xid: u32) -> std::result::Result<(), XactBufferError> { if self.allowed_xids.contains(&xid) { return Ok(()); @@ -2264,7 +2315,7 @@ pub struct OrderedEvent { /// slice a barrier the reorder coordinator serializes against ClickHouse. pub struct DrainedBatch { /// `source_lsn` ASC within the slice; later slices strictly follow. - pub heaps: Vec, + pub heaps: Vec, pub ordered_events: Vec, /// Chunk generations sealed so far, oldest first. A chunk's WAL position /// precedes its referrer, so every heap's value lives in exactly one @@ -2282,6 +2333,18 @@ pub struct DrainedBatch { pub is_final: bool, } +impl DrainedBatch { + /// Owned cells of every row change, inputs decoded through the cell + /// adapter + pub fn cells(&self) -> std::result::Result, XactBufferError> { + let mut out = Vec::with_capacity(self.heaps.len()); + for h in &self.heaps { + out.extend(h.clone().into_cells()?); + } + Ok(out) + } +} + /// One step of a [`DrainedBatch`] apply plan (see [`DrainedBatch::into_walk`]). pub enum WalkStep { /// Seal store rows: put `new_rows[cursor..upto]` before the next step @@ -2295,7 +2358,7 @@ pub enum WalkStep { heap: DescribedHeap, upto: usize, }, - Heap(DescribedHeap), + Heap(DrainHeap), } /// [`DrainedBatch`] decomposed into its apply plan plus the payload fields @@ -2331,14 +2394,15 @@ impl DrainedBatch { steps.push(WalkStep::Rows { upto: e.row_idx }); steps.push(WalkStep::Event(e.event)); } - if matches!(heap.decoded.op, HeapOp::Truncate) { - let upto = trunc - .next() - .expect("truncate_rows cursor per Truncate heap"); - steps.push(WalkStep::Rows { upto }); - steps.push(WalkStep::Truncate { heap, upto }); - } else { - steps.push(WalkStep::Heap(heap)); + match heap { + DrainHeap::Cells(heap) if heap.decoded.op == HeapOp::Truncate => { + let upto = trunc + .next() + .expect("truncate_rows cursor per Truncate heap"); + steps.push(WalkStep::Rows { upto }); + steps.push(WalkStep::Truncate { heap, upto }); + } + heap => steps.push(WalkStep::Heap(heap)), } } for e in events { @@ -2371,7 +2435,7 @@ enum Drained { /// Boxed so the commit path moves a pointer, not the merge state Merged(Box), /// WAL-ordered heaps needing no merge, within one slice's budget - Heaps(Vec), + Heaps(Vec), } impl CommittedDrain { @@ -2420,7 +2484,7 @@ impl CommittedDrain { })); } }; - let mut heaps: Vec = Vec::new(); + let mut heaps: Vec = Vec::new(); let mut ordered_events: Vec = Vec::new(); let mut truncate_rows: Vec = Vec::new(); let mut bytes = 0usize; @@ -2437,7 +2501,11 @@ impl CommittedDrain { truncate_rows.push(m.rows.len()); } bytes += h.approx_bytes(); - heaps.push(*h); + heaps.push(DrainHeap::Cells(*h)); + } + Some(MergeItem::Input(i)) => { + bytes += i.approx_bytes(); + heaps.push(DrainHeap::Input(i)); } } } @@ -4181,7 +4249,13 @@ mod tests { assert_eq!(buffer.stats().bytes_in_memory, 0); let mut lsns = Vec::new(); while let Some(batch) = drain.next_batch(3, usize::MAX, None).await.unwrap() { - lsns.extend(batch.heaps.into_iter().map(|heap| heap.decoded.source_lsn)); + lsns.extend( + batch + .cells() + .unwrap() + .into_iter() + .map(|heap| heap.decoded.source_lsn), + ); } assert_eq!(lsns, (2..=25).collect::>()); drain.finish().await.unwrap(); @@ -5351,7 +5425,7 @@ mod tests { let label = |e: &DrainEntry| format!("{e:?}"); let mut out = Vec::new(); let mut ev = 0usize; - for (i, h) in batch.heaps.iter().enumerate() { + for (i, h) in batch.cells().unwrap().iter().enumerate() { while ev < batch.ordered_events.len() && batch.ordered_events[ev].heap_idx <= i { out.push(label(&batch.ordered_events[ev].event)); ev += 1; @@ -5420,16 +5494,24 @@ mod tests { batch.ordered_events[0].heap_idx, 0, "same-LSN event precedes the fanout" ); - assert_eq!(batch.heaps.len(), 4, "3 fanned-out tuples + later heap"); + assert_eq!( + batch.cells().unwrap().len(), + 4, + "3 fanned-out tuples + later heap" + ); for (i, expected) in [100i32, 200, 300].iter().enumerate() { - let h = &batch.heaps[i]; + let h = &batch.cells().unwrap()[i]; assert_eq!(h.decoded.source_lsn, 100); assert_eq!(h.decoded.xid, 8, "writer xid survives subxact merge"); assert_eq!(h.descriptor_valid_from, 0x50); let new = h.decoded.new.as_ref().unwrap(); assert_eq!(new.columns[0], Some(ColumnValue::Int4(*expected))); } - assert_eq!(batch.heaps[3].decoded.source_lsn, 200, "fanout first"); + assert_eq!( + batch.cells().unwrap()[3].decoded.source_lsn, + 200, + "fanout first" + ); drain.finish().await.unwrap(); } @@ -5455,7 +5537,7 @@ mod tests { .await .unwrap() .expect("first slice"); - assert_eq!(batch.heaps.len(), 2); + assert_eq!(batch.cells().unwrap().len(), 2); assert_eq!(b.raw_pending_rows(), 1, "third fanned tuple still queued"); assert!(b.raw_pending_bytes() > 0); let ops = stats.raw_decode_ordinary_ops.load(); @@ -5467,7 +5549,7 @@ mod tests { .unwrap() .expect("final slice"); assert!(batch.is_final); - assert_eq!(batch.heaps.len(), 1); + assert_eq!(batch.cells().unwrap().len(), 1); assert_eq!(b.raw_pending_rows(), 0); assert_eq!(b.raw_pending_bytes(), 0); drain.finish().await.unwrap(); @@ -5851,7 +5933,11 @@ mod tests { .await .unwrap() .expect("one slice"); - assert_eq!(batch.heaps.len(), 2, "record at through_lsn decodes"); + assert_eq!( + batch.cells().unwrap().len(), + 2, + "record at through_lsn decodes" + ); drain.finish().await.unwrap(); } @@ -6011,8 +6097,13 @@ mod tests { .unwrap() .expect("one slice"); assert!(batch.is_final); - assert_eq!(batch.heaps.len(), 1, "maintenance skipped, row kept"); - let new = batch.heaps[0].decoded.new.as_ref().unwrap(); + assert_eq!( + batch.cells().unwrap().len(), + 1, + "maintenance skipped, row kept" + ); + let cells = batch.cells().unwrap(); + let new = cells[0].decoded.new.as_ref().unwrap(); assert_eq!(new.columns[0], Some(ColumnValue::Int4(7))); drain.finish().await.unwrap(); } @@ -6108,7 +6199,8 @@ mod tests { .expect("one slice"); assert!(batch.is_final); let ops: Vec<_> = batch - .heaps + .cells() + .unwrap() .iter() .map(|h| (h.decoded.source_lsn, h.decoded.op)) .collect(); @@ -6122,7 +6214,12 @@ mod tests { ], ); assert_eq!( - batch.heaps[3].decoded.new.as_ref().unwrap().columns[0], + batch.cells().unwrap()[3] + .decoded + .new + .as_ref() + .unwrap() + .columns[0], Some(ColumnValue::Int4(9)), ); drain.finish().await.unwrap(); @@ -6169,7 +6266,8 @@ mod tests { .unwrap() .expect("one slice"); let shapes: Vec<(u64, usize, u64)> = batch - .heaps + .cells() + .unwrap() .iter() .map(|h| { ( @@ -6218,9 +6316,12 @@ mod tests { .await .unwrap() .expect("one slice"); - assert_eq!(batch.heaps.len(), 1); - assert_eq!(batch.heaps[0].descriptor.rel_name, owner.rel_name); - assert_eq!(batch.heaps[0].descriptor_valid_from, 0x50); + assert_eq!(batch.cells().unwrap().len(), 1); + assert_eq!( + batch.cells().unwrap()[0].descriptor.rel_name, + owner.rel_name + ); + assert_eq!(batch.cells().unwrap()[0].descriptor_valid_from, 0x50); drain.finish().await.unwrap(); } @@ -6257,7 +6358,8 @@ mod tests { while let Some(batch) = drain.next_batch(64, usize::MAX, None).await.unwrap() { heaps.extend( batch - .heaps + .cells() + .unwrap() .iter() .map(|h| (h.decoded.source_lsn, h.decoded.xid)), ); @@ -6496,7 +6598,7 @@ mod tests { .await .unwrap() .expect("one slice"); - assert_eq!(batch.heaps.len(), 2); + assert_eq!(batch.cells().unwrap().len(), 2); assert!(!drain.spilled()); drop(drain); assert!(spill_files(tmp.path()).is_empty()); @@ -6533,7 +6635,7 @@ mod tests { let mut lsns = Vec::new(); let mut finals = Vec::new(); while let Some(batch) = drain.next_batch(4, usize::MAX, None).await.unwrap() { - lsns.extend(batch.heaps.iter().map(|h| h.decoded.source_lsn)); + lsns.extend(batch.cells().unwrap().iter().map(|h| h.decoded.source_lsn)); finals.push(batch.is_final); if batch.is_final { break; @@ -6573,7 +6675,7 @@ mod tests { .sum::(); let mut rows = 0usize; while let Some(batch) = drain.next_batch(4, usize::MAX, None).await.unwrap() { - rows += batch.heaps.len(); + rows += batch.cells().unwrap().len(); let resident = b.drain_resident_bytes(); assert!( resident < read_buffers + total / 4, @@ -6884,7 +6986,7 @@ mod tests { .await .unwrap() .expect("stashed rows drain"); - assert_eq!(batch.heaps.len(), 2); + assert_eq!(batch.cells().unwrap().len(), 2); drain.finish().await.unwrap(); assert_eq!(live.load(Ordering::Relaxed), 1, "commit"); @@ -6970,7 +7072,8 @@ mod tests { .unwrap() .expect("one slice"); let values: Vec<_> = out - .heaps + .cells() + .unwrap() .iter() .map(|h| h.decoded.new.as_ref().unwrap().columns[0].clone()) .collect(); @@ -6980,9 +7083,10 @@ mod tests { [int4(1), int4(10), int4(2), int4(20), int4(3), int4(30)] ); assert!( - out.heaps + out.cells() + .unwrap() .iter() - .all(|h| Arc::ptr_eq(&h.descriptor, &out.heaps[0].descriptor)) + .all(|h| Arc::ptr_eq(&h.descriptor, &out.cells().unwrap()[0].descriptor)) ); drain.finish().await.unwrap(); } diff --git a/tests/xact_buffer.rs b/tests/xact_buffer.rs index cb357f94..e1cfb5d2 100644 --- a/tests/xact_buffer.rs +++ b/tests/xact_buffer.rs @@ -249,7 +249,10 @@ async fn drain_all( let ref_maps: Vec<&ChunkRefMap> = walk.chunks.iter().map(|g| g.map()).collect(); let spool = walk.chunks.iter().find_map(|g| g.spool()); for step in walk.steps { - if let WalkStep::Heap(mut heap) = step { + let WalkStep::Heap(drained) = step else { + continue; + }; + for mut heap in drained.into_cells()? { detoast_heap(&mut heap, spool, &ref_maps, resolver).await?; out.push(CommittedTuple { decoded: heap.decoded, From cd30d34dd819be8ffe9d70e460e1aaf255286038 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:36:36 +0000 Subject: [PATCH 06/20] Keep sealed Native blocks compressed until INSERT acknowledgement Batcher separates Native block size from INSERT size: an open INSERT seals blocks at 1,048,576 rows, 16 MiB of column bytes, or when the oracle request would overflow, while row, byte and deadline triggers still bound the INSERT. Resolvers resolve each block, serialize it once through clickhouse-c BlockBuilder::write, compress it with the configured [ch] compression, drop slabs and oracle output, and shrink the output permit to stored bytes. Bodies that do not shrink stay raw. Inserters send every block of a batch as its own Data packet under one query, opening one block at a time into a per-inserter scratch buffer; retries resend all blocks in order and acknowledge sequence counts only after EndOfStream from every destination. This is the bounded decompression intermediate step: clickhouse-c-rs 0.2.2 has no send API for prepared payloads. New counters: walshadow_resident_seal_seconds_total and resident raw/stored bytes, whose ratio is achieved compression. inserter_encode_seconds_total now times decompress and decode. Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 8 + src/backfill/backfill_staging.rs | 4 +- src/bin/stream/metrics_publish.rs | 3 + src/ch/test_support.rs | 7 +- src/emit/ch_ddl.rs | 2 +- src/emit/ch_emitter.rs | 6 + src/emit/pipeline/batcher.rs | 126 ++++++++---- src/emit/pipeline/inserter.rs | 228 +++++++-------------- src/emit/pipeline/mod.rs | 1 + src/emit/pipeline/resolver.rs | 251 +++++++++++++++++++---- src/emit/pipeline/sealed.rs | 325 ++++++++++++++++++++++++++++++ src/emit/pipeline/tail.rs | 1 + src/ops/metrics.rs | 8 +- 13 files changed, 729 insertions(+), 241 deletions(-) create mode 100644 src/emit/pipeline/sealed.rs diff --git a/docs/configuration.md b/docs/configuration.md index 9bb072db..2e997d4f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -339,6 +339,14 @@ sealed batch can hold one additional batch. A single oversized row may exceed these limits. Oracle responses and native client buffers remain additional costs Sealed-buffer permits stay held through resolution, retries and insert completion. +After resolution each Native block is serialized once and compressed with +`[ch] compression`, and its permit shrinks to compressed bytes; blocks that do +not shrink stay uncompressed. Each inserter decompresses one block at a time to +send it, so one block of scratch per inserter adds to these limits. An INSERT +carries several blocks, sealed at 1,048,576 rows or 16 MiB of column bytes and +when oracle request size fills, while batch limits still bound each INSERT. +`walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` +reports achieved compression This pool is separate from decoded-payload admission to avoid circular waits. Live byte-budget reductions lower flush thresholds; increasing sealed-buffer capacity requires restarting walshadow diff --git a/src/backfill/backfill_staging.rs b/src/backfill/backfill_staging.rs index 01667872..45f1de02 100644 --- a/src/backfill/backfill_staging.rs +++ b/src/backfill/backfill_staging.rs @@ -337,7 +337,7 @@ mod tests { let sql = "DROP TABLE IF EXISTS staging"; for retries in 0..=2 { let (config, server) = - crate::ch::test_support::retry_server(retries, sql, false, false).await; + crate::ch::test_support::retry_server(retries, sql, 0, false).await; let mut session = StagingSession::connect(DestEmitter::new(Arc::new(config), None)) .await .unwrap(); @@ -349,7 +349,7 @@ mod tests { #[tokio::test] async fn exchange_ambiguity_does_not_retry() { let sql = "EXCHANGE TABLES staging AND live"; - let (config, server) = crate::ch::test_support::retry_server(2, sql, false, false).await; + let (config, server) = crate::ch::test_support::retry_server(2, sql, 0, false).await; let mut session = StagingSession::connect(DestEmitter::new(Arc::new(config), None)) .await .unwrap(); diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index e34c089d..737f8d85 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -615,6 +615,9 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M inserter_batches_in_total: emitter(|s| &s.inserter_batches_in), inserter_ch_seconds_total: emitter_seconds(|s| &s.inserter_ch_nanos), inserter_encode_seconds_total: emitter_seconds(|s| &s.inserter_encode_nanos), + resident_seal_seconds_total: emitter_seconds(|s| &s.resident_seal_nanos), + resident_raw_bytes_total: emitter(|s| &s.resident_raw_bytes), + resident_stored_bytes_total: emitter(|s| &s.resident_stored_bytes), oracle_resolve_seconds_total: emitter_seconds(|s| &s.oracle_resolve_nanos), process_cpu_seconds_total: proc_cpu, process_resident_memory_bytes: proc_rss, diff --git a/src/ch/test_support.rs b/src/ch/test_support.rs index e1563567..195a31a6 100644 --- a/src/ch/test_support.rs +++ b/src/ch/test_support.rs @@ -7,10 +7,11 @@ use tokio::task::JoinHandle; use crate::ch::CompressionChoice; use crate::emit::ch_emitter::{EmitterConfig, InstanceConfig, RetryConfig}; +/// Server expecting `sql` then `blocks` one-row Data blocks per attempt pub(crate) async fn retry_server( max_attempts: u32, sql: &str, - insert: bool, + blocks: usize, idle: bool, ) -> (EmitterConfig, JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -44,11 +45,13 @@ pub(crate) async fn retry_server( const EMPTY_DATA: &[u8] = &[2, 0, 1, 0, 2, 0xff, 0xff, 0xff, 0xff, 0, 0, 0]; expected.push(0); expected.extend_from_slice(EMPTY_DATA); - if insert { + for _ in 0..blocks { // One UInt8 column x without custom serialization, one row 42 expected.extend_from_slice( b"\x02\x00\x01\x00\x02\xff\xff\xff\xff\x00\x01\x01\x01x\x05UInt8\x00\x2a", ); + } + if blocks > 0 { expected.extend_from_slice(EMPTY_DATA); } let server = tokio::spawn(async move { diff --git a/src/emit/ch_ddl.rs b/src/emit/ch_ddl.rs index 728dc657..0b35acad 100644 --- a/src/emit/ch_ddl.rs +++ b/src/emit/ch_ddl.rs @@ -1774,7 +1774,7 @@ mod tests { let sql = "CREATE DATABASE IF NOT EXISTS retry_test"; for (retries, idle) in [(0, false), (1, false), (2, false), (1, true)] { let (config, server) = - crate::ch::test_support::retry_server(retries, sql, false, idle).await; + crate::ch::test_support::retry_server(retries, sql, 0, idle).await; let resolved = Arc::new(ResolvedConfig::default()); let ddl = DdlConfig::from_resolved( &resolved, diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index ddab5434..430f24dc 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -3258,6 +3258,12 @@ crate::atomic_stats! { /// Oracle-routed columns the daemon built itself: rendered cells /// against a `String` target, which PG would hand straight back pub oracle_local_columns, + /// Serialized Native block bytes before and after resident + /// compression; their ratio is what retained output saves + pub resident_raw_bytes, + pub resident_stored_bytes, + /// Resolver time serializing and compressing sealed blocks + pub resident_seal_nanos, } } diff --git a/src/emit/pipeline/batcher.rs b/src/emit/pipeline/batcher.rs index 93613805..e7df3e19 100644 --- a/src/emit/pipeline/batcher.rs +++ b/src/emit/pipeline/batcher.rs @@ -238,17 +238,28 @@ impl BatchMeta { } } -/// One independently-durable INSERT's worth of rows. `buffers` are owned -/// column slabs an inserter rebuilds a `BlockBuilder` over; `per_seq` tags -/// which xacts' rows it carries for ack accounting. +/// Rows per Native block; an INSERT carries several under its own limits +pub(crate) const NATIVE_BLOCK_ROWS: usize = 1 << 20; +/// Uncompressed column bytes per Native block +pub(crate) const NATIVE_BLOCK_BYTES: usize = 16 << 20; + +/// Owned column slabs of one Native block +pub(crate) struct BlockSlabs { + pub(crate) buffers: Vec, + pub(crate) n_rows: usize, +} + +/// One independently-durable INSERT's worth of rows in one or more blocks; +/// `per_seq` tags which xacts' rows it carries for ack accounting. pub(crate) struct InsertBatch { pub(crate) meta: Arc, - pub(crate) buffers: Vec, + pub(crate) blocks: Vec, pub(crate) n_rows: usize, pub(crate) per_seq: Vec<(u64, u64)>, /// Admission permit shares covering these rows, dropped post-insert-ack - _permits: Vec>, - _encoded: crate::budget::MemoryPermit, + pub(crate) permits: Vec>, + /// Output bytes held, shrunk once blocks compress + pub(crate) encoded: crate::budget::MemoryPermit, } /// Rows and `FlushAll` share one FIFO channel so a barrier's flush can never @@ -278,7 +289,12 @@ pub struct BatcherConfig { } struct Table { + /// Open block enc: TableEncoder, + /// Blocks sealed into the open INSERT + sealed: Vec, + sealed_rows: usize, + sealed_bytes: usize, allocated_bytes: usize, meta: Arc, seq_counts: Vec<(u64, u64)>, @@ -287,6 +303,28 @@ struct Table { deadline: Option, } +impl Table { + /// Rows in the open INSERT + fn rows(&self) -> usize { + self.sealed_rows + self.enc.rows + } + + /// Close the open block into the open INSERT + fn seal_block(&mut self) -> Result<(), String> { + if self.enc.rows == 0 { + return Ok(()); + } + let (buffers, n_rows) = self.enc.take_block().map_err(|e| e.to_string())?; + self.sealed_rows += n_rows; + self.sealed_bytes += buffers + .iter() + .map(ColumnBuf::allocated_bytes) + .sum::(); + self.sealed.push(BlockSlabs { buffers, n_rows }); + Ok(()) + } +} + #[derive(Default)] struct Tables { slots: HashMap, @@ -307,6 +345,9 @@ impl Tables { let (enc, meta) = table_encoder(ctx, e.key().clone(), t)?; self.open.push(Table { enc, + sealed: Vec::new(), + sealed_rows: 0, + sealed_bytes: 0, allocated_bytes: 0, meta, seq_counts: Vec::new(), @@ -514,7 +555,7 @@ async fn handle_row( .append_routed(&row.committed, &route.mapping, op, leaf) .map_err(|e| e.to_string())?; if append == Append::Full { - emit_batch(t, ctx.out, ctx.stats).await?; + t.seal_block()?; t.enc .append_routed(&row.committed, &route.mapping, op, leaf) .map_err(|e| e.to_string())?; @@ -556,7 +597,7 @@ async fn handle_segment( .map_err(|e| e.to_string())? == Append::Full { - emit_batch(t, ctx.out, ctx.stats).await?; + t.seal_block()?; t.enc .append_block(&mut chunk.buffers, rows) .map_err(|e| e.to_string())?; @@ -601,18 +642,21 @@ async fn appended( if t.deadline.is_none() { t.deadline = Some(Instant::now() + ctx.cfg.flush_timeout); } - let bytes = t - .enc - .buffers - .iter() - .map(ColumnBuf::allocated_bytes) - .sum::(); + if t.enc.rows >= NATIVE_BLOCK_ROWS || t.enc.approx_bytes >= NATIVE_BLOCK_BYTES { + t.seal_block()?; + } + let bytes = t.sealed_bytes + + t.enc + .buffers + .iter() + .map(ColumnBuf::allocated_bytes) + .sum::(); ctx.out .partial_bytes .fetch_add(bytes - t.allocated_bytes, Ordering::Relaxed); t.allocated_bytes = bytes; let batch_limit = ctx.cfg.byte_budget.min(ctx.out.batch_limit); - if t.enc.rows >= ctx.cfg.row_budget || bytes >= batch_limit { + if t.rows() >= ctx.cfg.row_budget || bytes >= batch_limit { emit_batch(t, ctx.out, ctx.stats).await?; } if ctx.out.partial_bytes.load(Ordering::Relaxed) @@ -632,8 +676,13 @@ async fn appended( async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> Result<(), String> { out.partial_bytes .fetch_sub(std::mem::take(&mut t.allocated_bytes), Ordering::Relaxed); - let (buffers, n_rows) = t.enc.take_block().map_err(|e| e.to_string())?; + t.seal_block()?; t.deadline = None; + let blocks = std::mem::take(&mut t.sealed); + let (n_rows, bytes) = ( + std::mem::take(&mut t.sealed_rows), + std::mem::take(&mut t.sealed_bytes), + ); if n_rows == 0 { t.seq_counts.clear(); t.slice_permits.clear(); @@ -643,16 +692,15 @@ async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> R let per_seq = std::mem::take(&mut t.seq_counts); let mut permits = std::mem::take(&mut t.slice_permits); permits.append(&mut t.value_permits); - let bytes = buffers.iter().map(ColumnBuf::allocated_bytes).sum(); // Separate pool avoids waiting on decoded payload held by this batch let encoded = out.budget.acquire(bytes).await; let batch = InsertBatch { meta: t.meta.clone(), - buffers, + blocks, n_rows, per_seq, - _permits: permits, - _encoded: encoded, + permits, + encoded, }; out.sender .send(batch) @@ -689,7 +737,7 @@ async fn flush_all( stats: &EmitterStats, ) -> Result<(), String> { for t in &mut tables.open { - if t.enc.rows > 0 { + if t.rows() > 0 { emit_batch(t, out, stats).await?; } } @@ -1340,7 +1388,7 @@ mod tests { tx.send(one(rel, route, r)).await.unwrap(); } let first = rx.recv().await.unwrap(); - assert!(first._encoded.bytes() >= 64 << 10); + assert!(first.encoded.bytes() >= 64 << 10); assert!( tokio::time::timeout(Duration::from_millis(20), rx.recv()) .await @@ -1429,7 +1477,8 @@ mod tests { assert_eq!(b.n_rows, 4); assert_eq!(b.per_seq, [(0, 2), (1, 1), (2, 1)]); wires.push( - b.buffers + b.blocks[0] + .buffers .iter() .map(|c| format!("{:?}", c.wire())) .collect::>(), @@ -1441,9 +1490,9 @@ mod tests { } /// Chunk that would push oracle cells past the seal threshold opens the - /// next batch whole + /// next block of the same INSERT #[tokio::test] - async fn oracle_seal_moves_chunk_to_next_batch() { + async fn oracle_seal_moves_chunk_to_next_block() { let (tx, rx, fatal, handle) = spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); let big = "x".repeat(crate::ops::oracle::ORACLE_BATCH_SEAL_BYTES * 3 / 5); @@ -1454,17 +1503,21 @@ mod tests { .unwrap(); } drop(tx); - let per_seq: Vec<_> = [rx.recv().await.unwrap(), rx.recv().await.unwrap()] - .map(|b| b.per_seq) - .into(); - assert_eq!(per_seq, [vec![(0, 1)], vec![(1, 1)]]); + let b = rx.recv().await.unwrap(); + assert!(rx.recv().await.is_err(), "one INSERT"); + assert_eq!(b.per_seq, [(0, 1), (1, 1)]); + assert_eq!( + b.blocks.iter().map(|b| b.n_rows).collect::>(), + [1, 1] + ); handle.await.unwrap(); assert!(!fatal.is_set()); } - /// Row that would push oracle cells past the seal threshold opens the next batch + /// Row that would push oracle cells past the seal threshold opens the + /// next block of the same INSERT #[tokio::test] - async fn oracle_seal_moves_row_to_next_batch() { + async fn oracle_seal_moves_row_to_next_block() { let (tx, rx, fatal, handle) = spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); let big = "x".repeat(crate::ops::oracle::ORACLE_BATCH_SEAL_BYTES * 3 / 5); @@ -1476,10 +1529,13 @@ mod tests { tx.send(one(rel, route, r)).await.unwrap(); } drop(tx); - let per_seq: Vec<_> = [rx.recv().await.unwrap(), rx.recv().await.unwrap()] - .map(|b| b.per_seq) - .into(); - assert_eq!(per_seq, [vec![(0, 1)], vec![(1, 1)]]); + let b = rx.recv().await.unwrap(); + assert!(rx.recv().await.is_err(), "one INSERT"); + assert_eq!(b.per_seq, [(0, 1), (1, 1)]); + assert_eq!( + b.blocks.iter().map(|b| b.n_rows).collect::>(), + [1, 1] + ); handle.await.unwrap(); assert!(!fatal.is_set()); } diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index 4fd7b10a..9ca276fb 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -12,74 +12,74 @@ //! dedups by `_lsn`). Retry-exhaustion is fatal: the watermark can't advance //! without this batch. -use clickhouse_c::{Allocator, BlockBuilder, ColumnBuilder, QueryOpts, TypeAst}; +use clickhouse_c::{Allocator, QueryOpts}; use tokio::task::JoinHandle; use crate::ch::{ChConn, EmitterError, EndOfStream, drain_to_end_of_stream, with_timeout}; use crate::config::DestEmitter; -use crate::emit::ch_emitter::{EmitterStats, build_leaf, build_root}; +use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::AckHandle; use crate::emit::pipeline::batcher::BatchMeta; use crate::emit::pipeline::resolver::ResolvedBatch; -use crate::schema::TableKey; -use foldhash::{HashMap, HashMapExt}; +use crate::emit::pipeline::sealed::{SealedBlock, builder_of}; use std::sync::Arc; use std::sync::atomic::Ordering; struct Inserter { client: ChConn, alloc: Allocator, - /// Parsed column types per table, refreshed when a batch's `schema_epoch` - /// changes. `TypeAst` is `Send` but not `Sync`, so each inserter parses - /// its own. - asts: HashMap)>, + /// One block's decompressed body, reused across sends; locked only + /// while a block opens + scratch: std::sync::Mutex>, ack: AckHandle, stats: Arc, } impl Inserter { - /// Owned out of the cache so no `&[TypeAst]` lives across the send - /// await; the caller puts them back - fn take_asts(&mut self, meta: &BatchMeta) -> Result<(u64, Vec), EmitterError> { - if let Some((epoch, asts)) = self.asts.remove(&meta.table_key) - && epoch == meta.schema_epoch - { - return Ok((epoch, asts)); - } - let mut parsed = Vec::with_capacity(meta.columns.len()); - for col in &meta.columns { - parsed.push(TypeAst::parse(&col.type_repr, self.alloc)?); - } - Ok((meta.schema_epoch, parsed)) - } - - /// Bounded reconnect+retry around one prepared INSERT. Only `bb` - /// (`Send + Sync`) and `self` cross the awaits, never a bare `&TypeAst` - /// (`!Sync`), so the task future stays `Send`. The block is unchanged - /// across retries, so a reconnect just resends. + /// Bounded reconnect+retry around one INSERT. Every attempt resends + /// every block in order, so a replay carries unchanged rows; blocks + /// open one at a time into `scratch` async fn send_with_retry( &mut self, sql: &str, - bb: &BlockBuilder<'_>, + blocks: &[SealedBlock], ) -> Result { let insert_timeout = self.client.config().insert_timeout; let started = std::time::Instant::now(); + let (alloc, stats) = (self.alloc, self.stats.clone()); + let scratch = &self.scratch; let result = self .client .retry( - |mut client| async move { - let result = with_timeout(insert_timeout, async { - client.send_query(sql, &QueryOpts::new()).await?; - client.send_data(bb).await?; - client.send_data_end().await?; - drain_to_end_of_stream(&mut client).await - }) - .await; - (client, result) + |mut client| { + let stats = &stats; + async move { + let result = with_timeout(insert_timeout, async { + client.send_query(sql, &QueryOpts::new()).await?; + for sealed in blocks { + let opened = std::time::Instant::now(); + let block = { + let mut scratch = + scratch.lock().expect("inserter scratch poisoned"); + sealed.open(alloc, &mut scratch)? + }; + let bb = builder_of(&block)?; + stats.inserter_encode_nanos.fetch_add( + opened.elapsed().as_nanos() as u64, + Ordering::Relaxed, + ); + client.send_data(&bb).await?; + } + client.send_data_end().await?; + drain_to_end_of_stream(&mut client).await + }) + .await; + (client, result) + } }, |_, _| { - self.stats.retries_attempted.fetch_add(1, Ordering::Relaxed); + stats.retries_attempted.fetch_add(1, Ordering::Relaxed); }, ) .await; @@ -99,82 +99,19 @@ impl Inserter { async fn send_all( &mut self, meta: &BatchMeta, - bb: &BlockBuilder<'_>, + blocks: &[SealedBlock], ) -> Result { let mut durable = None; for query in &meta.inserts { self.client.select(query.instance.as_deref())?; - durable = Some(self.send_with_retry(&query.sql, bb).await?); + durable = Some(self.send_with_retry(&query.sql, blocks).await?); } durable.ok_or_else(|| EmitterError::Config("batch has no INSERT target".into())) } async fn run(mut self, rx: async_channel::Receiver, fatal: Fatal) { - while let Ok(ResolvedBatch { batch, resolved }) = rx.recv().await { - let (epoch, asts) = match self.take_asts(&batch.meta) { - Ok(asts) => asts, - Err(e) => { - fatal.set(format!("inserter type parse: {e}")); - break; - } - }; - let result = 'send: { - let encode_started = std::time::Instant::now(); - let leaves: Vec>> = match batch - .buffers - .iter() - .map(|buf| build_leaf(buf, batch.n_rows)) - .collect::>() - { - Ok(v) => v, - Err(e) => break 'send Err(e), - }; - let roots: Vec>> = match batch - .buffers - .iter() - .zip(&leaves) - .map(|(buf, leaf)| build_root(buf, leaf.as_ref(), batch.n_rows)) - .collect::>() - { - Ok(v) => v, - Err(e) => break 'send Err(e), - }; - let mut bb = BlockBuilder::new(); - let appended: Result<(), EmitterError> = batch - .meta - .columns - .iter() - .enumerate() - .try_for_each(|(i, col)| match &roots[i] { - Some(node) => bb - .append(&col.name, asts[i].view(), node) - .map_err(Into::into), - None => { - let decoded = resolved - .as_ref() - .and_then(|r| r.column(i as u32)) - .ok_or_else(|| { - EmitterError::Type(format!( - "oracle answered no column for {}", - col.name - )) - })?; - bb.append_column(&col.name, asts[i].view(), decoded) - .map_err(Into::into) - } - }); - self.stats.inserter_encode_nanos.fetch_add( - encode_started.elapsed().as_nanos() as u64, - Ordering::Relaxed, - ); - match appended { - Ok(()) => self.send_all(&batch.meta, &bb).await, - Err(e) => Err(e), - } - }; - self.asts - .insert(batch.meta.table_key.clone(), (epoch, asts)); - match result { + while let Ok(batch) = rx.recv().await { + match self.send_all(&batch.meta, &batch.blocks).await { Ok(durable) => { self.stats .rows_emitted @@ -187,7 +124,9 @@ impl Inserter { col.cells_inserted .fetch_add(batch.n_rows as u64, Ordering::Relaxed); } - self.stats.blocks_sent.fetch_add(1, Ordering::Relaxed); + self.stats + .blocks_sent + .fetch_add(batch.blocks.len() as u64, Ordering::Relaxed); self.stats .inserter_batches_in .fetch_add(1, Ordering::Relaxed); @@ -216,7 +155,7 @@ pub(crate) async fn spawn_pool( let inserter = Inserter { client: ChConn::connect(dest.clone()).await?, alloc: Allocator::global(&mimalloc::MiMalloc), - asts: HashMap::new(), + scratch: Default::default(), ack: ack.clone(), stats: stats.clone(), }; @@ -230,61 +169,33 @@ pub(crate) async fn spawn_pool( #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::ColumnValue; + use crate::ch::CompressionChoice; use crate::emit::pipeline::ack; - use crate::emit::pipeline::batcher::InsertBatch; - use crate::emit::pipeline::resolver::tests::{GEOM_OID, seal, wkt_row}; - - async fn run_one(batch: InsertBatch) -> String { - let (ack, collector) = ack::spawn(Arc::default(), Fatal::new()); - let dest = DestEmitter::new(Arc::default(), None); - let inserter = Inserter { - client: ChConn::deferred(dest), - alloc: Allocator::global(&mimalloc::MiMalloc), - asts: HashMap::new(), - ack, - stats: Arc::default(), - }; - let (tx, rx) = async_channel::bounded(1); - let resolved = None; - tx.send(ResolvedBatch { batch, resolved }).await.unwrap(); - let fatal = Fatal::new(); - inserter.run(rx, fatal.clone()).await; - collector.await.unwrap(); - fatal.message().expect("inserter trips fatal") - } - - /// Batch shape faults fail before any connection dials - #[tokio::test] - async fn unanswered_oracle_column_and_bad_type_trip_fatal() { - let mut raw = wkt_row(0, None); - raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { - type_oid: GEOM_OID, - raw: vec![1], - }); - let msg = run_one(seal(vec![raw]).await).await; - assert!(msg.contains("oracle answered no column"), "{msg}"); + use clickhouse_c::{BlockBuilder, ColumnBuilder, TypeAst}; - let mut batch = seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await; - let mut columns = batch.meta.columns.clone(); - columns[0].type_repr = "Nope(((".into(); - batch.meta = Arc::new(BatchMeta { - table_key: batch.meta.table_key.clone(), - inserts: batch.meta.inserts.clone(), - columns, - schema_epoch: batch.meta.schema_epoch, - rows_inserted: Arc::default(), - }); - let msg = run_one(batch).await; - assert!(msg.starts_with("inserter type parse: "), "{msg}"); + /// One UInt8 column x holding one row 42 + fn one_row_block(codec: CompressionChoice) -> SealedBlock { + let ast = TypeAst::parse("UInt8", Allocator::global(&mimalloc::MiMalloc)).unwrap(); + let column = ColumnBuilder::fixed(&[42], 1, 1).unwrap(); + let mut bb = BlockBuilder::new(); + bb.append("x", ast.view(), &column).unwrap(); + SealedBlock::seal(&bb, 1, codec, &mut Vec::new()).unwrap() } + /// Every attempt resends every block of the INSERT, whatever codec held + /// them, through failed and idle reconnects #[tokio::test] async fn insert_retry_preserves_payload_through_failed_reconnect() { let sql = "INSERT INTO retry_test (x) VALUES"; - for (retries, idle) in [(0, false), (1, false), (2, false), (1, true)] { + for (retries, idle, blocks, codec) in [ + (0, false, 1, CompressionChoice::Lz4), + (1, false, 1, CompressionChoice::None), + (2, false, 1, CompressionChoice::Zstd), + (2, false, 3, CompressionChoice::Lz4), + (1, true, 2, CompressionChoice::Zstd), + ] { let (config, server) = - crate::ch::test_support::retry_server(retries, sql, true, idle).await; + crate::ch::test_support::retry_server(retries, sql, blocks, idle).await; let dest = DestEmitter::new(Arc::new(config), None); let client = ChConn::connect(dest).await.unwrap(); let (ack, collector) = ack::spawn(Arc::default(), Fatal::new()); @@ -292,16 +203,13 @@ mod tests { let mut inserter = Inserter { client, alloc: Allocator::global(&mimalloc::MiMalloc), - asts: HashMap::new(), + scratch: Default::default(), ack, stats: stats.clone(), }; - let ast = TypeAst::parse("UInt8", inserter.alloc).unwrap(); - let column = ColumnBuilder::fixed(&[42], 1, 1).unwrap(); - let mut block = BlockBuilder::new(); - block.append("x", ast.view(), &column).unwrap(); + let sealed: Vec<_> = (0..blocks).map(|_| one_row_block(codec)).collect(); assert_eq!( - inserter.send_with_retry(sql, &block).await.is_ok(), + inserter.send_with_retry(sql, &sealed).await.is_ok(), retries == 2 || idle ); assert_eq!( diff --git a/src/emit/pipeline/mod.rs b/src/emit/pipeline/mod.rs index 0d9f68d3..d4808383 100644 --- a/src/emit/pipeline/mod.rs +++ b/src/emit/pipeline/mod.rs @@ -19,6 +19,7 @@ pub mod planner; pub mod reorder; pub mod resolver; pub mod row_ledger; +pub(crate) mod sealed; pub mod tail; use std::sync::Arc; diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index 102e1f27..775e7a4a 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -13,24 +13,37 @@ //! A batch with no oracle column crosses unchanged, without touching the //! bridge. +use std::collections::hash_map::Entry; use std::sync::Arc; use std::sync::atomic::Ordering; use backon::Retryable; -use clickhouse_c::Allocator; +use clickhouse_c::{Allocator, BlockBuilder, ColumnBuilder, TypeAst}; use tokio::sync::watch; use tokio::task::JoinHandle; -use crate::ch::EmitterError; +use crate::ch::{CompressionChoice, EmitterError}; use crate::config::ResolvedConfig; -use crate::emit::ch_emitter::{ColumnBuf, EmitterStats, RetryConfig, literal_column}; +use crate::emit::ch_emitter::{ + ColumnBuf, EmitterStats, RetryConfig, build_leaf, build_root, literal_column, +}; use crate::emit::pipeline::Fatal; -use crate::emit::pipeline::batcher::InsertBatch; +use crate::emit::pipeline::batcher::{BatchMeta, BlockSlabs, InsertBatch}; +use crate::emit::pipeline::sealed::SealedBlock; use crate::ops::oracle::{Oracle, OracleBlock, OracleError, OracleRequestColumn}; +use crate::schema::TableKey; +use foldhash::HashMap; +/// Resolved INSERT: every block serialized and compressed, slabs and +/// oracle output released pub(crate) struct ResolvedBatch { - pub batch: InsertBatch, - pub resolved: Option, + pub meta: Arc, + pub blocks: Vec, + pub n_rows: usize, + pub per_seq: Vec<(u64, u64)>, + /// Admission permit shares covering these rows, dropped post-insert-ack + _permits: Vec>, + _encoded: crate::budget::MemoryPermit, } /// `None` oracle resolves nothing, which a pipeline with no oracle column @@ -41,6 +54,8 @@ pub(crate) struct ResolverOptions { pub oracle: Option>, /// Boot budget for oracle transport retries, re-read from `config_rx` pub retry: RetryConfig, + /// Codec compressing sealed blocks while they wait for acknowledgement + pub codec: CompressionChoice, pub stats: Arc, pub fatal: Fatal, pub config_rx: Option>>, @@ -59,6 +74,80 @@ pub(crate) fn spawn_pool( .collect() } +/// Per-task resolver state: parsed column types and serialization scratch +pub(crate) struct Sealer { + alloc: Allocator, + codec: CompressionChoice, + /// Parsed column types per table, refreshed when a batch's + /// `schema_epoch` changes. `TypeAst` is `Send` but not `Sync`, so each + /// task parses its own + asts: HashMap)>, + scratch: Vec, +} + +impl Sealer { + pub(crate) fn new(codec: CompressionChoice) -> Self { + Self { + alloc: Allocator::global(&mimalloc::MiMalloc), + codec, + asts: HashMap::default(), + scratch: Vec::new(), + } + } + + /// Serialize and compress one block over its slabs and oracle answer + pub(crate) fn seal( + &mut self, + meta: &BatchMeta, + block: &BlockSlabs, + resolved: Option<&OracleBlock>, + ) -> Result { + let asts = match self.asts.entry(meta.table_key.clone()) { + Entry::Occupied(e) if e.get().0 == meta.schema_epoch => e.into_mut(), + e => { + let parsed = meta + .columns + .iter() + .map(|c| TypeAst::parse(&c.type_repr, self.alloc)) + .collect::, _>>()?; + let fresh = (meta.schema_epoch, parsed); + match e { + Entry::Occupied(mut e) => { + e.insert(fresh); + e.into_mut() + } + Entry::Vacant(e) => e.insert(fresh), + } + } + }; + let n_rows = block.n_rows; + let leaves: Vec>> = block + .buffers + .iter() + .map(|buf| build_leaf(buf, n_rows)) + .collect::>()?; + let roots: Vec>> = block + .buffers + .iter() + .zip(&leaves) + .map(|(buf, leaf)| build_root(buf, leaf.as_ref(), n_rows)) + .collect::>()?; + let mut bb = BlockBuilder::new(); + for (i, col) in meta.columns.iter().enumerate() { + match &roots[i] { + Some(node) => bb.append(&col.name, asts.1[i].view(), node)?, + None => { + let decoded = resolved.and_then(|r| r.column(i as u32)).ok_or_else(|| { + EmitterError::Type(format!("oracle answered no column for {}", col.name)) + })?; + bb.append_column(&col.name, asts.1[i].view(), decoded)?; + } + } + } + SealedBlock::seal(&bb, n_rows, self.codec, &mut self.scratch) + } +} + async fn run( rx: async_channel::Receiver, tx: async_channel::Sender, @@ -67,40 +156,85 @@ async fn run( let ResolverOptions { oracle, mut retry, + codec, stats, fatal, config_rx, } = opts; let alloc = Allocator::global(&mimalloc::MiMalloc); - while let Ok(mut batch) = rx.recv().await { + let mut sealer = Sealer::new(codec); + while let Ok(batch) = rx.recv().await { if let Some(rx) = config_rx.as_ref() { retry.max_attempts = rx.borrow().retry_max_attempts; } - resolve_local_columns(&mut batch, &stats); - let started = std::time::Instant::now(); - let resolved = match resolve_oracle_with_retry(&oracle, alloc, &batch, &retry, &stats).await - { - Ok(v) => v, - Err(e) => { - // Seq stays unacknowledged, so a restart replays it - fatal.set(format!("oracle resolve: {e}")); - return; + let InsertBatch { + meta, + blocks, + n_rows, + per_seq, + permits, + mut encoded, + } = batch; + let mut sealed = Vec::with_capacity(blocks.len()); + for mut block in blocks { + resolve_local_columns(&mut block, &stats); + let started = std::time::Instant::now(); + let resolved = match resolve_oracle_with_retry( + &oracle, alloc, &meta, &block, &retry, &stats, + ) + .await + { + Ok(v) => v, + Err(e) => { + // Seq stays unacknowledged, so a restart replays it + fatal.set(format!("oracle resolve: {e}")); + return; + } + }; + stats + .oracle_resolve_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + let started = std::time::Instant::now(); + match sealer.seal(&meta, &block, resolved.as_ref()) { + Ok(b) => { + stats + .resident_raw_bytes + .fetch_add(b.raw_len() as u64, Ordering::Relaxed); + stats + .resident_stored_bytes + .fetch_add(b.allocated_bytes() as u64, Ordering::Relaxed); + sealed.push(b); + } + Err(e) => { + fatal.set(format!("seal block: {e}")); + return; + } } + stats + .resident_seal_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + } + // Slabs and oracle output already dropped + encoded.shrink(sealed.iter().map(|b| b.allocated_bytes() as u64).sum()); + let ready = ResolvedBatch { + meta, + blocks: sealed, + n_rows, + per_seq, + _permits: permits, + _encoded: encoded, }; - stats - .oracle_resolve_nanos - .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); - if tx.send(ResolvedBatch { batch, resolved }).await.is_err() { + if tx.send(ready).await.is_err() { return; } } } -fn resolve_local_columns(batch: &mut InsertBatch, stats: &EmitterStats) { +fn resolve_local_columns(block: &mut BlockSlabs, stats: &EmitterStats) { let mut n = 0; - for buf in &mut batch.buffers { + for buf in &mut block.buffers { if let ColumnBuf::Oracle(o) = buf - && let Some(local) = literal_column(o, batch.n_rows) + && let Some(local) = literal_column(o, block.n_rows) { *buf = local; n += 1; @@ -115,11 +249,12 @@ fn resolve_local_columns(batch: &mut InsertBatch, stats: &EmitterStats) { async fn resolve_oracle_with_retry( oracle: &Option>, alloc: Allocator, - batch: &InsertBatch, + meta: &BatchMeta, + block: &BlockSlabs, retry: &RetryConfig, stats: &EmitterStats, ) -> Result, EmitterError> { - (|| resolve_oracle(oracle, alloc, batch)) + (|| resolve_oracle(oracle, alloc, meta, block)) .retry(retry.backoff()) .when(OracleError::retryable) .notify(|_, _| { @@ -132,17 +267,18 @@ async fn resolve_oracle_with_retry( async fn resolve_oracle( oracle: &Option>, alloc: Allocator, - batch: &InsertBatch, + meta: &BatchMeta, + block: &BlockSlabs, ) -> Result, OracleError> { - let columns: Vec> = batch + let columns: Vec> = block .buffers .iter() .enumerate() .filter_map(|(i, buf)| match buf { ColumnBuf::Oracle(o) => Some(OracleRequestColumn { ordinal: i as u32, - name: &batch.meta.columns[i].name, - target_type: &batch.meta.columns[i].type_repr, + name: &meta.columns[i].name, + target_type: &meta.columns[i].type_repr, buf: o, }), _ => None, @@ -154,11 +290,11 @@ async fn resolve_oracle( let oracle = oracle.as_ref().ok_or_else(|| { OracleError::Absent(format!( "{} needs the shadow oracle, which this pipeline has none of", - batch.meta.table_key + meta.table_key )) })?; oracle - .encode_batch(batch.meta.table_key.db_oid, &columns, batch.n_rows, alloc) + .encode_batch(meta.table_key.db_oid, &columns, block.n_rows, alloc) .await .map(Some) } @@ -314,20 +450,20 @@ pub(crate) mod tests { ]) .await; assert!( - matches!(batch.buffers[0], ColumnBuf::Oracle(_)), + matches!(batch.blocks[0].buffers[0], ColumnBuf::Oracle(_)), "premise: an uncovered source type routes to the oracle", ); let alloc = Allocator::global(&mimalloc::MiMalloc); assert!(matches!( - resolve_oracle(&None, alloc, &batch).await, + resolve_oracle(&None, alloc, &batch.meta, &batch.blocks[0]).await, Err(OracleError::Absent(_)), )); let stats = EmitterStats::default(); - resolve_local_columns(&mut batch, &stats); + resolve_local_columns(&mut batch.blocks[0], &stats); assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 1); assert_eq!( - batch.buffers[0].wire(), + batch.blocks[0].buffers[0].wire(), Wire::NullableString { offsets: &[10, 10, 10, 20], data: b"POINT(1 2)POINT(3 4)", @@ -337,12 +473,12 @@ pub(crate) mod tests { ); assert!( - resolve_oracle(&None, alloc, &batch) + resolve_oracle(&None, alloc, &batch.meta, &batch.blocks[0]) .await .expect("no request to make") .is_none(), ); - resolve_local_columns(&mut batch, &stats); + resolve_local_columns(&mut batch.blocks[0], &stats); assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 1); } @@ -360,11 +496,17 @@ pub(crate) mod tests { let mut batch = seal(rows).await; let stats = EmitterStats::default(); - resolve_local_columns(&mut batch, &stats); - assert!(matches!(batch.buffers[0], ColumnBuf::Oracle(_))); + resolve_local_columns(&mut batch.blocks[0], &stats); + assert!(matches!(batch.blocks[0].buffers[0], ColumnBuf::Oracle(_))); assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 0); assert!(matches!( - resolve_oracle(&None, Allocator::global(&mimalloc::MiMalloc), &batch).await, + resolve_oracle( + &None, + Allocator::global(&mimalloc::MiMalloc), + &batch.meta, + &batch.blocks[0], + ) + .await, Err(OracleError::Absent(_)), )); } @@ -374,6 +516,7 @@ pub(crate) mod tests { let opts = ResolverOptions { oracle: None, retry: RetryConfig::default(), + codec: CompressionChoice::None, stats: Arc::default(), fatal: fatal.clone(), config_rx: None, @@ -402,4 +545,32 @@ pub(crate) mod tests { let fatal = run_one(seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await, false).await; assert!(fatal.message().is_none()); } + + /// Block shape faults fail sealing + #[tokio::test] + async fn unanswered_oracle_column_and_bad_type_fail_seal() { + let mut sealer = Sealer::new(CompressionChoice::None); + let mut raw = wkt_row(0, None); + raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { + type_oid: GEOM_OID, + raw: vec![1], + }); + let batch = seal(vec![raw]).await; + let Err(e) = sealer.seal(&batch.meta, &batch.blocks[0], None) else { + panic!("unanswered oracle column seals"); + }; + assert!(e.to_string().contains("oracle answered no column"), "{e}"); + + let batch = seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await; + let mut columns = batch.meta.columns.clone(); + columns[0].type_repr = "Nope(((".into(); + let meta = BatchMeta { + table_key: batch.meta.table_key.clone(), + inserts: batch.meta.inserts.clone(), + columns, + schema_epoch: batch.meta.schema_epoch, + rows_inserted: Arc::default(), + }; + assert!(sealer.seal(&meta, &batch.blocks[0], None).is_err()); + } } diff --git a/src/emit/pipeline/sealed.rs b/src/emit/pipeline/sealed.rs new file mode 100644 index 00000000..e30ff509 --- /dev/null +++ b/src/emit/pipeline/sealed.rs @@ -0,0 +1,325 @@ +//! Resolved Native blocks kept compressed from seal until INSERT +//! acknowledgement. +//! +//! clickhouse-c serializes each block once; the body is compressed with the +//! configured codec and column slabs drop. An inserter decodes one block at +//! a time back into a scratch buffer to send it, so resident output costs +//! compressed bytes plus one block per inserter. Incompressible bodies stay +//! raw rather than expand + +use std::pin::Pin; + +use clickhouse_c::{Allocator, Block, BlockBuilder, BlockOpts, BlockReader, Io, SliceIo, sys}; + +use crate::ch::{CompressionChoice, EmitterError}; + +/// Native file framing: blocks never leave this process in this form +const OPTS: BlockOpts = BlockOpts { + has_block_info: false, + has_custom_serialization: false, + read_buffer_bytes: 0, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Stored { + Raw, + Lz4, + Zstd, +} + +/// One serialized block body, owned until its INSERT is durable +pub(crate) struct SealedBlock { + stored: Stored, + payload: Box<[u8]>, + raw_len: usize, + n_rows: usize, +} + +impl SealedBlock { + /// Serialize `bb` and compress its body. `scratch` holds the raw body + /// between calls so sealing reuses one buffer + pub(crate) fn seal( + bb: &BlockBuilder<'_>, + n_rows: usize, + codec: CompressionChoice, + scratch: &mut Vec, + ) -> Result { + scratch.clear(); + let mut io = VecIo::new(std::mem::take(scratch)); + let written = bb.write(io.as_mut(), OPTS); + *scratch = io.as_mut().take(); + written?; + let raw_len = scratch.len(); + let compressed = match codec { + CompressionChoice::None => None, + CompressionChoice::Lz4 => lz4::block::compress(scratch, None, false) + .ok() + .map(|c| (Stored::Lz4, c)), + CompressionChoice::Zstd => zstd::bulk::compress(scratch, 0) + .ok() + .map(|c| (Stored::Zstd, c)), + }; + let (stored, payload) = match compressed { + Some((stored, c)) if c.len() < raw_len => (stored, c.into_boxed_slice()), + _ => (Stored::Raw, scratch.as_slice().into()), + }; + Ok(Self { + stored, + payload, + raw_len, + n_rows, + }) + } + + /// Serialized body bytes before compression + pub(crate) fn raw_len(&self) -> usize { + self.raw_len + } + + pub(crate) fn allocated_bytes(&self) -> usize { + self.payload.len() + } + + /// Decode the block, decompressing into `scratch`; the block owns its + /// columns, so `scratch` is free again once this returns + pub(crate) fn open( + &self, + alloc: Allocator, + scratch: &mut Vec, + ) -> Result { + let body: &[u8] = match self.stored { + Stored::Raw => &self.payload, + Stored::Lz4 => { + scratch.resize(self.raw_len, 0); + let n = lz4::block::decompress_to_buffer( + &self.payload, + Some(self.raw_len as i32), + scratch, + ) + .map_err(|e| EmitterError::Type(format!("sealed block lz4: {e}")))?; + &scratch[..n] + } + Stored::Zstd => { + scratch.resize(self.raw_len, 0); + let n = zstd::bulk::decompress_to_buffer(&self.payload, &mut scratch[..]) + .map_err(|e| EmitterError::Type(format!("sealed block zstd: {e}")))?; + &scratch[..n] + } + }; + if body.len() != self.raw_len { + return Err(EmitterError::Type(format!( + "sealed block body {} bytes, sealed {}", + body.len(), + self.raw_len + ))); + } + let mut io = SliceIo::new(body); + let mut reader = BlockReader::new(io.as_mut(), alloc, OPTS)?; + reader + .read()? + .filter(|b| b.n_rows() == self.n_rows) + .ok_or_else(|| EmitterError::Type("sealed block decoded short".into())) + } +} + +/// Rebuild a builder over a decoded block's columns, for `send_data` +pub(crate) fn builder_of(block: &Block) -> Result, EmitterError> { + let mut bb = BlockBuilder::new(); + for i in 0..block.n_columns() { + let name = block + .column_name(i) + .and_then(|n| std::str::from_utf8(n).ok()) + .ok_or_else(|| EmitterError::Type(format!("sealed block column {i} name")))?; + let (Some(ty), Some(col)) = (block.column_type(i), block.column(i)) else { + return Err(EmitterError::Type(format!("sealed block column {name}"))); + }; + bb.append_column(name, ty, col)?; + } + Ok(bb) +} + +/// Write-only transport appending to an owned buffer +struct VecIo { + io: sys::chc_io, + out: Vec, + _pin: std::marker::PhantomPinned, +} + +impl VecIo { + fn new(out: Vec) -> Pin> { + let mut boxed = Box::pin(Self { + io: sys::chc_io { + ud: std::ptr::null_mut(), + read: Some(vec_read), + write: Some(vec_write), + check_cancel: None, + }, + out, + _pin: std::marker::PhantomPinned, + }); + // SAFETY: context set once address is stable; value stays pinned + unsafe { + let this = boxed.as_mut().get_unchecked_mut(); + this.io.ud = (this as *mut Self).cast(); + } + boxed + } + + fn take(self: Pin<&mut Self>) -> Vec { + // SAFETY: buffer is not structurally pinned; callbacks reach it only + // through `ud` during a write call + std::mem::take(unsafe { &mut self.get_unchecked_mut().out }) + } +} + +// SAFETY: `io_ptr` points at the pinned callback table whose `ud` names +// this value; callbacks follow the chc_io contract +unsafe impl Io for VecIo { + fn io_ptr(self: Pin<&mut Self>) -> *mut sys::chc_io { + // SAFETY: table address stays fixed while pinned + unsafe { &mut self.get_unchecked_mut().io } + } +} + +unsafe extern "C" fn vec_write( + ud: *mut std::ffi::c_void, + buf: *const std::ffi::c_void, + len: usize, + _err: *mut sys::chc_err, +) -> std::ffi::c_int { + // SAFETY: `ud` points at the pinned VecIo owning this table; C passes + // `len` readable bytes + unsafe { + let this = &mut *ud.cast::(); + this.out + .extend_from_slice(std::slice::from_raw_parts(buf.cast::(), len)); + } + sys::CHC_OK +} + +unsafe extern "C" fn vec_read( + _ud: *mut std::ffi::c_void, + _buf: *mut std::ffi::c_void, + _len: usize, + _out_n: *mut usize, + err: *mut sys::chc_err, +) -> std::ffi::c_int { + let _ = err; + sys::CHC_ERR_USAGE +} + +#[cfg(test)] +mod tests { + use super::*; + use clickhouse_c::{ColumnBuilder, TypeAst}; + + fn alloc() -> Allocator { + Allocator::global(&mimalloc::MiMalloc) + } + + /// Block of a UInt64, a Nullable(String) and a UUID column + fn seal_rows(strings: &[Option<&[u8]>], codec: CompressionChoice) -> (SealedBlock, Vec) { + let n = strings.len(); + let ids: Vec = (0..n as u64).flat_map(u64::to_le_bytes).collect(); + let uuids: Vec = (0..16 * n).map(|i| (i * 7) as u8).collect(); + let (mut offsets, mut data, mut nulls) = (Vec::new(), Vec::new(), Vec::new()); + for s in strings { + data.extend_from_slice(s.unwrap_or_default()); + offsets.push(data.len() as u64); + nulls.push(u8::from(s.is_none())); + } + let types = + ["UInt64", "Nullable(String)", "UUID"].map(|t| TypeAst::parse(t, alloc()).unwrap()); + let id = ColumnBuilder::fixed(&ids, 8, n).unwrap(); + let leaf = ColumnBuilder::string(&offsets, &data, n).unwrap(); + let text = leaf.nullable(&nulls).unwrap(); + let uuid = ColumnBuilder::fixed(&uuids, 16, n).unwrap(); + let mut bb = BlockBuilder::new(); + bb.append("id", types[0].view(), &id).unwrap(); + bb.append("text", types[1].view(), &text).unwrap(); + bb.append("uuid", types[2].view(), &uuid).unwrap(); + let mut scratch = Vec::new(); + let sealed = SealedBlock::seal(&bb, n, codec, &mut scratch).unwrap(); + (sealed, scratch) + } + + fn check(strings: &[Option<&[u8]>], codec: CompressionChoice) -> SealedBlock { + let (sealed, raw) = seal_rows(strings, codec); + assert_eq!(sealed.raw_len(), raw.len()); + let mut scratch = Vec::new(); + let block = sealed.open(alloc(), &mut scratch).unwrap(); + assert_eq!(block.n_rows(), strings.len()); + let text = block.column(1).unwrap(); + let nulls = text.null_map().unwrap(); + let (offsets, data) = text.nullable_inner().unwrap().string().unwrap(); + let mut start = 0; + for (i, s) in strings.iter().enumerate() { + let end = offsets[i] as usize; + assert_eq!(nulls[i] == 1, s.is_none()); + assert_eq!(&data[start..end], s.unwrap_or_default()); + start = end; + } + let (width, ids) = block.column(0).unwrap().fixed().unwrap(); + assert_eq!(width, 8); + assert_eq!(ids.len(), 8 * strings.len()); + // Reserialized for the wire, the block keeps its body + let mut again = Vec::new(); + SealedBlock::seal( + &builder_of(&block).unwrap(), + strings.len(), + CompressionChoice::None, + &mut again, + ) + .unwrap(); + assert_eq!(again, raw); + sealed + } + + #[test] + fn blocks_round_trip_through_every_codec() { + let big = vec![b'x'; 1 << 20]; + let rows: Vec> = vec![Some(b"a"), None, Some(b""), Some(&big), None]; + for codec in [ + CompressionChoice::None, + CompressionChoice::Lz4, + CompressionChoice::Zstd, + ] { + let sealed = check(&rows, codec); + if codec == CompressionChoice::None { + assert_eq!(sealed.stored, Stored::Raw); + } else { + assert!( + sealed.allocated_bytes() * 10 < sealed.raw_len(), + "{codec:?}" + ); + } + } + } + + /// Bodies that do not shrink stay raw rather than expand + #[test] + fn incompressible_body_stays_raw() { + let mut x = 0x9E37_79B9_7F4A_7C15u64; + let noise: Vec = (0..64 << 10) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect(); + for codec in [CompressionChoice::Lz4, CompressionChoice::Zstd] { + let sealed = check(&[Some(&noise)], codec); + assert_eq!(sealed.stored, Stored::Raw); + assert_eq!(sealed.allocated_bytes(), sealed.raw_len()); + } + } + + #[test] + fn corrupt_body_fails_to_open() { + let (mut sealed, _) = seal_rows(&[Some(&b"abc"[..]); 100], CompressionChoice::Lz4); + assert_eq!(sealed.stored, Stored::Lz4); + sealed.raw_len += 1; + assert!(sealed.open(alloc(), &mut Vec::new()).is_err()); + } +} diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 50d4a881..85f8f003 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -309,6 +309,7 @@ pub async fn spawn_with_config( resolver::ResolverOptions { oracle, retry: emitter.retry.clone(), + codec: emitter.conn.compression, stats: stats.clone(), fatal: fatal.clone(), config_rx: config_rx.clone(), diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index bb42b4bb..c16e44b8 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -419,7 +419,13 @@ snapshot! { counter inserter_ch_seconds_total: f64 = "Cumulative inserter time inside the ClickHouse INSERT round trip. Against inserter_pool_size x uptime this is CH-side utilization.", counter inserter_encode_seconds_total: f64 = - "Cumulative inserter time rebuilding the Native block over a batch's owned slabs.", + "Cumulative inserter time decompressing and decoding sealed blocks to send them.", + counter resident_seal_seconds_total: f64 = + "Cumulative resolver time serializing and compressing sealed Native blocks.", + counter resident_raw_bytes_total: u64 = + "Serialized Native block bytes sealed for insertion, before resident compression.", + counter resident_stored_bytes_total: u64 = + "Bytes those sealed blocks held while awaiting insertion; raw over stored is the resident compression ratio.", counter oracle_resolve_seconds_total: f64 = "Cumulative resolver time inside the oracle round trip. Overlaps inserter_ch_seconds_total, so the larger of the two is the tail's limiter.", counter process_cpu_seconds_total: f64 = From fded642cca79c65400057e01b6cdd731b3779425 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:38:58 +0000 Subject: [PATCH 07/20] Describe direct encoding and trim its plan to open work Architecture now describes commits planned from WAL bytes into column chunks, blocks merged across commits and grouped per INSERT, and resolved blocks kept compressed until acknowledgement. The native encoding plan keeps only unfinished work: direct encoding for COPY and heap-page loads, detoast without cells, adopting prepared payload sends, memory bounds across owners, and end-to-end acceptance against the more-types baseline. Co-Authored-By: Claude Opus 5.5 --- architecture/README.md | 16 +- architecture/values.md | 8 +- bench/README.md | 26 +-- plans/INDEX.md | 2 +- plans/native_encoding.md | 353 ++++++++++----------------------------- 5 files changed, 114 insertions(+), 291 deletions(-) diff --git a/architecture/README.md b/architecture/README.md index b11a113d..9626bfad 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -40,9 +40,11 @@ planning committed transactions ![Commit pipeline: reorder places rows inline, one batcher feeds a conversion pool and ClickHouse inserters, and acknowledgements advance a contiguous watermark](workers.svg) `BufferingDecoderSink` and `ReorderSink` share one record-queue worker -`ReorderSink` plans each commit and places its rows onto the batcher in order -Batcher seals per-table batches. A separate conversion pool resolves PostgreSQL -types before inserters send complete Native blocks. Conversion concurrency +`ReorderSink` plans each commit, encoding its rows into per-table column +chunks, and places them onto the batcher in order. Batcher merges chunks of +many commits into Native blocks and groups blocks into per-table INSERTs. A +separate conversion pool resolves PostgreSQL types, then serializes each block +once and keeps it compressed until its INSERT is durable. Conversion concurrency matches oracle capacity; `[ch].inserter_pool_size` sizes the inserter pool Each inserter owns a ClickHouse connection and takes any ready batch @@ -51,9 +53,11 @@ Only a commit's final slice publishes its LSN, after all earlier work finishes Bounded queues and a shared payload budget limit work in flight; transaction and plan data can spill to disk -Queue worker runs on a dedicated thread and receives batches with heap decode -started ahead of dispatch. It checks decoded results against descriptor history -and decodes again if needed. Commits drain only their own transaction trees. +Queue worker runs on a dedicated thread. A dispatcher scans each batch ahead of +it to start TOAST store reads for values updated rows keep. Ordinary heap +records stay WAL bytes under their record-time descriptor until commit; +planning reads tuples in place and builds cells only for rows whose values must +be detoasted. Commits drain only their own transaction trees. Consecutive small commits can share row chunks while retaining separate acknowledgements diff --git a/architecture/values.md b/architecture/values.md index 3eef993c..5e17e7ba 100644 --- a/architecture/values.md +++ b/architecture/values.md @@ -41,9 +41,11 @@ columns for destination insertion. Batch conversion avoids a database round trip per row and keeps type semantics in PostgreSQL Conversion workers run between batcher and inserters, so PostgreSQL conversion -overlaps ClickHouse insertion. Batches carrying only local columns pass through -without a PostgreSQL request. Inserters combine local and converted columns into -complete Native blocks and retain them across insert retries +overlaps ClickHouse insertion. Blocks carrying only local columns pass through +without a PostgreSQL request. Conversion workers combine local and converted +columns into complete Native blocks, serialize and compress each once, and +release column buffers. Inserters decode one block at a time to send it and +resend every block of an INSERT across retries Conversion failure stops batch with column and row context. Module pins output settings to make conversion reproducible. Greenfield bootstrap uses a temporary diff --git a/bench/README.md b/bench/README.md index b1890474..dfe582c5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -34,21 +34,23 @@ cargo bench --bench spool_read -- --plan-only --planner --mapped --rows 1000 --p cargo bench --bench spool_read -- --plan-only --planner --mapped --rows 200000 --payload-bytes 256 --relations 8 --control-every 10000 ``` -Use `--planner` to run `Planner::plan_batch` before replay and place replayed rows -in batches. Use `--mapped` to assign frozen routes. By default, heaps have no -mapping: planning counts and discards them without writing them to spool. +Use `--planner` to run `Planner::plan_batch` over drained slices built before +timing starts. Rows enter as WAL insert records and encode from tuple views; +add `--cells` to plan decoded rows through the cell adapter instead. Use +`--mapped` to assign frozen routes over an int4 and a text column. By default, +heaps have no mapping: planning counts and discards them without encoding. Use `--relations` to cycle through relation descriptors, one per row. Use `--control-every` to insert unrelated schema controls and clear cached routes. Use `--plan-only` to skip deferred spool measurements. Plans spill to disk when -they exceed default 1 MiB memory limit. - -Read `plan_total` for combined write and replay time per sample, including row -placement with `--planner`. For plans in memory, byte counts estimate memory use; -for spilled plans, they report encoded file size. Route caching measures descriptor -identity checks and reuse of cached references. Measurements exclude mapping -resolution, WAL draining, TOAST I/O, control execution, and PostgreSQL/ClickHouse I/O. -`plan` reports allocations and bytes retained by sealed plan; `replay` reports -peak bytes while replaying it. Replay compares every row's values exactly +encoded chunks exceed default 1 MiB memory limit. + +Read `plan_total` for combined write and replay time per sample. For plans in +memory, byte counts report allocated chunk bytes; for spilled plans, encoded +file size. Route caching measures descriptor identity checks and reuse of cached +references. Measurements exclude mapping resolution, WAL draining, TOAST I/O, +control execution, and PostgreSQL/ClickHouse I/O. `plan` reports allocations and +bytes retained by sealed plan; `replay` reports peak bytes while replaying it. +With `--mapped`, replayed rows must match a reference plan encoded from cells ## pgbench workload diff --git a/plans/INDEX.md b/plans/INDEX.md index 2b8a2852..68b7b666 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -39,7 +39,7 @@ writes. A documented limitation does not imply code rejects it | [Multi-database loads and metrics](multi_database.md) | Extend heap-page bootstrap and backup loads beyond primary database, attribute metrics | | [Fuzzing](fuzzing.md) | Find parser and schema-transition interactions beyond fixed regressions | | [Performance](performance.md) | Establish healthy sustained WAL baseline with exact reconciliation before tuning | -| [Direct Native encoding](native_encoding.md) | Build on `more-types`: retain compact PG inputs, transcode into column chunks, compress sealed output | +| [Direct Native encoding](native_encoding.md) | Encode loads and TOAST rows without cells, send prepared payloads, prove resource gains end to end | | [Failover](failover.md) | Fence unplanned promotion and prove remaining archive/restart fault cases | | [Shadow TOAST reclamation](shadow_toast.md) | Keep historical values readable under lag and restart | | [Replay callback](custom_rmgr.md) | Reduce measured command-boundary capture stalls | diff --git a/plans/native_encoding.md b/plans/native_encoding.md index e34f954f..6bbef8a2 100644 --- a/plans/native_encoding.md +++ b/plans/native_encoding.md @@ -1,279 +1,94 @@ # Encode PostgreSQL values directly into ClickHouse blocks -Build on `more-types` at `69f1700` (`Decode more PostgreSQL fixed-width types -locally`). Replace long-lived decoded row graphs with compact PG inputs and -direct column encoding. Retain transaction coordination, then compress sealed -output blocks while queued or retained for retry. - -Keep three changes independently reviewable: transaction storage, transcoding, -and compressed output ownership. Measure each against `more-types` before -combining results. Follow [shared constraints](coordination.md), -[value coercion](value_coercion.md), and [performance acceptance](performance.md). - -## Preserve more types behavior - -Reuse branch codecs, target mappings, defaults, and type coverage: - -| Source types | Existing branch behavior to preserve | -|---|---| -| `oid`, `xid`, `cid` | `UInt32` | -| `oid8`, `xid8`, `pg_lsn` | `UInt64`; retain `oid8` availability on PG 19+ | -| `money` | `Int64` containing source minor units; do not introduce decimal rescaling | -| `name` | Trim NUL padding and preserve existing text behavior | -| `macaddr`, `macaddr8`, `tid` | Render existing PostgreSQL-compatible text | - -Keep source type identity beside storage class. An eight-byte value alone -cannot distinguish timestamp epoch conversion, unsigned integers, and money. -Share rendering helpers from branch's `src/decode/codecs/fixed.rs` rather than -copying their logic into new encoders. Preserve COPY coverage without assuming -every newly local heap type also uses binary COPY. - -## Replace representation and retain coordination - -Current [heap decoder](../src/decode/heap_decoder.rs) constructs owned -`ColumnValue` cells. [Transaction buffer](../src/xact/xact_buffer.rs) retains -decoded heaps and TOAST entries until commit or abort. -[Planner](../src/emit/pipeline/planner.rs) resolves TOAST and routing, then -[plan spool](../src/emit/pipeline/plan_spool.rs) retains or serializes decoded -heaps. [Batcher](../src/emit/pipeline/batcher.rs) traverses rows again to fill -`ColumnBuf` slabs. [Resolver](../src/emit/pipeline/resolver.rs) converts remaining -oracle columns before [inserter](../src/emit/pipeline/inserter.rs) builds and -sends Native blocks. Sealed batches retain uncompressed slabs. - -Use this proposed flow: - -```text -WAL -> compact PG inputs owned by xid -> commit merge - -> resolve descriptor, route, TOAST, old and new key - -> append directly to column chunks in private transaction plan - -> validate and seal plan - -> coalesce compatible committed chunks across transactions - -> resolve oracle columns -> serialize and compress Native blocks - -> retain compressed blocks through INSERT acknowledgement -``` - -Encode after commit initially so subtransaction rollback never requires editing -compressed columns, and descriptor/TOAST resolution precedes final encoding. -Compression here reduces retained output, not open transaction inputs. Evaluate -compression of raw transaction segments separately if those inputs dominate RSS; -keep spill available regardless of compression ratio. Defer Native encoding -before commit until measurements justify its unresolved-cell and rollback state. - -Keep `XactBuffer` responsible for aborts, subtransactions, WAL ordering, spill, -and resume floor. Replace its heap payload, not its correctness responsibilities. -Keep `ColumnValue` for runtime configuration, SQL defaults, diagnostics, and -temporary adapters where useful. Remove it from ordinary retained row payloads; -deleting enum globally is not a completion requirement. - -## Store compact inputs before commit - -Introduce an owned heap input containing writer xid, source LSN, physical -identity, descriptor span key when covered, and bytes needed to reconstruct -logical old/new images. Preserve absent versus SQL NULL, dropped attributes, -missing defaults, operation flags, and partial-image diagnostics. - -Start from existing `RawRecord` ownership and reconstruction in -[spill codec](../src/xact/spill.rs). Reuse `drop_redundant_images`; retain images -when decoding actually requires them. Do not retain a complete WAL segment or -backup page through a small borrowed tuple slice. Compact further only after -measuring retained bytes against decoded rows, including full-page images and -multi-insert records. - -Give covered raw heaps a distinct representation from unresolved catalog stash -records. Existing `SpillEntry::Raw` carries commit-time stash classification; -routing every ordinary heap through that arm would change discard and rejection -semantics. Pin covered descriptor versions exactly as `DescribedHeap` does; -resolve uncovered records through existing fail-closed stash verdicts. - -Keep TOAST chunk and delete handling intact during initial migration. Preserve -chunk generation fences, store references, same-transaction births, and mirror -row ordering. Decode small runtime configuration rows through existing adapter -until a separate reader is useful. - -Replace speculative owned `Predecoded` row graphs in -[queue](../src/source/queueing_record_sink.rs) and `ReorderSink::predecoder` -with bounded framing or borrowed scans. Preserve TOAST read-ahead for unchanged -external values without retaining decoded copies. Measure lost decode overlap -and read-ahead effectiveness before removing existing path. - -## Transcode during transaction planning - -Split tuple walking from owned cell construction. Expose short-lived views over -fixed bytes, varlena bodies, SQL NULL, missing attributes, and external pointers. -Borrow from current input or bounded detoast scratch; never let views outlive -their owners. Retain a cell-producing adapter for remaining callers. - -Compile conversion decisions once per descriptor version and frozen route, -using source type and destination type together. Keep mapping changes and -schema epochs in cache identity. Reuse existing codecs for epoch shifts, UUID -layout, numeric rendering, JSONB, interval, inet, and branch's added types. -Append formatted output directly where helpers allow it; accept bounded scratch -when a codec needs materialization. - -Coordinate [local container codecs](tier2.md) around recursive column builders. -Do not require a retained nested `ColumnValue` graph when array or map support -can append directly; preserve oracle fallback for uncovered source/target pairs. - -| Input | Direct encoding rule | -|---|---| -| Compatible fixed heap datum | Copy validated little-endian bytes into matching column storage | -| Binary COPY field | Decode big-endian `typsend` representation; never reuse heap byte-copy rule | -| Inline text or bytea | Strip storage framing and append body after existing validation | -| Compressed or external varlena | Resolve and decompress before encoding; bound scratch and preserve size policy | -| Missing default | Apply shared default conversion with source descriptor and target policy | -| Oracle type or configured composite target | Retain compact raw/text conversion input; preserve oracle fallback | - -Do not treat equal width as type compatibility. Preserve configured substitutes -and existing errors; compose outstanding coercion fixes without introducing a -second policy implementation. Keep absence defaults for tombstones distinct -from real source NULL. Preserve NaN payloads and signed zero where existing -fixed-width mappings permit bit-preserving output. - -Inspect old and new replica identity before releasing either view. Emit old-key -tombstones for key changes, choose correct image for DELETE, and keep configured -append-only delete policy. Preserve source `_lsn` and writer `_xid`; attach commit -timestamp after commit is known. Keep load-specific synthetic versions unchanged. - -Store encoded chunks and oracle inputs in private transaction plan. Seal chunks -at row/byte limits, schema or route changes, and ordered control entries. Preserve -TOAST mirror row positions and truncate fences. Keep all existing planning -rejections before transaction effects; do not send partially planned chunks. -Oracle transport and conversion remain explicit later failure points. - -Replace heap frames in plan spool with versioned encoded frames once adapter -parity passes. Retain plan memory/disk caps and bounded replay. Reuse chunk -metadata across rows instead of copying descriptor and routing ownership per -cell. Treat plans and transaction spills as scratch recovered from source WAL, -not as a new durable destination queue. - -## Preserve batching across transactions - -Accept column chunks in batcher instead of `RoutedRow` graphs. Coalesce adjacent -compatible small chunks by appending fixed bytes and rebasing string offsets, -null maps, and oracle cell ranges. Preserve `(seq, row_count)` contributions. -Keep compatibility keyed by destination, schema epoch, route, and encoding plan. - -Separate Native block size from INSERT size. Seal bounded blocks, then group -compatible blocks from multiple transactions into one INSERT under existing row, -byte, and deadline triggers. Do not compress tiny per-transaction blocks and -assume concatenating their byte streams produces one merged Native block. -Send several Data blocks under one query when grouping sealed blocks. - -Preserve FIFO `FlushAll`, destructive-operation drains, and configuration -barriers. Never move chunks across DDL or TRUNCATE to improve compression. -Measure actual destination parts: several blocks in one INSERT do not establish -unchanged part creation by themselves. - -## Compress sealed output blocks - -Resolve oracle columns before final serialization. Serialize final Native block -body and compress once; release column slabs and oracle output after compressed -owner is admitted. Keep schema/route metadata and acknowledgement counts beside -payload. Retain original logical row/byte counts for batching independently of -compressed resident size. Start with existing configured codecs and measure -incompressible expansion as well as useful compression. - -Add a narrow prepare/send interface in `clickhouse-c` and `clickhouse-c-rs`. -Current client accepts `BlockBuilder` through `send_data`; it does not expose an -owned prepared payload send API. Keep serialization, compression framing, packet -state, and transport handling in dependency rather than duplicating protocol in -walshadow. Read existing [client](../pgext/pg-clickhouse-c/clickhouse-c/doc/clickhouse-client.md) -and [compression](../pgext/pg-clickhouse-c/clickhouse-c/doc/clickhouse-compression.md) -contracts before choosing interface. - -Prepare immutable block payload with explicit codec and protocol compatibility -requirements; generate connection-specific packet framing at send. Validate -negotiated revision and serialization settings on reconnect and across named -destinations. For incompatible connections, reject reuse or rebuild through a -bounded decode/re-encode path. Do not replay captured Query packets, session -state, or unchecked raw socket bytes. - -Stream prepared payload through bounded transport buffers. Account for any C -output-buffer copy and retained capacity; adding compressed owner while keeping -slabs or an unbounded transport copy defeats resident-memory goal. If prepared -send cannot land first, allow storage compression with one bounded decompression -buffer per inserter as an intermediate step, and measure its CPU cost separately. - -Retain every block in an INSERT until `EndOfStream` from every required -destination. Retry entire INSERT after ambiguous failure using unchanged rows, -versions, block order, and metadata. Acknowledge its combined sequence counts -only after success. Keep counts logical, independent of compression frames. -Compression, cancellation, and transport errors must neither drop ownership nor -advance contiguous acknowledgement. - -## Bound resident memory and preserve progress - -Track allocated capacity for raw inputs, open column builders, oracle inputs and -responses, sealed compressed blocks, compression scratch, and transport buffers. -Keep decoded/encoded sizes distinct from compressed allocation and process RSS. -Bound queue bytes as well as item counts; include spare slab capacity and -per-chunk metadata when measuring small rows and many tables. - -Preserve separate transaction spill and output admission mechanisms. If retaining -an open transaction would exhaust resident allowance, spill and continue reading -WAL; never wait for its own COMMIT while blocking ingestion. Reserve bounded -conversion/compression workspace separately from queued output so memory held by -unfinished work cannot prevent its completion. Specify acquisition and release -order before changing pools, including incompressible output and oversized rows. - -Keep startup scratch cleanup and source-WAL recovery. Version changed spill and -plan formats; never interpret old frames under new layout. Retain first-record -resume floor until corresponding transaction output is durable. Test restart -from recorded source positions rather than treating a compressed buffer as proof -of persistence. - -## Implementation sequence - -1. Record baseline on `more-types`. Extend existing `xact_spill`, `spool_read`, - and `bootstrap_pump` benchmarks with retained allocation measurements and - exact result comparison. Establish source-to-destination baseline using - existing benchmark driver. -2. Add borrowed tuple views and direct column append behind internal adapters. - Keep production ownership unchanged while comparing against existing encoder. - Cover branch's added types and configured target overrides first. -3. Replace ordinary transaction heap storage with owned PG inputs. Reuse spill, - keep covered descriptors separate from unresolved stash, and remove retained - predecoded copies. Preserve temporary cell adapter at commit during migration. -4. Fuse commit planning and encoding. Store column chunks in private plan, - replace typed heap frames, and accept chunks in batcher. Delete superseded - row materialization paths after parity; avoid a permanent second pipeline. -5. Introduce prepared payload support in dependencies, then compressed block - ownership and multi-block INSERTs. Validate retries and memory handoff before - enabling resident compression by default. -6. Apply shared direct append to COPY and heap-page/bootstrap loads. Keep - visibility, pending rows, deferred TOAST, replacement evidence, and staging - publication unchanged. Stop retaining enum rows in ordinary load queues. -7. Reconcile end-to-end results and publish per-stage resource comparisons. - Update settled architecture and operator documentation; remove completed - proposal sections and retain unresolved acceptance work. +Streaming commits plan rows straight from WAL bytes into column chunks, and +sealed Native blocks stay compressed until acknowledgement; see +[architecture](../architecture/README.md#commit-pipeline). Remaining work +removes owned cells from load paths and TOAST rows, sends prepared payloads, +and proves resource gains end to end. Follow [shared constraints](coordination.md), +[value coercion](value_coercion.md), and [performance acceptance](performance.md) + +## Encode loads directly + +COPY and heap-page/bootstrap loads still decode every tuple into `ColumnValue` +cells: [page walk](../src/backfill/backup_page_walk.rs) and +[COPY](../src/backfill/copy_backfill.rs) fill `BackfillTuple` slabs, and +[bootstrap drain](../src/emit/pipeline/bootstrap.rs) and +[gap replay](../src/backfill/wal_replay.rs) stage cell rows through +`RowBuffer`, the batcher's second input next to planned segments + +Carry tuple bytes or COPY fields through load queues instead of cells. Read +visibility from tuple headers, which needs no columns. Encode visible rows into +chunks with the shared view encoder; decode cells only where pending rows, +deferred TOAST, and replacement evidence need them, keeping those mechanisms +and staging publication unchanged. Binary COPY fields are big-endian `typsend` +representations: add a field-view decoder beside the heap one, never reuse the +heap byte-copy rule. Then ship load rows as planned segments and delete +`RowChunk`/`RoutedRow` from the batcher + +Version the deferred TOAST spool if its tuple format changes. Rewrite bootstrap +tests that inspect routed cells to read chunk columns + +## Detoast without cells + +Inputs whose images hold external pointers, stash-resolved records, and +directly buffered heaps plan through the owned-cell adapter, since +`detoast_heap` resolves cells. Resolve pointers from tuple views into bounded +scratch, then encode with the view path, preserving size policy, fill modes, +read-ahead transfer, and chunk generation fences. Compressed inline values and +`numeric`/`jsonb` already decode through per-row scratch + +## Send prepared payloads + +clickhouse-c `chc_block_prepare` / `chc_client_send_prepared`, bound in +clickhouse-c-rs as `BlockBuilder::prepare` and `send_prepared`, write prepared +Data bytes on connections negotiating the same compression and revision. The +clickhouse-c-rs walshadow depends on predates them, so +[sealed blocks](../src/emit/pipeline/sealed.rs) store a Native body compressed +by walshadow and each inserter decodes it back into a block to send, +recompressing on the wire. With prepared sends, seal through +`BlockBuilder::prepare` with the destination's negotiated codec and send +without decoding; rebuild through the decode path only for a connection that +rejects the block, such as a named destination configured differently. Count +rebuilds separately + +## Bound memory across owners + +Track allocated capacity, not lengths, for raw inputs, open column builders, +oracle cells and responses, sealed blocks, compression scratch, and transport +buffers, and keep them distinct from process RSS. Bound queue bytes as well as +item counts, including spare slab capacity and per-chunk metadata for small +rows across many tables. Oracle cells own one allocation each; pack them into +shared buffers if they dominate + +Reserve conversion and compression workspace separately from queued output so +unfinished work cannot block its own completion. Specify acquisition and release +order before changing pools, including incompressible output and oversized rows + +Evaluate compressing raw transaction segments if open inputs dominate RSS; keep +spill available regardless of ratio. Defer encoding before commit until +measurements justify its unresolved-cell and rollback state ## Acceptance -Extend existing behavioral suites rather than creating a parallel harness: - | Concern | Required evidence | |---|---| -| Type fidelity | Compare old/direct output across `types_sweep`, `emitter_native_types`, oracle and bootstrap type suites; include branch types, defaults, NULL, empty values, signed boundaries, float bits, timestamp epochs, UUIDs, numeric, and JSONB | -| Source representation | Compare WAL/heap and binary COPY outputs; exercise inline, compressed, external, and unchanged TOAST values | -| Transaction outcomes | Extend `xact_buffer`, `subxact`, and bootstrap crossing-transaction cases with interleaved parent/subxid rows, savepoint rollback, abort after spill, and late subxid assignment | -| Ordering | Extend grouping/spill, schema evolution, parallel DDL, runtime config, and truncate tests with chunk boundaries on both sides of controls | -| Planning failures | Force missing descriptor, malformed input, missing TOAST, and spool-cap failure after earlier chunks have sealed; observe no transaction effects from rejected plan | -| Oracle coexistence | Mix direct and oracle columns, oracle retries, local literal resolution, and conversion errors without rebuilding retained row graphs | -| Compressed output | Decode prepared blocks through existing Native reader; verify none/LZ4/ZSTD, incompressible data, large values, nullable strings, frame boundaries, and multiple blocks per INSERT | -| Retry and cancellation | Disconnect after an intermediate Data block and after server acceptance before reply; cancel compression, queue handoff, and send; verify replay, ownership cleanup, and pinned acknowledgement | +| Load coverage | Compare cell and direct output across COPY, direct bootstrap, base backup, object store, pending transactions, deferred TOAST, and restart using existing load fixtures | +| TOAST | Compare cell and view detoast across inline, compressed, external, unchanged, and same-transaction values under every value mode | +| Prepared send | Decode prepared blocks through existing Native reader; verify none/LZ4/ZSTD, mismatched destinations, and retries after an intermediate Data block and after server acceptance before reply | | Recovery | Restart with open raw transactions, spilled plans, and ambiguous INSERT outcomes; verify exact destination state and resume floor across required destinations | -| Memory progress | Exhaust transaction/output budgets with many tables, a large transaction, and stalled inserters; prove spill reaches COMMIT and compression can finish without circular waits | -| Load coverage | Exercise COPY, direct bootstrap, base backup, object store, pending transactions, deferred TOAST, and restart using existing load fixtures | +| Memory progress | Exhaust transaction/output budgets with many tables, a large transaction, and stalled inserters; prove spill reaches COMMIT and compression finishes without circular waits | Measure narrow fixed rows, short strings, wide JSONB/text, TOAST-heavy updates, -mixed oracle tables, many small transactions, and a spilling large transaction. -Run with unchanged worker counts, batching limits, and codec settings. Record -rows/s, source WAL bytes/s, CPU, allocation count, peak/steady RSS, retained bytes -by owner, spill volume, compression ratio, durable lag, blocks/INSERT, and actual -ClickHouse part creation. Compare compressible and incompressible payloads with -both healthy and stalled destination. - -Require exact reconciliation and no new progress stalls before accepting resource -gains. Attribute savings separately to removal of enum rows, compact transaction -storage, and compression. Choose block size and default compression from measured -CPU/memory/part tradeoffs; do not claim gains from wire compression alone. +mixed oracle tables, many small transactions, and a spilling large transaction +against `more-types` (`69f1700`) using `xact_spill`, `spool_read`, and +`bootstrap_pump` allocation reports, and source-to-destination with the +[benchmark driver](../bench/README.md). Run with unchanged worker counts, +batching limits, and codec settings. Record rows/s, source WAL bytes/s, CPU, +allocation count, peak/steady RSS, retained bytes by owner, spill volume, +`walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total`, +durable lag, blocks per INSERT, and actual ClickHouse part creation, with +compressible and incompressible payloads and both healthy and stalled +destinations. Attribute savings separately to compact transaction storage, +direct encoding, and compression. Choose block size and default compression +from measured CPU/memory/part tradeoffs; do not claim gains from wire +compression alone From 454e2121eb7da78e3dadbe26ffef0343cb81b584 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:27:52 +0000 Subject: [PATCH 08/20] avoid needlessly replanning --- src/emit/pipeline/plan_spool.rs | 113 +++++++++++++++++++++++++++++--- src/emit/pipeline/planner.rs | 15 ++++- src/emit/pipeline/reorder.rs | 13 +++- 3 files changed, 129 insertions(+), 12 deletions(-) diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index 905c877c..6e17426d 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -169,6 +169,43 @@ struct OpenTable { enc: TableEncoder, } +/// Empty encoders kept across plans, so a route compiles its table plan +/// once rather than per transaction. One per relation name: a new +/// descriptor or route version replaces the old +#[derive(Default)] +pub struct EncoderCache { + entries: Vec, +} + +struct CachedEncoder { + rel: Arc, + route: Arc, + enc: TableEncoder, +} + +impl EncoderCache { + fn take( + &mut self, + rel: &Arc, + route: &Arc, + ) -> Option { + let i = self + .entries + .iter() + .position(|e| Arc::ptr_eq(&e.rel, rel) && Arc::ptr_eq(&e.route, route))?; + Some(self.entries.swap_remove(i).enc) + } + + fn put(&mut self, rel: Arc, route: Arc, enc: TableEncoder) { + self.entries.retain(|e| e.rel.rel_name != rel.rel_name); + self.entries.push(CachedEncoder { rel, route, enc }); + } + + pub fn clear(&mut self) { + self.entries.clear(); + } +} + /// Streams one transaction's validated rows into column chunks, in memory /// or on disk, while accumulating the bounded plan header. Dropped unsealed /// (any planning error), the file unlinks and nothing replays @@ -190,6 +227,7 @@ pub struct PlanWriter { /// second live table appears, so single-relation plans never hash table_ids: Option>, open: Vec, + encoders: EncoderCache, /// Chunks of the open segment sealed early at an oracle frame limit sealed: Vec<(u32, EncodedChunk)>, commit_ts: i64, @@ -219,6 +257,7 @@ impl PlanWriter { last_table: 0, table_ids: None, open: Vec::new(), + encoders: EncoderCache::default(), sealed: Vec::new(), commit_ts: 0, open_rows: 0, @@ -245,6 +284,12 @@ impl PlanWriter { self } + /// Reuse encoders an earlier plan returned through [`Self::seal_keeping`] + pub fn with_encoders(mut self, encoders: EncoderCache) -> Self { + self.encoders = encoders; + self + } + /// Disk bytes written so far, header included pub fn bytes(&self) -> u64 { self.bytes @@ -381,11 +426,13 @@ impl PlanWriter { None => { let t = &self.tables[table as usize]; let route = t.route.as_ref().expect("routed table"); - let plan = TablePlan::for_route(self.alloc, &t.descriptor, route)?; - self.open.push(OpenTable { - table, - enc: TableEncoder::new(plan)?, - }); + let enc = match self.encoders.take(&t.descriptor, route) { + Some(enc) => enc, + None => { + TableEncoder::new(TablePlan::for_route(self.alloc, &t.descriptor, route)?)? + } + }; + self.open.push(OpenTable { table, enc }); self.open.len() - 1 } }; @@ -540,8 +587,24 @@ impl PlanWriter { } /// Flush spool file and finalize plan metadata - pub fn seal(mut self, commit_lsn: u64, commit_ts: i64) -> Result { + pub fn seal(self, commit_lsn: u64, commit_ts: i64) -> Result { + self.seal_keeping(commit_lsn, commit_ts) + .map(|(plan, _)| plan) + } + + /// [`Self::seal`], returning emptied encoders for the next plan + pub fn seal_keeping( + mut self, + commit_lsn: u64, + commit_ts: i64, + ) -> Result<(SealedPlan, EncoderCache)> { self.seal_segment()?; + let mut encoders = std::mem::take(&mut self.encoders); + for o in self.open.drain(..) { + let t = &self.tables[o.table as usize]; + let route = t.route.clone().expect("routed table"); + encoders.put(t.descriptor.clone(), route, o.enc); + } let file = match self.file.take() { None => None, Some((mut out, file)) => { @@ -549,7 +612,7 @@ impl PlanWriter { Some(file) } }; - Ok(SealedPlan { + let plan = SealedPlan { file, steps: self.steps, tables: self.tables, @@ -559,7 +622,8 @@ impl PlanWriter { size_bytes: self.bytes, commit_lsn, commit_ts, - }) + }; + Ok((plan, encoders)) } } @@ -1125,6 +1189,39 @@ mod tests { assert_eq!(seen, (0..64).collect::>()); } + /// Sealing returns emptied encoders that serve the next plan under the + /// same route; another route version compiles its own + #[test] + fn encoders_carry_across_plans() { + let tmp = tempfile::tempdir().unwrap(); + let (d, r) = (descriptor(16500), route()); + let mut encoders = EncoderCache::default(); + let routes = [r.clone(), r, route()]; + for (i, route) in routes.into_iter().enumerate() { + let mut w = PlanWriter::create(tmp.path().join(format!("{i}.plan")), 1 << 20, 1 << 20) + .with_encoders(encoders); + let cached = w.encoders.entries.len(); + push( + &mut w, + &d, + heap(&d, 100 + i as u64, i as i32), + Some(route.clone()), + ) + .unwrap(); + let reused = cached - w.encoders.entries.len(); + assert_eq!((cached, reused), [(0, 0), (1, 1), (1, 0)][i]); + let plan; + (plan, encoders) = w.seal_keeping(0x2000, 42).unwrap(); + assert_eq!(encoders.entries.len(), 1); + assert!(Arc::ptr_eq(&encoders.entries[0].route, &route)); + let mut rd = plan.replay().unwrap(); + let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() else { + panic!("one segment"); + }; + assert_eq!(rows(&s.chunks), [(Some(i as i32), 100 + i as u64, 7, 0)]); + } + } + /// Oracle cells and strings survive the spool byte-identical #[test] fn oracle_and_string_chunks_round_trip() { diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 8b92109a..01ae8663 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -42,7 +42,8 @@ use std::sync::Arc; use crate::decode::datum::DatumRef; use crate::decode::heap_decoder::view_heap_record; use crate::emit::pipeline::plan_spool::{ - DEFAULT_PLAN_MEM_MAX, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, + DEFAULT_PLAN_MEM_MAX, EncoderCache, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, + ViewChange, }; use crate::emit::route::RouteSnapshot; use crate::schema::RelDescriptor; @@ -68,6 +69,12 @@ pub trait PlanRouteView { &mut self, entry: &DrainEntry, ) -> impl std::future::Future> + Send; + /// Encoders an earlier plan compiled against routes this view still + /// answers with + fn take_encoders(&mut self) -> EncoderCache { + EncoderCache::default() + } + fn keep_encoders(&mut self, _encoders: EncoderCache) {} } #[derive(Debug, thiserror::Error)] @@ -143,7 +150,8 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Self { writer: PlanWriter::create(path, disk_cap, DEFAULT_PLAN_MEM_MAX) .with_segment_limits(segment.0, segment.1) - .for_commit(commit_ts), + .for_commit(commit_ts) + .with_encoders(view.take_encoders()), view, resolver, rows_base: 0, @@ -280,7 +288,8 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { /// Freeze the validated plan for the executor pub fn seal(self, commit_lsn: u64, commit_ts: i64) -> Result { - let plan = self.writer.seal(commit_lsn, commit_ts)?; + let (plan, encoders) = self.writer.seal_keeping(commit_lsn, commit_ts)?; + self.view.keep_encoders(encoders); for t in &plan.tables { if t.route.is_none() { self.view.unmapped(&t.descriptor, t.heaps); diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index 1ada5aa1..a2659840 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -49,7 +49,7 @@ use crate::config::ResolvedConfig; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::{AckHandle, BufferedRowSender, Publish, SeqAlloc}; use crate::emit::pipeline::batcher::{BatcherMsg, PlacedSegment}; -use crate::emit::pipeline::plan_spool::{PlanItem, PlanPath, SealedPlan, Segment}; +use crate::emit::pipeline::plan_spool::{EncoderCache, PlanItem, PlanPath, SealedPlan, Segment}; use crate::emit::pipeline::planner::{PlanRouteView, Planner, drain_reason}; use crate::emit::route::{RouteSnapshot, RowPolicy, route_via}; use crate::mapping::{MappingSnapshot, TableMapping}; @@ -1362,6 +1362,8 @@ pub struct RouteMaps { memo: HashMap, Option>)>, /// Routes by owning relation owned: HashMap>>, + /// Compiled against `memo` routes, cleared with them + encoders: EncoderCache, } impl RouteMaps { @@ -1369,6 +1371,7 @@ impl RouteMaps { self.overlay.clear(); self.memo.clear(); self.owned.clear(); + self.encoders.clear(); } } @@ -1432,6 +1435,14 @@ impl PlanRouteView for ReorderRouteView<'_> { } } + fn take_encoders(&mut self) -> EncoderCache { + std::mem::take(&mut self.maps.encoders) + } + + fn keep_encoders(&mut self, encoders: EncoderCache) { + self.maps.encoders = encoders; + } + fn unmapped(&mut self, rel: &RelDescriptor, heaps: u64) { self.stats .unsupported_relations From 64067c41ecfec98e8bd05ad28d7b4988a0165460 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 7 Nov 2026 05:05:35 +0000 Subject: [PATCH 09/20] prepared blocks & optimization --- docs/configuration.md | 12 +- plans/INDEX.md | 2 +- plans/native_encoding.md | 24 +-- src/bin/stream/metrics_publish.rs | 3 +- src/ch.rs | 2 +- src/emit/ch_emitter.rs | 6 +- src/emit/pipeline/inserter.rs | 43 +++-- src/emit/pipeline/resolver.rs | 36 ++-- src/emit/pipeline/sealed.rs | 304 +++++++++++++----------------- src/ops/metrics.rs | 8 +- 10 files changed, 208 insertions(+), 232 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2e997d4f..8b4ea18f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -339,14 +339,16 @@ sealed batch can hold one additional batch. A single oversized row may exceed these limits. Oracle responses and native client buffers remain additional costs Sealed-buffer permits stay held through resolution, retries and insert completion. -After resolution each Native block is serialized once and compressed with -`[ch] compression`, and its permit shrinks to compressed bytes; blocks that do -not shrink stay uncompressed. Each inserter decompresses one block at a time to -send it, so one block of scratch per inserter adds to these limits. An INSERT +After resolution each Native block is serialized once, compressed with +`[ch] compression`, and kept in that form until acknowledged; its permit shrinks +to memory those bytes occupy, which can exceed their length. Inserters send +those bytes unchanged. A connection unable to accept them, such as a +destination configured with other compression, gets each block decoded and +rebuilt, so one block of scratch per inserter adds to these limits. An INSERT carries several blocks, sealed at 1,048,576 rows or 16 MiB of column bytes and when oracle request size fills, while batch limits still bound each INSERT. `walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` -reports achieved compression +reports how far sealing shrinks resident blocks This pool is separate from decoded-payload admission to avoid circular waits. Live byte-budget reductions lower flush thresholds; increasing sealed-buffer capacity requires restarting walshadow diff --git a/plans/INDEX.md b/plans/INDEX.md index 68b7b666..83690ee5 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -39,7 +39,7 @@ writes. A documented limitation does not imply code rejects it | [Multi-database loads and metrics](multi_database.md) | Extend heap-page bootstrap and backup loads beyond primary database, attribute metrics | | [Fuzzing](fuzzing.md) | Find parser and schema-transition interactions beyond fixed regressions | | [Performance](performance.md) | Establish healthy sustained WAL baseline with exact reconciliation before tuning | -| [Direct Native encoding](native_encoding.md) | Encode loads and TOAST rows without cells, send prepared payloads, prove resource gains end to end | +| [Direct Native encoding](native_encoding.md) | Encode loads and TOAST rows without cells, prove resource gains end to end | | [Failover](failover.md) | Fence unplanned promotion and prove remaining archive/restart fault cases | | [Shadow TOAST reclamation](shadow_toast.md) | Keep historical values readable under lag and restart | | [Replay callback](custom_rmgr.md) | Reduce measured command-boundary capture stalls | diff --git a/plans/native_encoding.md b/plans/native_encoding.md index 6bbef8a2..94263248 100644 --- a/plans/native_encoding.md +++ b/plans/native_encoding.md @@ -3,8 +3,7 @@ Streaming commits plan rows straight from WAL bytes into column chunks, and sealed Native blocks stay compressed until acknowledgement; see [architecture](../architecture/README.md#commit-pipeline). Remaining work -removes owned cells from load paths and TOAST rows, sends prepared payloads, -and proves resource gains end to end. Follow [shared constraints](coordination.md), +removes owned cells from load paths and TOAST rows and proves resource gains end to end. Follow [shared constraints](coordination.md), [value coercion](value_coercion.md), and [performance acceptance](performance.md) ## Encode loads directly @@ -37,25 +36,10 @@ scratch, then encode with the view path, preserving size policy, fill modes, read-ahead transfer, and chunk generation fences. Compressed inline values and `numeric`/`jsonb` already decode through per-row scratch -## Send prepared payloads - -clickhouse-c `chc_block_prepare` / `chc_client_send_prepared`, bound in -clickhouse-c-rs as `BlockBuilder::prepare` and `send_prepared`, write prepared -Data bytes on connections negotiating the same compression and revision. The -clickhouse-c-rs walshadow depends on predates them, so -[sealed blocks](../src/emit/pipeline/sealed.rs) store a Native body compressed -by walshadow and each inserter decodes it back into a block to send, -recompressing on the wire. With prepared sends, seal through -`BlockBuilder::prepare` with the destination's negotiated codec and send -without decoding; rebuild through the decode path only for a connection that -rejects the block, such as a named destination configured differently. Count -rebuilds separately - ## Bound memory across owners Track allocated capacity, not lengths, for raw inputs, open column builders, -oracle cells and responses, sealed blocks, compression scratch, and transport -buffers, and keep them distinct from process RSS. Bound queue bytes as well as +oracle cells and responses, compression scratch, and transport buffers, and keep them distinct from process RSS. Bound queue bytes as well as item counts, including spare slab capacity and per-chunk metadata for small rows across many tables. Oracle cells own one allocation each; pack them into shared buffers if they dominate @@ -85,8 +69,8 @@ against `more-types` (`69f1700`) using `xact_spill`, `spool_read`, and [benchmark driver](../bench/README.md). Run with unchanged worker counts, batching limits, and codec settings. Record rows/s, source WAL bytes/s, CPU, allocation count, peak/steady RSS, retained bytes by owner, spill volume, -`walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total`, -durable lag, blocks per INSERT, and actual ClickHouse part creation, with +`walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` +(stored includes allocation slack), durable lag, blocks per INSERT, and actual ClickHouse part creation, with compressible and incompressible payloads and both healthy and stalled destinations. Attribute savings separately to compact transaction storage, direct encoding, and compression. Choose block size and default compression diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 737f8d85..5f64f426 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -614,7 +614,8 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M insertbatch_batches_out_total: emitter(|s| &s.insertbatch_batches_out), inserter_batches_in_total: emitter(|s| &s.inserter_batches_in), inserter_ch_seconds_total: emitter_seconds(|s| &s.inserter_ch_nanos), - inserter_encode_seconds_total: emitter_seconds(|s| &s.inserter_encode_nanos), + inserter_rebuilds_total: emitter(|s| &s.inserter_rebuilds), + inserter_rebuild_seconds_total: emitter_seconds(|s| &s.inserter_rebuild_nanos), resident_seal_seconds_total: emitter_seconds(|s| &s.resident_seal_nanos), resident_raw_bytes_total: emitter(|s| &s.resident_raw_bytes), resident_stored_bytes_total: emitter(|s| &s.resident_stored_bytes), diff --git a/src/ch.rs b/src/ch.rs index e6fbd3da..1cea8371 100644 --- a/src/ch.rs +++ b/src/ch.rs @@ -77,7 +77,7 @@ impl std::str::FromStr for CompressionChoice { } impl CompressionChoice { - fn to_wire(self) -> Compression { + pub(crate) fn to_wire(self) -> Compression { match self { Self::None => Compression::None, Self::Lz4 => Compression::Lz4, diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 430f24dc..d62bf04f 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -3249,8 +3249,10 @@ crate::atomic_stats! { /// `EndOfStream`), retries included. Against `inserter_pool_size × /// elapsed` this is CH-side utilization pub inserter_ch_nanos, - /// Inserter time rebuilding the Native block over the batch's slabs - pub inserter_encode_nanos, + /// Blocks decoded and rebuilt for a connection refusing their + /// prepared bytes, and inserter time doing so + pub inserter_rebuilds, + pub inserter_rebuild_nanos, /// Resolver time inside the oracle round trip, retries included. /// Overlaps the inserters' ClickHouse time, so against /// `inserter_ch_nanos` it says which stage is the limiter diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index 9ca276fb..d2c85f1e 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -3,16 +3,16 @@ //! ClickHouse Cloud INSERT cost is mostly RTT + object-store part commit, so //! throughput comes from keeping many INSERTs in flight. Each inserter pulls //! `InsertBatch`es off the shared spmc queue (any idle inserter takes any -//! batch, so a hot table can use more than one connection), rebuilds the -//! Native block over the batch's owned slabs, and runs one `send_query` + -//! `send_data` + `send_data_end` + drain-to-`EndOfStream` INSERT. +//! batch, so a hot table can use more than one connection), and runs one +//! `send_query`, a prepared Data packet per block, `send_data_end` and +//! drain-to-`EndOfStream` INSERT. //! //! Durability invariant: [`AckHandle::acked`] takes the drain's //! [`EndOfStream`] proof. Until then a connection drop replays the still-owned batch (CH //! dedups by `_lsn`). Retry-exhaustion is fatal: the watermark can't advance //! without this batch. -use clickhouse_c::{Allocator, QueryOpts}; +use clickhouse_c::{Allocator, ErrorKind, QueryOpts}; use tokio::task::JoinHandle; use crate::ch::{ChConn, EmitterError, EndOfStream, drain_to_end_of_stream, with_timeout}; @@ -29,8 +29,8 @@ use std::sync::atomic::Ordering; struct Inserter { client: ChConn, alloc: Allocator, - /// One block's decompressed body, reused across sends; locked only - /// while a block opens + /// One block's decompressed body for rebuilds, reused across sends; + /// locked only while a block opens scratch: std::sync::Mutex>, ack: AckHandle, stats: Arc, @@ -38,8 +38,9 @@ struct Inserter { impl Inserter { /// Bounded reconnect+retry around one INSERT. Every attempt resends - /// every block in order, so a replay carries unchanged rows; blocks - /// open one at a time into `scratch` + /// every block in order, so a replay carries unchanged rows. A + /// connection refusing prepared bytes gets blocks opened one at a time + /// into `scratch` and rebuilt async fn send_with_retry( &mut self, sql: &str, @@ -58,6 +59,11 @@ impl Inserter { let result = with_timeout(insert_timeout, async { client.send_query(sql, &QueryOpts::new()).await?; for sealed in blocks { + match client.send_prepared(sealed.prepared()).await { + Ok(()) => continue, + Err(e) if e.kind != ErrorKind::Usage => return Err(e.into()), + Err(_) => {} + } let opened = std::time::Instant::now(); let block = { let mut scratch = @@ -65,7 +71,8 @@ impl Inserter { sealed.open(alloc, &mut scratch)? }; let bb = builder_of(&block)?; - stats.inserter_encode_nanos.fetch_add( + stats.inserter_rebuilds.fetch_add(1, Ordering::Relaxed); + stats.inserter_rebuild_nanos.fetch_add( opened.elapsed().as_nanos() as u64, Ordering::Relaxed, ); @@ -174,16 +181,19 @@ mod tests { use clickhouse_c::{BlockBuilder, ColumnBuilder, TypeAst}; /// One UInt8 column x holding one row 42 - fn one_row_block(codec: CompressionChoice) -> SealedBlock { - let ast = TypeAst::parse("UInt8", Allocator::global(&mimalloc::MiMalloc)).unwrap(); + fn one_row_block(compression: CompressionChoice) -> SealedBlock { + let alloc = Allocator::global(&mimalloc::MiMalloc); + let ast = TypeAst::parse("UInt8", alloc).unwrap(); let column = ColumnBuilder::fixed(&[42], 1, 1).unwrap(); let mut bb = BlockBuilder::new(); bb.append("x", ast.view(), &column).unwrap(); - SealedBlock::seal(&bb, 1, codec, &mut Vec::new()).unwrap() + let codec = compression.build_codec().unwrap(); + SealedBlock::seal(&bb, compression, codec.as_ref().map(|c| c.as_ref()), alloc).unwrap() } - /// Every attempt resends every block of the INSERT, whatever codec held - /// them, through failed and idle reconnects + /// Every attempt resends every block of the INSERT through failed and + /// idle reconnects; blocks prepared for other compression than the + /// uncompressed server connection rebuild #[tokio::test] async fn insert_retry_preserves_payload_through_failed_reconnect() { let sql = "INSERT INTO retry_test (x) VALUES"; @@ -220,6 +230,11 @@ mod tests { stats.reconnects.load(Ordering::Relaxed), u64::from(retries == 2 || idle) ); + assert_eq!( + stats.inserter_rebuilds.load(Ordering::Relaxed) > 0, + codec != CompressionChoice::None, + "{codec:?}" + ); server.await.unwrap(); drop(inserter); collector.await.unwrap(); diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index 775e7a4a..6bf00b26 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -54,7 +54,8 @@ pub(crate) struct ResolverOptions { pub oracle: Option>, /// Boot budget for oracle transport retries, re-read from `config_rx` pub retry: RetryConfig, - /// Codec compressing sealed blocks while they wait for acknowledgement + /// Compression of prepared blocks, matching connections that negotiate + /// it; others rebuild pub codec: CompressionChoice, pub stats: Arc, pub fatal: Fatal, @@ -74,25 +75,25 @@ pub(crate) fn spawn_pool( .collect() } -/// Per-task resolver state: parsed column types and serialization scratch +/// Per-task resolver state: parsed column types and compression codec pub(crate) struct Sealer { alloc: Allocator, - codec: CompressionChoice, + compression: CompressionChoice, + codec: Option>>, /// Parsed column types per table, refreshed when a batch's /// `schema_epoch` changes. `TypeAst` is `Send` but not `Sync`, so each /// task parses its own asts: HashMap)>, - scratch: Vec, } impl Sealer { - pub(crate) fn new(codec: CompressionChoice) -> Self { - Self { + pub(crate) fn new(compression: CompressionChoice) -> Result { + Ok(Self { alloc: Allocator::global(&mimalloc::MiMalloc), - codec, + compression, + codec: compression.build_codec()?, asts: HashMap::default(), - scratch: Vec::new(), - } + }) } /// Serialize and compress one block over its slabs and oracle answer @@ -144,7 +145,12 @@ impl Sealer { } } } - SealedBlock::seal(&bb, n_rows, self.codec, &mut self.scratch) + SealedBlock::seal( + &bb, + self.compression, + self.codec.as_ref().map(|c| c.as_ref()), + self.alloc, + ) } } @@ -162,7 +168,13 @@ async fn run( config_rx, } = opts; let alloc = Allocator::global(&mimalloc::MiMalloc); - let mut sealer = Sealer::new(codec); + let mut sealer = match Sealer::new(codec) { + Ok(sealer) => sealer, + Err(e) => { + fatal.set(format!("seal codec: {e}")); + return; + } + }; while let Ok(batch) = rx.recv().await { if let Some(rx) = config_rx.as_ref() { retry.max_attempts = rx.borrow().retry_max_attempts; @@ -549,7 +561,7 @@ pub(crate) mod tests { /// Block shape faults fail sealing #[tokio::test] async fn unanswered_oracle_column_and_bad_type_fail_seal() { - let mut sealer = Sealer::new(CompressionChoice::None); + let mut sealer = Sealer::new(CompressionChoice::None).unwrap(); let mut raw = wkt_row(0, None); raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { type_oid: GEOM_OID, diff --git a/src/emit/pipeline/sealed.rs b/src/emit/pipeline/sealed.rs index e30ff509..6ffec5ae 100644 --- a/src/emit/pipeline/sealed.rs +++ b/src/emit/pipeline/sealed.rs @@ -1,83 +1,66 @@ -//! Resolved Native blocks kept compressed from seal until INSERT -//! acknowledgement. +//! Resolved Native blocks kept as prepared Data packet bodies from seal until +//! INSERT acknowledgement. //! -//! clickhouse-c serializes each block once; the body is compressed with the -//! configured codec and column slabs drop. An inserter decodes one block at -//! a time back into a scratch buffer to send it, so resident output costs -//! compressed bytes plus one block per inserter. Incompressible bodies stay -//! raw rather than expand +//! clickhouse-c serializes each block once, compressing wire frames as it +//! goes, and column slabs drop. Inserters write those bytes on connections +//! negotiating the same compression; a connection refusing them, such as a +//! named destination configured otherwise, gets the block decoded and rebuilt use std::pin::Pin; -use clickhouse_c::{Allocator, Block, BlockBuilder, BlockOpts, BlockReader, Io, SliceIo, sys}; +use clickhouse_c::{ + Allocator, Block, BlockBuilder, BlockOpts, BlockReader, Codec, PreparedBlock, SliceIo, +}; use crate::ch::{CompressionChoice, EmitterError}; -/// Native file framing: blocks never leave this process in this form +/// TCP Data packet body framing, which prepared bytes carry const OPTS: BlockOpts = BlockOpts { - has_block_info: false, - has_custom_serialization: false, + has_block_info: true, + has_custom_serialization: true, read_buffer_bytes: 0, }; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Stored { - Raw, - Lz4, - Zstd, -} +/// CityHash128 checksum, then method byte and two little-endian u32 sizes; +/// ClickHouse `src/Compression/CompressionInfo.h` +const CHECKSUM: usize = 16; +const FRAME_HEADER: usize = 9; +const METHOD_NONE: u8 = 0x02; +const METHOD_LZ4: u8 = 0x82; +const METHOD_ZSTD: u8 = 0x90; -/// One serialized block body, owned until its INSERT is durable +/// One prepared block body, owned until its INSERT is durable pub(crate) struct SealedBlock { - stored: Stored, - payload: Box<[u8]>, - raw_len: usize, - n_rows: usize, + prepared: PreparedBlock, + compression: CompressionChoice, } impl SealedBlock { - /// Serialize `bb` and compress its body. `scratch` holds the raw body - /// between calls so sealing reuses one buffer + /// Serialize `bb` once in `compression`; `codec` must support it pub(crate) fn seal( bb: &BlockBuilder<'_>, - n_rows: usize, - codec: CompressionChoice, - scratch: &mut Vec, + compression: CompressionChoice, + codec: Option>, + alloc: Allocator, ) -> Result { - scratch.clear(); - let mut io = VecIo::new(std::mem::take(scratch)); - let written = bb.write(io.as_mut(), OPTS); - *scratch = io.as_mut().take(); - written?; - let raw_len = scratch.len(); - let compressed = match codec { - CompressionChoice::None => None, - CompressionChoice::Lz4 => lz4::block::compress(scratch, None, false) - .ok() - .map(|c| (Stored::Lz4, c)), - CompressionChoice::Zstd => zstd::bulk::compress(scratch, 0) - .ok() - .map(|c| (Stored::Zstd, c)), - }; - let (stored, payload) = match compressed { - Some((stored, c)) if c.len() < raw_len => (stored, c.into_boxed_slice()), - _ => (Stored::Raw, scratch.as_slice().into()), - }; + let codec = codec.filter(|_| compression != CompressionChoice::None); Ok(Self { - stored, - payload, - raw_len, - n_rows, + prepared: bb.prepare(compression.to_wire(), codec, alloc)?, + compression, }) } + pub(crate) fn prepared(&self) -> &PreparedBlock { + &self.prepared + } + /// Serialized body bytes before compression pub(crate) fn raw_len(&self) -> usize { - self.raw_len + self.prepared.raw_len() } pub(crate) fn allocated_bytes(&self) -> usize { - self.payload.len() + self.prepared.capacity() } /// Decode the block, decompressing into `scratch`; the block owns its @@ -87,41 +70,66 @@ impl SealedBlock { alloc: Allocator, scratch: &mut Vec, ) -> Result { - let body: &[u8] = match self.stored { - Stored::Raw => &self.payload, - Stored::Lz4 => { - scratch.resize(self.raw_len, 0); - let n = lz4::block::decompress_to_buffer( - &self.payload, - Some(self.raw_len as i32), - scratch, - ) - .map_err(|e| EmitterError::Type(format!("sealed block lz4: {e}")))?; - &scratch[..n] - } - Stored::Zstd => { - scratch.resize(self.raw_len, 0); - let n = zstd::bulk::decompress_to_buffer(&self.payload, &mut scratch[..]) - .map_err(|e| EmitterError::Type(format!("sealed block zstd: {e}")))?; - &scratch[..n] + let bytes = self.prepared.as_bytes(); + let body = match self.compression { + CompressionChoice::None => bytes, + _ => { + unframe(bytes, scratch)?; + &scratch[..] } }; - if body.len() != self.raw_len { + if body.len() != self.raw_len() { return Err(EmitterError::Type(format!( "sealed block body {} bytes, sealed {}", body.len(), - self.raw_len + self.raw_len() ))); } let mut io = SliceIo::new(body); let mut reader = BlockReader::new(io.as_mut(), alloc, OPTS)?; reader .read()? - .filter(|b| b.n_rows() == self.n_rows) + .filter(|b| b.n_rows() == self.prepared.n_rows()) .ok_or_else(|| EmitterError::Type("sealed block decoded short".into())) } } +/// Concatenate decompressed frames of `bytes` into `out` +fn unframe(mut bytes: &[u8], out: &mut Vec) -> Result<(), EmitterError> { + let corrupt = |what: &str| EmitterError::Type(format!("sealed block frame {what}")); + out.clear(); + while !bytes.is_empty() { + let header = bytes + .get(CHECKSUM..CHECKSUM + FRAME_HEADER) + .ok_or_else(|| corrupt("header short"))?; + let size = |at: usize| u32::from_le_bytes(header[at..at + 4].try_into().unwrap()) as usize; + let (stored, raw) = (size(1), size(5)); + let data = stored + .checked_sub(FRAME_HEADER) + .and_then(|n| bytes.get(CHECKSUM + FRAME_HEADER..CHECKSUM + FRAME_HEADER + n)) + .ok_or_else(|| corrupt("body short"))?; + let at = out.len(); + out.resize(at + raw, 0); + let dst = &mut out[at..]; + let n = match header[0] { + METHOD_LZ4 => lz4::block::decompress_to_buffer(data, Some(raw as i32), dst) + .map_err(|e| corrupt(&format!("lz4: {e}")))?, + METHOD_ZSTD => zstd::bulk::decompress_to_buffer(data, dst) + .map_err(|e| corrupt(&format!("zstd: {e}")))?, + METHOD_NONE if data.len() == raw => { + dst.copy_from_slice(data); + raw + } + m => return Err(corrupt(&format!("method {m:#x}"))), + }; + if n != raw { + return Err(corrupt("size")); + } + bytes = &bytes[CHECKSUM + stored..]; + } + Ok(()) +} + /// Rebuild a builder over a decoded block's columns, for `send_data` pub(crate) fn builder_of(block: &Block) -> Result, EmitterError> { let mut bb = BlockBuilder::new(); @@ -138,76 +146,6 @@ pub(crate) fn builder_of(block: &Block) -> Result, EmitterError Ok(bb) } -/// Write-only transport appending to an owned buffer -struct VecIo { - io: sys::chc_io, - out: Vec, - _pin: std::marker::PhantomPinned, -} - -impl VecIo { - fn new(out: Vec) -> Pin> { - let mut boxed = Box::pin(Self { - io: sys::chc_io { - ud: std::ptr::null_mut(), - read: Some(vec_read), - write: Some(vec_write), - check_cancel: None, - }, - out, - _pin: std::marker::PhantomPinned, - }); - // SAFETY: context set once address is stable; value stays pinned - unsafe { - let this = boxed.as_mut().get_unchecked_mut(); - this.io.ud = (this as *mut Self).cast(); - } - boxed - } - - fn take(self: Pin<&mut Self>) -> Vec { - // SAFETY: buffer is not structurally pinned; callbacks reach it only - // through `ud` during a write call - std::mem::take(unsafe { &mut self.get_unchecked_mut().out }) - } -} - -// SAFETY: `io_ptr` points at the pinned callback table whose `ud` names -// this value; callbacks follow the chc_io contract -unsafe impl Io for VecIo { - fn io_ptr(self: Pin<&mut Self>) -> *mut sys::chc_io { - // SAFETY: table address stays fixed while pinned - unsafe { &mut self.get_unchecked_mut().io } - } -} - -unsafe extern "C" fn vec_write( - ud: *mut std::ffi::c_void, - buf: *const std::ffi::c_void, - len: usize, - _err: *mut sys::chc_err, -) -> std::ffi::c_int { - // SAFETY: `ud` points at the pinned VecIo owning this table; C passes - // `len` readable bytes - unsafe { - let this = &mut *ud.cast::(); - this.out - .extend_from_slice(std::slice::from_raw_parts(buf.cast::(), len)); - } - sys::CHC_OK -} - -unsafe extern "C" fn vec_read( - _ud: *mut std::ffi::c_void, - _buf: *mut std::ffi::c_void, - _len: usize, - _out_n: *mut usize, - err: *mut sys::chc_err, -) -> std::ffi::c_int { - let _ = err; - sys::CHC_ERR_USAGE -} - #[cfg(test)] mod tests { use super::*; @@ -218,7 +156,7 @@ mod tests { } /// Block of a UInt64, a Nullable(String) and a UUID column - fn seal_rows(strings: &[Option<&[u8]>], codec: CompressionChoice) -> (SealedBlock, Vec) { + fn seal_rows(strings: &[Option<&[u8]>], compression: CompressionChoice) -> SealedBlock { let n = strings.len(); let ids: Vec = (0..n as u64).flat_map(u64::to_le_bytes).collect(); let uuids: Vec = (0..16 * n).map(|i| (i * 7) as u8).collect(); @@ -238,14 +176,18 @@ mod tests { bb.append("id", types[0].view(), &id).unwrap(); bb.append("text", types[1].view(), &text).unwrap(); bb.append("uuid", types[2].view(), &uuid).unwrap(); - let mut scratch = Vec::new(); - let sealed = SealedBlock::seal(&bb, n, codec, &mut scratch).unwrap(); - (sealed, scratch) + let codec = compression.build_codec().unwrap(); + SealedBlock::seal( + &bb, + compression, + codec.as_ref().map(|c| c.as_ref()), + alloc(), + ) + .unwrap() } - fn check(strings: &[Option<&[u8]>], codec: CompressionChoice) -> SealedBlock { - let (sealed, raw) = seal_rows(strings, codec); - assert_eq!(sealed.raw_len(), raw.len()); + fn check(strings: &[Option<&[u8]>], compression: CompressionChoice) -> SealedBlock { + let sealed = seal_rows(strings, compression); let mut scratch = Vec::new(); let block = sealed.open(alloc(), &mut scratch).unwrap(); assert_eq!(block.n_rows(), strings.len()); @@ -262,16 +204,16 @@ mod tests { let (width, ids) = block.column(0).unwrap().fixed().unwrap(); assert_eq!(width, 8); assert_eq!(ids.len(), 8 * strings.len()); - // Reserialized for the wire, the block keeps its body - let mut again = Vec::new(); - SealedBlock::seal( + // Rebuilt for a refusing connection, the block prepares identically + let codec = compression.build_codec().unwrap(); + let again = SealedBlock::seal( &builder_of(&block).unwrap(), - strings.len(), - CompressionChoice::None, - &mut again, + compression, + codec.as_ref().map(|c| c.as_ref()), + alloc(), ) .unwrap(); - assert_eq!(again, raw); + assert_eq!(again.prepared().as_bytes(), sealed.prepared().as_bytes()); sealed } @@ -279,28 +221,27 @@ mod tests { fn blocks_round_trip_through_every_codec() { let big = vec![b'x'; 1 << 20]; let rows: Vec> = vec![Some(b"a"), None, Some(b""), Some(&big), None]; - for codec in [ + for compression in [ CompressionChoice::None, CompressionChoice::Lz4, CompressionChoice::Zstd, ] { - let sealed = check(&rows, codec); - if codec == CompressionChoice::None { - assert_eq!(sealed.stored, Stored::Raw); + let sealed = check(&rows, compression); + if compression == CompressionChoice::None { + assert_eq!(sealed.prepared().as_bytes().len(), sealed.raw_len()); } else { assert!( sealed.allocated_bytes() * 10 < sealed.raw_len(), - "{codec:?}" + "{compression:?}" ); } } } - /// Bodies that do not shrink stay raw rather than expand #[test] - fn incompressible_body_stays_raw() { + fn incompressible_body_round_trips() { let mut x = 0x9E37_79B9_7F4A_7C15u64; - let noise: Vec = (0..64 << 10) + let noise: Vec = (0..200 << 10) .map(|_| { x ^= x << 13; x ^= x >> 7; @@ -308,18 +249,35 @@ mod tests { x as u8 }) .collect(); - for codec in [CompressionChoice::Lz4, CompressionChoice::Zstd] { - let sealed = check(&[Some(&noise)], codec); - assert_eq!(sealed.stored, Stored::Raw); - assert_eq!(sealed.allocated_bytes(), sealed.raw_len()); + for compression in [CompressionChoice::Lz4, CompressionChoice::Zstd] { + check(&[Some(&noise)], compression); } } #[test] - fn corrupt_body_fails_to_open() { - let (mut sealed, _) = seal_rows(&[Some(&b"abc"[..]); 100], CompressionChoice::Lz4); - assert_eq!(sealed.stored, Stored::Lz4); - sealed.raw_len += 1; - assert!(sealed.open(alloc(), &mut Vec::new()).is_err()); + fn corrupt_frames_fail_to_open() { + let sealed = seal_rows(&[Some(&b"abc"[..]); 100], CompressionChoice::Lz4); + let bytes = sealed.prepared().as_bytes(); + let mut out = Vec::new(); + unframe(bytes, &mut out).unwrap(); + assert_eq!(out.len(), sealed.raw_len()); + assert!(unframe(&bytes[..bytes.len() - 1], &mut out).is_err()); + assert!(unframe(&bytes[..CHECKSUM + 4], &mut out).is_err()); + let mut method = bytes.to_vec(); + method[CHECKSUM] = 0x7f; + assert!(unframe(&method, &mut out).is_err()); + let mut sizes = bytes.to_vec(); + sizes[CHECKSUM + 5] ^= 1; + assert!(unframe(&sizes, &mut out).is_err()); + let none = [ + &[0u8; CHECKSUM][..], + &[METHOD_NONE, 12, 0, 0, 0, 3, 0, 0, 0], + b"xyz", + ] + .concat(); + unframe(&none, &mut out).unwrap(); + assert_eq!(out, b"xyz"); + let short = [&none[..CHECKSUM + 5], &[4, 0, 0, 0], b"xyz"].concat(); + assert!(unframe(&short, &mut out).is_err()); } } diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index c16e44b8..f8161585 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -418,14 +418,16 @@ snapshot! { "InsertBatches an inserter finished draining to ClickHouse. insertbatch_batches_out - inserter_batches_in is the live backlog; their rates show inserter-pool saturation.", counter inserter_ch_seconds_total: f64 = "Cumulative inserter time inside the ClickHouse INSERT round trip. Against inserter_pool_size x uptime this is CH-side utilization.", - counter inserter_encode_seconds_total: f64 = - "Cumulative inserter time decompressing and decoding sealed blocks to send them.", + counter inserter_rebuilds_total: u64 = + "Sealed blocks decoded and rebuilt because a connection negotiated other compression or an older revision than they were prepared for.", + counter inserter_rebuild_seconds_total: f64 = + "Cumulative inserter time decoding and rebuilding sealed blocks.", counter resident_seal_seconds_total: f64 = "Cumulative resolver time serializing and compressing sealed Native blocks.", counter resident_raw_bytes_total: u64 = "Serialized Native block bytes sealed for insertion, before resident compression.", counter resident_stored_bytes_total: u64 = - "Bytes those sealed blocks held while awaiting insertion; raw over stored is the resident compression ratio.", + "Bytes allocated for those sealed blocks while awaiting insertion, including growth slack.", counter oracle_resolve_seconds_total: f64 = "Cumulative resolver time inside the oracle round trip. Overlaps inserter_ch_seconds_total, so the larger of the two is the tail's limiter.", counter process_cpu_seconds_total: f64 = From 20580ee93cef4076c37bec30e142058669aa6c70 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:18:51 +0000 Subject: [PATCH 10/20] update docs/plans/architecture --- architecture/README.md | 17 +++-- architecture/overview.svg | 2 +- architecture/values.md | 6 +- architecture/values.svg | 27 ++++---- architecture/workers.svg | 10 +-- docs/destination-tables.md | 7 +- docs/limitations.md | 6 +- plans/native_encoding.md | 18 +++-- plans/performance.md | 9 +-- plans/tier2.md | 131 +++++++++++++++++++++++-------------- 10 files changed, 144 insertions(+), 89 deletions(-) diff --git a/architecture/README.md b/architecture/README.md index 9626bfad..ae2ca46b 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -37,15 +37,17 @@ planning committed transactions ## Commit pipeline -![Commit pipeline: reorder places rows inline, one batcher feeds a conversion pool and ClickHouse inserters, and acknowledgements advance a contiguous watermark](workers.svg) +![Commit pipeline: reorder plans column chunks, one batcher feeds a conversion and sealing pool, inserters send prepared blocks, and acknowledgements advance a contiguous watermark](workers.svg) `BufferingDecoderSink` and `ReorderSink` share one record-queue worker `ReorderSink` plans each commit, encoding its rows into per-table column chunks, and places them onto the batcher in order. Batcher merges chunks of many commits into Native blocks and groups blocks into per-table INSERTs. A separate conversion pool resolves PostgreSQL types, then serializes each block -once and keeps it compressed until its INSERT is durable. Conversion concurrency -matches oracle capacity; `[ch].inserter_pool_size` sizes the inserter pool +once and keeps prepared bytes until its INSERT is durable. Compression follows +`[ch] compression`; inserters rebuild blocks only when a connection rejects +prepared bytes. Conversion concurrency matches oracle capacity; +`[ch].inserter_pool_size` sizes inserter pool Each inserter owns a ClickHouse connection and takes any ready batch Sequence numbers identify work slices, not necessarily whole transactions @@ -56,10 +58,11 @@ and plan data can spill to disk Queue worker runs on a dedicated thread. A dispatcher scans each batch ahead of it to start TOAST store reads for values updated rows keep. Ordinary heap records stay WAL bytes under their record-time descriptor until commit; -planning reads tuples in place and builds cells only for rows whose values must -be detoasted. Commits drain only their own transaction trees. -Consecutive small commits can share row chunks while retaining separate -acknowledgements +planning reads tuples in place. External TOAST rows and other owned inputs use +cell adapters; compressed inline values, numeric, JSONB, and fast defaults can +also need row scratch. Commits drain only their own transaction trees +Consecutive small commits can share column blocks while retaining separate +acknowledgements. COPY and backup loads still enter batcher as cell rows ## Design boundaries diff --git a/architecture/overview.svg b/architecture/overview.svg index f7ac034d..23790359 100644 --- a/architecture/overview.svg +++ b/architecture/overview.svg @@ -68,7 +68,7 @@ lookup ReorderSink → commit pipeline - Inline rows → batcher → conversion pool → inserters + Column chunks → batcher → conversion / sealing → inserters SchemaEvent / TRUNCATE use an ordered DDL barrier diff --git a/architecture/values.md b/architecture/values.md index 5e17e7ba..cbe701ed 100644 --- a/architecture/values.md +++ b/architecture/values.md @@ -44,8 +44,10 @@ Conversion workers run between batcher and inserters, so PostgreSQL conversion overlaps ClickHouse insertion. Blocks carrying only local columns pass through without a PostgreSQL request. Conversion workers combine local and converted columns into complete Native blocks, serialize and compress each once, and -release column buffers. Inserters decode one block at a time to send it and -resend every block of an INSERT across retries +release column buffers. Inserters send prepared bytes directly. A connection +rejecting prepared bytes, such as one using different compression, gets one +block decoded and rebuilt at a time. Retries resend every block of an INSERT; +prepared blocks remain owned until every required destination acknowledges Conversion failure stops batch with column and row context. Module pins output settings to make conversion reproducible. Greenfield bootstrap uses a temporary diff --git a/architecture/values.svg b/architecture/values.svg index 1525b98b..5604d733 100644 --- a/architecture/values.svg +++ b/architecture/values.svg @@ -1,6 +1,6 @@ TOAST resolution and PostgreSQL type conversion - Transaction heaps and TOAST chunks enter planning and resolution. ToastResolver reads or writes versioned ClickHouse mirrors for values outside the current transaction. InsertBatcher stores locally encoded column slabs or Oracle cells. A separate conversion pool sends ENCODE_NATIVE requests to shadow and validates returned Native columns. Inserters combine them with local columns and send complete ClickHouse INSERTs. + Transaction WAL and TOAST chunks enter planning and resolution. ToastResolver reads or writes versioned ClickHouse mirrors for values outside current transaction. Commit planning encodes column chunks; InsertBatcher merges them across commits. Conversion workers request and validate oracle columns, combine them with local columns, and seal prepared Native blocks. Inserters retain those blocks across retries and send complete ClickHouse INSERTs. @@ -15,11 +15,11 @@ XactBuffer - Heap tuples + TOAST chunks + WAL + cells + TOAST chunks In-memory or spilled ReorderSink / ToastResolver - Plan + resolve external values + Resolve TOAST; encode chunks Persist TOAST before dispatch pg_toast_<relid> @@ -29,10 +29,10 @@ put/fetch - Rows placed inline, in commit order + Column chunks, in commit order - InsertBatcher / TableEncoder - ColumnEncoding selects local slabs or Oracle cells; both remain in one sealed InsertBatch + InsertBatcher + Merge column chunks across commits; encode load rows; batch local columns and Oracle cells @@ -48,20 +48,21 @@ walshadow worker Bridge: ENCODE_NATIVE - One request per batch + One request per block Return partial Native block - Native response checks - Validate row count, names, - column types and block shape + Validate + seal + Check shape; combine columns + Serialize + compress once Native columns Inserter - Combine Native columns - Retain block across retries - + Send prepared blocks + Retain until acknowledged + Rebuild only if incompatible + diff --git a/architecture/workers.svg b/architecture/workers.svg index 661d3819..45831e19 100644 --- a/architecture/workers.svg +++ b/architecture/workers.svg @@ -1,6 +1,6 @@ Commit pipeline, queues and acknowledgements - ReorderSink plans commits and places rows inline in commit order. One batcher seals per-table batches. Conversion workers resolve PostgreSQL types before inserters send complete Native blocks to ClickHouse. Bounded queues separate stages. Register, Placed and Acked events track work slices; only contiguous completed work advances emitter_ack_lsn. + ReorderSink plans commits into encoded column chunks in commit order. One batcher merges chunks into per-table batches and encodes load rows. Conversion workers resolve PostgreSQL types and seal prepared Native blocks before inserters send them to ClickHouse. Bounded queues separate stages. Register, Placed and Acked events track work slices; only contiguous completed work advances emitter_ack_lsn. @@ -15,11 +15,11 @@ ReorderSink · record-queue worker - Plan commits and place rows inline, in commit order + Plan column chunks; dispatch in commit order Assign sequence numbers to work slices - Rows + FlushAll · bounded FIFO queue + Chunks / load rows / FlushAll · bounded FIFO @@ -36,7 +36,7 @@ Conversion pool · W tasks - Convert PostgreSQL types; pass local columns through + Resolve oracle columns; serialize and compress blocks resolver[1] @@ -52,7 +52,7 @@ Inserter pool · N tasks - Build Native blocks; retain each block across insert retries + Send prepared blocks; retain across INSERT retries inserter[1] diff --git a/docs/destination-tables.md b/docs/destination-tables.md index 884006f6..61861445 100644 --- a/docs/destination-tables.md +++ b/docs/destination-tables.md @@ -343,7 +343,12 @@ Common mappings include: | `inet`, `cidr`, `interval`, unknown types | `String` | Nullable source columns become `Nullable(...)` unless used as ClickHouse sort -keys. ClickHouse deployments using PostgreSQL `time` columns must enable +keys or mapped to arrays or maps. Default array and map mappings turn NULL into +empty containers +Arrays, maps, and vectors currently use shadow PostgreSQL conversion, including +when destination uses Native container types + +ClickHouse deployments using PostgreSQL `time` columns must enable `Time64` support Use a column rule to override an inferred type; see diff --git a/docs/limitations.md b/docs/limitations.md index 95d0cf3e..99030960 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -58,8 +58,10 @@ and workload limits before attaching ## Types shadow PostgreSQL converts Values outside walshadow's own codec set (arrays, `hstore`, enums, ranges, -domains, extension types) are converted by shadow PostgreSQL, one request per -insert batch +domains, extension types) are converted by shadow PostgreSQL in batches, one +request per Native block needing conversion. Native array or map destination +types still require this conversion; prepared sending and compression do not +remove shadow requests - a value shadow PostgreSQL cannot convert stops the batch and names the column and row, rather than writing a substituted value diff --git a/plans/native_encoding.md b/plans/native_encoding.md index 94263248..8f638c68 100644 --- a/plans/native_encoding.md +++ b/plans/native_encoding.md @@ -3,9 +3,14 @@ Streaming commits plan rows straight from WAL bytes into column chunks, and sealed Native blocks stay compressed until acknowledgement; see [architecture](../architecture/README.md#commit-pipeline). Remaining work -removes owned cells from load paths and TOAST rows and proves resource gains end to end. Follow [shared constraints](coordination.md), +removes owned cells from load paths and TOAST rows and proves resource gains end +to end. Follow [shared constraints](coordination.md), [value coercion](value_coercion.md), and [performance acceptance](performance.md) +Share view encoders and column buffers with [container codecs](tier2.md) +Container promotion can retain existing raw pending cells in owned adapters; +complete load and TOAST cell removal is not a prerequisite + ## Encode loads directly COPY and heap-page/bootstrap loads still decode every tuple into `ColumnValue` @@ -39,7 +44,8 @@ read-ahead transfer, and chunk generation fences. Compressed inline values and ## Bound memory across owners Track allocated capacity, not lengths, for raw inputs, open column builders, -oracle cells and responses, compression scratch, and transport buffers, and keep them distinct from process RSS. Bound queue bytes as well as +oracle cells and responses, compression scratch, and transport buffers, and keep +them distinct from process RSS. Bound queue bytes as well as item counts, including spare slab capacity and per-chunk metadata for small rows across many tables. Oracle cells own one allocation each; pack them into shared buffers if they dominate @@ -70,9 +76,9 @@ against `more-types` (`69f1700`) using `xact_spill`, `spool_read`, and batching limits, and codec settings. Record rows/s, source WAL bytes/s, CPU, allocation count, peak/steady RSS, retained bytes by owner, spill volume, `walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` -(stored includes allocation slack), durable lag, blocks per INSERT, and actual ClickHouse part creation, with -compressible and incompressible payloads and both healthy and stalled -destinations. Attribute savings separately to compact transaction storage, -direct encoding, and compression. Choose block size and default compression +(stored includes allocation slack), durable lag, blocks per INSERT, and actual +ClickHouse part creation, with compressible and incompressible payloads and +both healthy and stalled destinations. Attribute savings separately to compact +transaction storage, direct encoding, and compression. Choose block size and default compression from measured CPU/memory/part tradeoffs; do not claim gains from wire compression alone diff --git a/plans/performance.md b/plans/performance.md index 082e35c2..d433c134 100644 --- a/plans/performance.md +++ b/plans/performance.md @@ -89,10 +89,11 @@ Try these in order: insert latency, part count, RSS, and durable completion counts 2. Profile serial WAL dispatch before splitting it further [Queue worker](../src/source/queueing_record_sink.rs) now dispatches batches - with heap decode started ahead of dispatch. Measure transaction buffer lock - waits, commit draining, shadow replay, and downstream waits before changing - execution. Preserve commit/DDL order, byte-before-record reachability, and - contiguous durable acknowledgements + with TOAST reads started by a prescan ahead of dispatch. Ordinary heaps stay + WAL bytes until commit planning. Measure transaction buffer lock waits, + commit-time encoding, plan spill, oracle conversion, block sealing, shadow + replay, and downstream waits before changing execution. Preserve commit/DDL + order, byte-before-record reachability, and contiguous durable acknowledgements ## Bootstrap worker shape diff --git a/plans/tier2.md b/plans/tier2.md index b183894e..4973f9d1 100644 --- a/plans/tier2.md +++ b/plans/tier2.md @@ -5,11 +5,11 @@ process and writes ClickHouse wire columns without PostgreSQL. Tier 3 defers to shadow PostgreSQL through [conversion client](../src/ops/oracle.rs), described in [value architecture](../architecture/values.md) -Arrays, `hstore` maps, and pgvector vectors already resolve correctly through -tier 3. Promotion buys latency only. One tier-3 column makes every batch for that -table wait on a shadow round trip, so a single `int[]` column costs the round trip -even when every other column encodes locally. Use `oracle_resolve_seconds_total` -and `oracle_blocks_total` as baseline and acceptance measure +Arrays, `hstore` maps, and pgvector vectors already use tier 3. Promotion removes +shadow conversion work and round trips without changing destination values. One +tier-3 column makes each Native block needing conversion wait on shadow, even +when every other column encodes locally. Measure oracle requests and CPU beside +latency, allocations, and retained bytes ## Why jsonb was cheap and containers are not @@ -19,14 +19,19 @@ row. `ColumnBuf` offers `Fixed`, `String`, and nullable pairs, and accepts `JSON` as a string body, so `jsonb` needed a decoder and nothing else Containers break that assumption. `Array(T)` on wire is a `UInt64` offsets column -plus a flattened element column carrying its own null map. Rows contribute -variable element counts to shared element storage, so one row no longer maps to -one buffer write. `Array` reports `elem_size` zero, so an array target reaching +plus a flattened element column, with a null map when `T` is nullable. Rows +contribute variable element counts to shared storage, so one row no longer maps +to one buffer write. `Array` reports `elem_size` zero, so an array target reaching `new_for_ast` would silently take the `String` shape and write wrong bytes. Two -guards prevent that: `composite_target` routes `Kind::Array | Map | Object` to the -oracle, and `local_matrix_covers` omits array type OIDs +guards prevent that: `composite_target` routes `Array`, `Map`, `Object`, and +`QBit` to oracle, and `local_matrix_covers` omits array type OIDs -Promotion cost is therefore dominated by wire shape and value model, not by codecs +Commit planning already reads ordinary WAL tuples through borrowed datums and +writes encoded column chunks. Large plans spill those chunks; conversion workers +combine local and oracle columns, then seal prepared Native blocks. Extend this +path with container buffers and a borrowed element walk. Nested owned values +are not a prerequisite. See [direct encoding](native_encoding.md) for remaining +load and TOAST work ## Dependencies need no change @@ -37,21 +42,27 @@ through `chc_build_array`, and builds `Array(Tuple(..))` for maps through One optional upstream change is ergonomic. `ColumnBuilder::array` and `ColumnBuilder::nullable` store a raw pointer into the child builder, so every -wrap level must outlive the level above it. The inserter already holds one vector -per level, leaves then roots. `Array(Nullable(T))` needs three levels and `Map` -needs four, so each adds a vector and matching index discipline. An arena owning -nodes and returning indices would collapse those vectors into one structure -Decide it on readability, do not expect throughput from it +wrap level must outlive the level above it. Conversion workers currently hold +leaves then roots. Containers need deeper builder storage, with tuple pointer +scratch for maps. Choose stable per-level storage or an owning arena after +checking lifetimes. Treat this as builder organization, not a throughput claim ## Work in walshadow | Area | Change | |---|---| -| Wire shape | `ColumnBuf` array variant holding row offsets plus element buffer, and a build chain deep enough for `Array(Nullable(T))` | -| Value model | Nested `ColumnValue` array variant, spill tag and body encoding, `SPILL_VERSION` bump, recursive `approx_bytes` | -| Decode | Generalize on-disk `ArrayType` walk over element type, including null bitmap | -| Gates | Admit covered array OIDs in `local_matrix_covers`, narrow `composite_target` to shapes still lacking a local buffer | -| Type bridge | None, `array_ch_type` already fixes target types | +| Wire shape | Recursive array buffer with row offsets and child buffer; add tuple children for maps | +| Buffer operations | Append empty/default rows, account allocated capacity, compare shapes, and merge chunks with offsets rebased by child element count | +| Plan spill | Serialize and restore recursive column buffers; bump `PLAN_VERSION` when adding buffer tags | +| Decode | Walk borrowed on-disk `ArrayType` bodies, including null bitmap, into child buffers | +| Adapters | Reuse container encoding from borrowed datums and owned raw pending values; handle COPY text and fast defaults explicitly | +| Gates | Admit only supported source/target pairs and input representations; retain oracle routes for other shapes | +| Type bridge | Preserve existing mappings; define multidimensional behavior against configured target | + +Keep raw transaction spill unchanged when existing pending-value forms suffice +Version transaction or deferred-load formats only if their representation changes +Container encoding need not wait for complete removal of cells from load and +TOAST paths Give the walk its own `CodecError` variants for a rejected array header and an uncovered element type, so a malformed body is distinguishable from a type the @@ -59,12 +70,15 @@ table does not admit ### Pair source shape with target shape -`ColumnEncoding::choose` tests target and source independently. Once array OIDs -pass `local_matrix_covers`, an operator pin mapping an array column to `String` -would select local encoding with no buffer able to hold the value. Make the -choice consider both, so a pin to a scalar target still routes to the oracle -That keeps existing table configuration as the recovery valve for a divergence, -with no release needed +Choose encoding from source type, element codec, and target shape together +Admitting an array OID must not enable local encoding for a `String` override, +unsupported nested target, or `QBit`. Keep those pairs on oracle until covered + +Column-wide admission also affects owned load and TOAST rows. Heap values can +retain raw bodies in existing pending cells; COPY currently casts containers to +text. Support that text form or introduce an explicit oracle fallback before +opening gates. Fast defaults need equivalent handling. An unsupported input +must not enter a local-only buffer merely because WAL encoding works ### Element metadata @@ -72,12 +86,17 @@ The walk needs element length, alignment, and by-value flag. `RelAttr` carries those for the column type, which for an array column describes the array rather than the element, and carries no `typelem` -Prefer a static element table keyed by array type name, matching the mapping -`type_bridge::array_ch_type` already performs. Built-in array element OIDs are -fixed, and one table can supply length and alignment beside the ClickHouse type -This leaves both catalog paths untouched. Add `typelem` and element metadata to -`RelAttr` only when domains or extension arrays need coverage, and expect the -pinned worker and SQL snapshot paths to agree once it exists +Start with a static layout table keyed by built-in element OID. Validate header +element OID against admitted source type; type name alone is insufficient + +Separate `domain` branch resolves domain base OIDs and typmods during catalog +construction and reads `typelem`, but does not carry element layout into +`RelAttr`. Reuse that resolution when composing changes. Array bodies still +carry original element OIDs, including domain OIDs after descriptor +normalization. Retain enough metadata to resolve those identities at record +boundary, or keep such arrays on oracle. Cover domains over arrays and arrays +of domains separately. Keep pinned worker, SQL snapshot, and source bootstrap +metadata consistent Wrong alignment misreads silently instead of failing. Take alignment per element from PostgreSQL `src/include/catalog/pg_type.dat` and assert it per type, rather @@ -94,10 +113,16 @@ reproduce results `tests/oracle_types_e2e.rs` already asserts for tier 3 | NULL column | Empty array | | Empty array | Empty array | | Array holding NULL elements | Null elements in `Array(Nullable(T))` | -| Multi-dimensional array | Oracle, target stays `String` | +| Multi-dimensional array | Preserve configured target; use oracle for supported nested overrides until local nested encoding exists | | Non-zero lower bound | Elements in stored order, bounds dropped | | Element type outside the table | Oracle | +Default mapping uses array type name without inspecting dimensions, so a +multidimensional value does not automatically become `String`. Prove behavior +for default one-layer target, explicit nested target, and `String` override +Reject unsupported shape before emitting a partial row; never change target +schema from a value's dimensions + `decode_text_array` refuses a null bitmap, correct for its own caller and wrong for a column codec. Runtime configuration reads its own key columns before the bridge exists and wants `Vec`, so let it reuse the general walk behind a @@ -105,28 +130,38 @@ nulls-refusing adapter, as noted in [runtime configuration](runtime_config.md) ## Staging -1. Decode only: general `ArrayType` walk, nested `ColumnValue`, spill round trip, - element metadata table. Gates stay closed, so the oracle keeps serving arrays - and behavior does not move -2. Wire shape: array `ColumnBuf` and the deeper build chain, proven on - `Array(Nullable(T))` over both fixed and string elements -3. Open gates per element family, paired source and target shapes first -4. `vector` and `halfvec`: pgvector stores a dimension header plus float payload - rather than `ArrayType`, so it needs its own codec feeding the same array - buffer with non-nullable elements -5. `hstore`: `Map(String, Nullable(String))` is +1. Add recursive array buffers, merge/accounting/spill operations, and builder + storage. Prove `Array(Nullable(T))` over fixed and string elements while + production gates stay closed +2. Add borrowed `ArrayType` walk and element metadata. Reuse it for owned raw + pending bodies; cover COPY text, fast defaults, compressed and external values +3. Open gates per source/target pair after local/oracle parity across input + paths. Keep unsupported pairs on oracle +4. Add `vector` and `halfvec` codecs feeding same array buffer with non-nullable + elements. Their dimension header and float payload bypass `ArrayType`; this + can be first codec after buffer support if minimizing initial scope +5. Add `hstore`: `Map(String, Nullable(String))` is `Array(Tuple(String, Nullable(String)))`, needing tuple children, pointer scratch, and expansion equivalent to the module's `hstore_to_matrix` -Steps 1 and 2 carry no divergence risk because gates stay shut. Step 3 is where -divergence can first reach a destination +Keep existing production behavior while gates stay closed. Open each later +family only after proving its supported representations and overrides ## Completion Require identical destination contents through local and oracle paths for every admitted element type, across insert, update, delete, empty array, NULL column, -null elements, and TOASTed arrays. Cover spill and restore of nested values, and -a schema change adding an array column with a fast default +null elements, non-default lower bounds, and TOASTed arrays. Extend +[direct/owned parity](../src/emit/direct_parity.rs), then compare destination +contents against oracle independently across streaming, COPY, and backup loads +Cover raw transaction spill, recursive plan spill, merging chunks with empty and +NULL-bearing rows, and a schema change adding an array column with a fast +default. Test malformed headers separately from unsupported element types + +When composing domain support, include original domain element OIDs, nested +domain bases, typmods, and domains created in same transaction as their table +Round-trip prepared container blocks with none/LZ4/ZSTD and compression-mismatch +rebuilds; retain existing INSERT retry and acknowledgement guarantees Measure oracle round trips removed per table shape before widening element coverage From 4b0659c3571829be615c3ba2a60049b7bf611933 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:31:20 +0000 Subject: [PATCH 11/20] Move backfill & TOAST off of ColumnValue --- Cargo.lock | 2 +- architecture/README.md | 11 +- benches/bootstrap_pump.rs | 335 ++++++++---- benches/spool_read.rs | 24 +- plans/INDEX.md | 2 +- plans/native_encoding.md | 48 +- plans/tier2.md | 8 +- src/backfill/backfill_bootstrap.rs | 59 ++- src/backfill/backup_backfill.rs | 4 +- src/backfill/backup_page_walk.rs | 210 +++++--- src/backfill/copy_backfill.rs | 260 ++++++--- src/backfill/spool.rs | 115 ++-- src/backfill/visibility_gate.rs | 29 +- src/backfill/visibility_pending.rs | 79 +-- src/backfill/wal_replay.rs | 317 +++++++---- src/bin/stream/bootstrap.rs | 3 +- src/decode/heap_decoder.rs | 141 ++++- src/emit/ch_emitter.rs | 235 ++++++--- src/emit/direct_parity.rs | 77 ++- src/emit/pipeline/ack.rs | 5 + src/emit/pipeline/batcher.rs | 452 +++------------- src/emit/pipeline/bootstrap.rs | 789 +++++++++++++++++++++------- src/emit/pipeline/plan_spool.rs | 451 ++++++++++------ src/emit/pipeline/planner.rs | 74 +-- src/emit/pipeline/resolver.rs | 34 +- src/emit/pipeline/tail.rs | 52 +- src/toast/resolver.rs | 55 +- src/xact/spill.rs | 1 + src/xact/xact_buffer.rs | 280 ++++++---- tests/bootstrap_direct_e2e.rs | 4 +- tests/bootstrap_object_store_e2e.rs | 5 +- tests/bootstrap_pipeline_ch.rs | 5 +- tests/copy_chunk_resume_e2e.rs | 5 +- tests/emitter_budget_flush.rs | 22 +- tests/emitter_native_types.rs | 24 +- tests/emitter_tls.rs | 24 +- 36 files changed, 2700 insertions(+), 1541 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afb789fe..f800be2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2415,7 +2415,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/architecture/README.md b/architecture/README.md index ae2ca46b..5edf6a4d 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -58,11 +58,12 @@ and plan data can spill to disk Queue worker runs on a dedicated thread. A dispatcher scans each batch ahead of it to start TOAST store reads for values updated rows keep. Ordinary heap records stay WAL bytes under their record-time descriptor until commit; -planning reads tuples in place. External TOAST rows and other owned inputs use -cell adapters; compressed inline values, numeric, JSONB, and fast defaults can -also need row scratch. Commits drain only their own transaction trees -Consecutive small commits can share column blocks while retaining separate -acknowledgements. COPY and backup loads still enter batcher as cell rows +planning reads tuples in place, resolved external values patching them. +Stash-resolved records use cell adapters; compressed inline values, numeric, +JSONB, and fast defaults can also need row scratch. Commits drain only their own +transaction trees. Consecutive small commits can share column blocks while +retaining separate acknowledgements. COPY and backup loads carry tuple bytes or +COPY fields and encode their own column chunks ## Design boundaries diff --git a/benches/bootstrap_pump.rs b/benches/bootstrap_pump.rs index 8a185320..4c61cc7b 100644 --- a/benches/bootstrap_pump.rs +++ b/benches/bootstrap_pump.rs @@ -8,6 +8,11 @@ //! concurrent tar parts so shared-sink contention shows up. //! - `drain`: synthetic `BackfillTuple`s → `pipeline::bootstrap::drain` → //! metrics-only tail. Covers routing plus the `BatcherMsg` hop. +//! - `load`: `pump` feeding `drain` feeding the real batcher, sealed batches +//! dropped instead of inserted. Ends at the same `InsertBatch` blocks +//! whichever stage encodes rows, so it compares across encoder designs. +//! +//! `--text N` appends a text column of N bytes to every tuple. //! //! ```text //! cargo bench --bench bootstrap_pump -- \ @@ -45,12 +50,13 @@ use walshadow::backfill::backup_source::{ BackupSink, BackupSource, EndInfo, PumpStats, PumpTarget, StartInfo, pump_tar_to_sink, }; use walshadow::backfill_bootstrap::{BootstrapConfig, spawn_greenfield_bootstrap}; -use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, PAGE_BYTES}; +use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, PAGE_BYTES, TupleData}; +use walshadow::decode::datum::DatumRef; use walshadow::heap_decoder::ColumnValue; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::bootstrap; use walshadow::pos::{EmitterAck, Monotone}; -use walshadow::schema::{INT4OID, RelAttr, RelDescriptor, RelName, ReplIdent}; +use walshadow::schema::{INT4OID, RelAttr, RelDescriptor, RelName, ReplIdent, TEXTOID}; use walshadow::toast::ToastResolver; const DB_NODE: u32 = 5; @@ -72,6 +78,8 @@ struct Shape { tuples_per_page: usize, /// int4 columns per tuple columns: usize, + /// Trailing text column bytes, none at 0 + text: usize, parallelism: usize, } @@ -83,6 +91,7 @@ impl Default for Shape { pages_per_segment: 256, tuples_per_page: 100, columns: 8, + text: 0, parallelism: 4, } } @@ -102,7 +111,35 @@ impl Shape { } } -fn descriptor(filenode: u32, columns: usize) -> Arc { +fn descriptor(filenode: u32, shape: Shape) -> Arc { + let mut attributes: Vec = (0..shape.columns) + .map(|i| RelAttr { + attnum: i as i16 + 1, + name: format!("c{i}"), + type_oid: INT4OID, + typmod: -1, + not_null: false, + dropped: false, + type_name: "int4".into(), + type_byval: true, + type_len: 4, + type_align: 'i', + type_storage: 'p', + missing_default: None, + }) + .collect(); + if shape.text > 0 { + attributes.push(RelAttr { + attnum: shape.columns as i16 + 1, + name: "body".into(), + type_oid: TEXTOID, + type_name: "text".into(), + type_byval: false, + type_len: -1, + type_storage: 'x', + ..attributes[0].clone() + }); + } RelDescriptor { rfn: RelFileNode { spc_node: 1663, @@ -117,57 +154,69 @@ fn descriptor(filenode: u32, columns: usize) -> Arc { persistence: 'p', ancestors: Vec::new(), replident: ReplIdent::Default { pk_attnums: None }, - attributes: (0..columns) - .map(|i| RelAttr { - attnum: i as i16 + 1, - name: format!("c{i}"), - type_oid: INT4OID, - typmod: -1, - not_null: false, - dropped: false, - type_name: "int4".into(), - type_byval: true, - type_len: 4, - type_align: 'i', - type_storage: 'p', - missing_default: None, - }) - .collect(), + attributes, } .into() } -fn mapping(filenode: u32, columns: usize) -> TableMapping { +fn mapping(filenode: u32, shape: Shape) -> TableMapping { + let mut columns: Vec = (0..shape.columns) + .map(|i| ColumnMapping { + src_attnum: i as i16 + 1, + target_name: format!("c{i}"), + target_type: "Int32".into(), + type_pinned: false, + }) + .collect(); + if shape.text > 0 { + columns.push(ColumnMapping { + src_attnum: shape.columns as i16 + 1, + target_name: "body".into(), + target_type: "String".into(), + type_pinned: false, + }); + } TableMapping { target: TableTarget::new("bench", &format!("t{filenode}")), - columns: (0..columns) - .map(|i| ColumnMapping { - src_attnum: i as i16 + 1, - target_name: format!("c{i}"), - target_type: "Int32".into(), - type_pinned: false, - }) - .collect(), + columns, tees: Vec::new(), } } -/// One 8 KiB heap page carrying `tuples` live int4-only tuples, laid out -/// as PG writes them: line pointers up from the header, tuple bodies down -/// from the page end. Returns page and wrapping sum of its values -fn heap_page(tuples: usize, columns: usize, seed: i32) -> (Vec, i64) { - let body = TUPLE_HEADER + 4 * columns; +/// Stored text datum: 1-byte header below 127 bytes, else 4-byte +fn text_datum(len: usize) -> Vec { + let mut v = if len < 127 { + vec![(((len + 1) << 1) | 1) as u8] + } else { + (((len + 4) as u32) << 2).to_le_bytes().to_vec() + }; + v.resize(v.len() + len, b'x'); + v +} + +/// One 8 KiB heap page carrying `tuples` live tuples, laid out as PG writes +/// them: line pointers up from the header, MAXALIGNed tuple bodies down +/// from the page end. Returns page, tuples written, and wrapping sum of +/// int4 values +fn heap_page(tuples: usize, shape: Shape, seed: i32) -> (Vec, usize, i64) { + let columns = shape.columns; + let text = (shape.text > 0).then(|| text_datum(shape.text)); + let natts = columns + usize::from(text.is_some()); + let body = TUPLE_HEADER + 4 * columns + text.as_ref().map_or(0, Vec::len); + let stride = body.next_multiple_of(8); let mut page = vec![0u8; PAGE_BYTES]; let mut written = 0usize; let mut digest = 0i64; for i in 0..tuples { - let off = PAGE_BYTES - (i + 1) * body; + let Some(off) = PAGE_BYTES.checked_sub((i + 1) * stride) else { + break; + }; if off < SIZE_OF_PAGE_HEADER + (i + 1) * SIZE_OF_ITEM_ID { break; } let xmin = 100u32 + i as u32; page[off..off + 4].copy_from_slice(&xmin.to_le_bytes()); - page[off + 18..off + 20].copy_from_slice(&(columns as u16).to_le_bytes()); + page[off + 18..off + 20].copy_from_slice(&(natts as u16).to_le_bytes()); page[off + 20..off + 22].copy_from_slice(&0u16.to_le_bytes()); page[off + 22] = TUPLE_HEADER as u8; for c in 0..columns { @@ -176,36 +225,42 @@ fn heap_page(tuples: usize, columns: usize, seed: i32) -> (Vec, i64) { page[at..at + 4].copy_from_slice(&v.to_le_bytes()); digest = digest.wrapping_add(i64::from(v)); } + if let Some(t) = &text { + let at = off + TUPLE_HEADER + columns * 4; + page[at..at + t.len()].copy_from_slice(t); + } let slot = SIZE_OF_PAGE_HEADER + i * SIZE_OF_ITEM_ID; let raw = ((off as u32) & 0x7FFF) | (LP_NORMAL << 15) | (((body as u32) & 0x7FFF) << 17); page[slot..slot + SIZE_OF_ITEM_ID].copy_from_slice(&raw.to_le_bytes()); written = i + 1; } let pd_lower = SIZE_OF_PAGE_HEADER + written * SIZE_OF_ITEM_ID; - let pd_upper = PAGE_BYTES - written * body; + let pd_upper = PAGE_BYTES - written * stride; page[12..14].copy_from_slice(&(pd_lower as u16).to_le_bytes()); page[14..16].copy_from_slice(&(pd_upper as u16).to_le_bytes()); - (page, digest) + (page, written, digest) } /// One tar part holding `segments_per_part` `base//` entries, -/// with wrapping sum of values written -async fn build_part(shape: Shape, part: usize) -> (Vec, i64) { +/// with tuples and wrapping sum of int4 values written +async fn build_part(shape: Shape, part: usize) -> (Vec, u64, i64) { let mut writer = Writer::new(Vec::new()); let mut digest = 0i64; + let mut tuples = 0u64; for s in 0..shape.segments_per_part { let filenode = FIRST_FILENODE + (part * shape.segments_per_part + s) as u32; let mut body = Vec::with_capacity(shape.pages_per_segment * PAGE_BYTES); for p in 0..shape.pages_per_segment { - let (page, d) = heap_page(shape.tuples_per_page, shape.columns, (p * 1_000) as i32); + let (page, n, d) = heap_page(shape.tuples_per_page, shape, (p * 1_000) as i32); body.extend_from_slice(&page); + tuples += n as u64; digest = digest.wrapping_add(d); } let path = format!("base/{DB_NODE}/{filenode}"); let entry = Entry::new(path, Kind::Regular, body.len() as u64, 0o600, 0); writer.append(&entry, body.as_slice()).await.unwrap(); } - (writer.finish().await.unwrap(), digest) + (writer.finish().await.unwrap(), tuples, digest) } /// Object-store-shaped source over pre-built tars: one `pump_tar_to_sink` @@ -279,20 +334,40 @@ impl Report { } } -async fn bench_pump(shape: Shape) -> Report { +/// Tars, their tuple count and int4 digest, and the catalog naming them +async fn backup(shape: Shape) -> (Vec>, u64, i64, CatalogMap) { let mut parts = Vec::with_capacity(shape.parts); - let mut expected = 0i64; + let (mut tuples, mut digest) = (0u64, 0i64); for p in 0..shape.parts { - let (part, digest) = build_part(shape, p).await; + let (part, n, d) = build_part(shape, p).await; parts.push(part); - expected = expected.wrapping_add(digest); + tuples += n; + digest = digest.wrapping_add(d); } - let mut catalog = CatalogMap::new(); for i in 0..shape.segments() { - catalog.insert(descriptor(FIRST_FILENODE + i as u32, shape.columns)); + catalog.insert(descriptor(FIRST_FILENODE + i as u32, shape)); } + (parts, tuples, digest, catalog) +} + +fn pump_detail( + progress: &walshadow::backfill_bootstrap::BootstrapProgress, +) -> Vec<(&'static str, f64)> { + let ld = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64; + vec![ + ("pages_walked", ld(&progress.page_walk.pages_walked)), + ("decode_secs", ld(&progress.page_walk.decode_nanos) / 1e9), + ("tap_secs", ld(&progress.pump.sink_chunk_nanos) / 1e9), + ( + "channel_block_secs", + ld(&progress.page_walk.channel_block_nanos) / 1e9, + ), + ] +} +async fn bench_pump(shape: Shape) -> Report { + let (parts, tuples, expected, catalog) = backup(shape).await; let tmp = tempfile::tempdir().unwrap(); let cfg = BootstrapConfig::new(tmp.path().join("data")); let progress = cfg.progress.clone(); @@ -303,57 +378,52 @@ async fn bench_pump(shape: Shape) -> Report { let base = GLOBAL.reset_peak(); let started = Instant::now(); - let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog, false); + let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog.clone(), false); let drained = tokio::spawn(async move { let (mut n, mut digest) = (0u64, 0i64); while let Some(slab) = rx.recv().await { for t in &slab { - n += t.columns.len() as u64; - for c in &t.columns { - let Some(ColumnValue::Int4(v)) = c else { - panic!("int4 column decoded as {c:?}"); - }; - digest = digest.wrapping_add(i64::from(*v)); + let rel = catalog.get(t.rfn.db_node, t.rfn.rel_node).unwrap(); + let view = t.view(&rel).unwrap().unwrap(); + n += 1; + for d in &view.datums { + if let DatumRef::Fixed(b) = d { + let v = i32::from_le_bytes((*b).try_into().unwrap()); + digest = digest.wrapping_add(i64::from(v)); + } } } } (n, digest) }); pump.await.unwrap().unwrap(); - let (touched, digest) = drained.await.unwrap(); + let (walked, digest) = drained.await.unwrap(); let elapsed = started.elapsed().as_secs_f64(); let metered = GLOBAL.snapshot().since(base); - assert_eq!(digest, expected, "decoded values differ from written pages"); + assert_eq!(digest, expected, "walked values differ from written pages"); + assert_eq!(walked, tuples); - let ld = |a: &AtomicU64| a.load(Ordering::Relaxed) as f64; + let mut detail = pump_detail(&progress); + detail.push(("allocs", metered.allocs as f64)); + detail.push(("peak_mib", metered.peak as f64 / (1 << 20) as f64)); Report { label: "pump".into(), elapsed_secs: elapsed, - tuples: touched / shape.columns as u64, - bytes: ld(&progress.pump.bytes_tapped) as u64, - detail: vec![ - ("pages_walked", ld(&progress.page_walk.pages_walked)), - ("decode_secs", ld(&progress.page_walk.decode_nanos) / 1e9), - ("tap_secs", ld(&progress.pump.sink_chunk_nanos) / 1e9), - ( - "channel_block_secs", - ld(&progress.page_walk.channel_block_nanos) / 1e9, - ), - ("allocs", metered.allocs as f64), - ("peak_mib", metered.peak as f64 / (1 << 20) as f64), - ], + tuples, + bytes: progress.pump.bytes_tapped.load(Ordering::Relaxed), + detail, } } async fn bench_drain(shape: Shape) -> Report { let rels: Vec> = (0..shape.segments()) - .map(|i| descriptor(FIRST_FILENODE + i as u32, shape.columns)) + .map(|i| descriptor(FIRST_FILENODE + i as u32, shape)) .collect(); let mut catalog = CatalogMap::new(); let mut tables = foldhash::HashMap::default(); for r in &rels { catalog.insert(r.clone()); - tables.insert(r.rel_name.clone(), mapping(r.oid, shape.columns)); + tables.insert(r.rel_name.clone(), mapping(r.oid, shape)); } let routes = Arc::new(walshadow::mapping::shared(tables)); @@ -364,25 +434,32 @@ async fn bench_drain(shape: Shape) -> Report { let (tup_tx, tup_rx) = tokio::sync::mpsc::channel::>(16); let per_rel = shape.pages_per_segment as u64 * shape.tuples_per_page as u64; - let columns = shape.columns; + let text = "x".repeat(shape.text); let feeder = tokio::spawn(async move { for r in rels { // Slab at the walk's grain so the drain sees production shape for slab_start in (0..per_rel).step_by(shape.tuples_per_page * 16) { let end = (slab_start + (shape.tuples_per_page * 16) as u64).min(per_rel); let slab: Vec = (slab_start..end) - .map(|i| BackfillTuple { - rfn: r.rfn, - xid: 100 + i as u32, - xmax: 0, - infomask: 0, - source_lsn: START_LSN, - page_lsn: 0, - blkno: (i / 100) as u32, - offnum: (i % 100) as u16 + 1, - columns: (0..columns) + .map(|i| { + let mut cells: Vec<_> = (0..shape.columns) .map(|c| Some(ColumnValue::Int4(i as i32 + c as i32))) - .collect(), + .collect(); + if shape.text > 0 { + cells.push(Some(ColumnValue::Text(text.clone()))); + } + BackfillTuple { + rfn: r.rfn, + xid: 100 + i as u32, + xmax: 0, + infomask: 0, + source_lsn: START_LSN, + page_lsn: 0, + blkno: (i / 100) as u32, + offnum: (i % 100) as u16 + 1, + data: TupleData::from_cells(&cells), + pending: None, + } }) .collect(); if tup_tx.send(slab).await.is_err() { @@ -419,8 +496,8 @@ async fn bench_drain(shape: Shape) -> Report { label: "drain".into(), elapsed_secs: elapsed, tuples: outcome.rows_routed, - // Int4 payload only; the interesting rate here is rows, not bytes - bytes: outcome.rows_routed * 4 * shape.columns as u64, + // Row payload only; the interesting rate here is rows, not bytes + bytes: outcome.rows_routed * (4 * shape.columns + shape.text) as u64, detail: vec![ ("seqs", outcome.next_seq as f64), ("allocs", metered.allocs as f64), @@ -429,11 +506,85 @@ async fn bench_drain(shape: Shape) -> Report { } } +/// Backup pages through walk, drain, and batcher to sealed `InsertBatch`es, +/// which drop instead of inserting +async fn bench_load(shape: Shape) -> Report { + let (parts, tuples, _, catalog) = backup(shape).await; + let tables = catalog + .descriptors() + .map(|d| (d.rel_name.clone(), mapping(d.oid, shape))) + .collect(); + let routes = Arc::new(walshadow::mapping::shared(tables)); + let tmp = tempfile::tempdir().unwrap(); + let cfg = BootstrapConfig::new(tmp.path().join("data")); + let progress = cfg.progress.clone(); + let source = Box::new(MemSource { + parts, + parallelism: shape.parallelism, + }); + let emitter = walshadow::ch_emitter::EmitterConfig::default(); + let stats = Arc::new(walshadow::ch_emitter::EmitterStats::default()); + let fatal = walshadow::pipeline::Fatal::new(); + let batcher = walshadow::pipeline::batcher::BatcherConfig { + row_budget: emitter.row_budget, + byte_budget: emitter.byte_budget, + inserters: 1, + flush_timeout: std::time::Duration::from_secs(1), + }; + + let base = GLOBAL.reset_peak(); + let started = Instant::now(); + let (msg_tx, ack, tail) = + walshadow::pipeline::tail::spawn_discard(batcher, stats.clone(), fatal.clone()); + let (rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog.clone(), false); + let drain = bootstrap::drain( + rx, + catalog, + walshadow::emit::route::freeze_routes(&routes, None, &Default::default()), + msg_tx.clone(), + ack.clone(), + stats.clone(), + ToastResolver::disabled(), + bootstrap::Deferral::Rejected, + foldhash::HashSet::default(), + None, + ); + let (outcome, pumped) = tokio::join!(drain, pump); + pumped.unwrap().unwrap(); + let outcome = outcome.unwrap(); + tail.finish(msg_tx, ack, outcome.next_seq, &fatal) + .await + .unwrap(); + let elapsed = started.elapsed().as_secs_f64(); + let metered = GLOBAL.snapshot().since(base); + assert_eq!(outcome.rows_routed, tuples); + assert_eq!( + stats.insertbatch_rows_in.load(Ordering::Relaxed), + tuples, + "every row reached the batcher" + ); + + let mut detail = pump_detail(&progress); + detail.push(( + "batches", + stats.insertbatch_batches_out.load(Ordering::Relaxed) as f64, + )); + detail.push(("allocs", metered.allocs as f64)); + detail.push(("peak_mib", metered.peak as f64 / (1 << 20) as f64)); + Report { + label: "load".into(), + elapsed_secs: elapsed, + tuples, + bytes: progress.pump.bytes_tapped.load(Ordering::Relaxed), + detail, + } +} + fn usage() -> ! { eprintln!( - "usage: bootstrap_pump [--list] [--stage pump|drain|all] [--parts N] \ + "usage: bootstrap_pump [--list] [--stage pump|drain|load|all] [--parts N] \ [--segments-per-part N] [--pages N] [--tuples-per-page N] [--columns N] \ - [--parallelism N]" + [--text N] [--parallelism N]" ); std::process::exit(2) } @@ -460,6 +611,7 @@ fn main() { "--pages" => shape.pages_per_segment = val(&next).max(1), "--tuples-per-page" => shape.tuples_per_page = val(&next).max(1), "--columns" => shape.columns = val(&next).max(1), + "--text" => shape.text = val(&next), "--parallelism" => shape.parallelism = val(&next).max(1), // cargo bench passes --bench through "--bench" => { @@ -473,11 +625,13 @@ fn main() { let rt = tokio::runtime::Runtime::new().unwrap(); println!( - "shape: {} segments x {} pages x {} tuples x {} int4 cols, {} parts, parallelism {}", + "shape: {} segments x {} pages x {} tuples x {} int4 cols + {} text bytes, {} parts, \ + parallelism {}", shape.segments(), shape.pages_per_segment, shape.tuples_per_page, shape.columns, + shape.text, shape.parts, shape.parallelism, ); @@ -492,4 +646,7 @@ fn main() { if stage == "drain" || stage == "all" { rt.block_on(bench_drain(shape)).print(); } + if stage == "load" || stage == "all" { + rt.block_on(bench_load(shape)).print(); + } } diff --git a/benches/spool_read.rs b/benches/spool_read.rs index 9ab0d7f6..e7fdd411 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -15,7 +15,7 @@ use walrus::pg::walparser::RelFileNode; use walrus::pg::walparser::{ BlockLocation, RmId, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, XLogRecordHeader, }; -use walshadow::backup_page_walk::BackfillTuple; +use walshadow::backup_page_walk::{BackfillTuple, TupleData}; use walshadow::ch_emitter::EncodedChunk; use walshadow::emit::route::RouteSnapshot; use walshadow::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; @@ -106,6 +106,14 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let mut spool = DeferredSpool::new(dir.path().join("deferred.spool"), 1 << 20); let base = GLOBAL.reset_peak(); let start = Instant::now(); + let data = |i: usize| { + let cells = [ + Some(ColumnValue::Int4(i as i32)), + Some(ColumnValue::Text(payload.clone())), + ]; + walshadow::heap_decoder::form_block_data(&cells).into_boxed_slice() + }; + let data_len = data(0).len(); for i in 0..args.rows { spool .push(BackfillTuple { @@ -117,10 +125,8 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { page_lsn: 0, blkno: i as u32, offnum: 1, - columns: vec![ - Some(ColumnValue::Int4(i as i32)), - Some(ColumnValue::Text(payload.clone())), - ], + data: TupleData::Heap(data(i)), + pending: None, }) .await?; } @@ -133,11 +139,7 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let mut n = 0; while let Some(t) = reader.next().await? { assert_eq!(t.xid, n as u32); - assert!(matches!( - t.columns.as_slice(), - [Some(ColumnValue::Int4(v)), Some(ColumnValue::Text(s))] - if *v == n as i32 && *s == payload - )); + assert!(matches!(&t.data, TupleData::Heap(b) if b.len() == data_len)); n += 1; } reader.finish().await?; @@ -393,6 +395,8 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { source_lsn: input.input.source_lsn, new: images[0].new.as_ref(), old: None, + new_patch: &[], + old_patch: &[], }; writer.push_views(change, table)?; } diff --git a/plans/INDEX.md b/plans/INDEX.md index 83690ee5..7f21ea2c 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -39,7 +39,7 @@ writes. A documented limitation does not imply code rejects it | [Multi-database loads and metrics](multi_database.md) | Extend heap-page bootstrap and backup loads beyond primary database, attribute metrics | | [Fuzzing](fuzzing.md) | Find parser and schema-transition interactions beyond fixed regressions | | [Performance](performance.md) | Establish healthy sustained WAL baseline with exact reconciliation before tuning | -| [Direct Native encoding](native_encoding.md) | Encode loads and TOAST rows without cells, prove resource gains end to end | +| [Direct Native encoding](native_encoding.md) | Move stash-resolved rows off cells, bound memory across owners, prove resource gains end to end | | [Failover](failover.md) | Fence unplanned promotion and prove remaining archive/restart fault cases | | [Shadow TOAST reclamation](shadow_toast.md) | Keep historical values readable under lag and restart | | [Replay callback](custom_rmgr.md) | Reduce measured command-boundary capture stalls | diff --git a/plans/native_encoding.md b/plans/native_encoding.md index 8f638c68..b23ca990 100644 --- a/plans/native_encoding.md +++ b/plans/native_encoding.md @@ -1,45 +1,23 @@ # Encode PostgreSQL values directly into ClickHouse blocks -Streaming commits plan rows straight from WAL bytes into column chunks, and -sealed Native blocks stay compressed until acknowledgement; see -[architecture](../architecture/README.md#commit-pipeline). Remaining work -removes owned cells from load paths and TOAST rows and proves resource gains end -to end. Follow [shared constraints](coordination.md), +Commits, loads, and gap replay encode rows from tuple bytes or COPY fields into +column chunks, and sealed Native blocks stay compressed until acknowledgement; +see [architecture](../architecture/README.md#commit-pipeline). Remaining work +moves stash-resolved records off owned cells, bounds memory across owners, and +proves resource gains end to end. Follow [shared constraints](coordination.md), [value coercion](value_coercion.md), and [performance acceptance](performance.md) Share view encoders and column buffers with [container codecs](tier2.md) -Container promotion can retain existing raw pending cells in owned adapters; -complete load and TOAST cell removal is not a prerequisite +Container promotion can retain existing raw pending cells in owned adapters -## Encode loads directly +## Stash-resolved records -COPY and heap-page/bootstrap loads still decode every tuple into `ColumnValue` -cells: [page walk](../src/backfill/backup_page_walk.rs) and -[COPY](../src/backfill/copy_backfill.rs) fill `BackfillTuple` slabs, and -[bootstrap drain](../src/emit/pipeline/bootstrap.rs) and -[gap replay](../src/backfill/wal_replay.rs) stage cell rows through -`RowBuffer`, the batcher's second input next to planned segments - -Carry tuple bytes or COPY fields through load queues instead of cells. Read -visibility from tuple headers, which needs no columns. Encode visible rows into -chunks with the shared view encoder; decode cells only where pending rows, -deferred TOAST, and replacement evidence need them, keeping those mechanisms -and staging publication unchanged. Binary COPY fields are big-endian `typsend` -representations: add a field-view decoder beside the heap one, never reuse the -heap byte-copy rule. Then ship load rows as planned segments and delete -`RowChunk`/`RoutedRow` from the batcher - -Version the deferred TOAST spool if its tuple format changes. Rewrite bootstrap -tests that inspect routed cells to read chunk columns - -## Detoast without cells - -Inputs whose images hold external pointers, stash-resolved records, and -directly buffered heaps plan through the owned-cell adapter, since -`detoast_heap` resolves cells. Resolve pointers from tuple views into bounded -scratch, then encode with the view path, preserving size policy, fill modes, -read-ahead transfer, and chunk generation fences. Compressed inline values and -`numeric`/`jsonb` already decode through per-row scratch +Records resolved at commit decode into owned cells in +[transaction buffer](../src/xact/xact_buffer.rs) `fold_raw_ordinary`, then plan +through the cell adapter. Carry them as raw inputs under their resolved +descriptor instead, keeping operation policy and ambiguity fences at fold time. +Resolved TOAST values stay owned cells patching tuple views; read-ahead hands +them over rendered ## Bound memory across owners diff --git a/plans/tier2.md b/plans/tier2.md index 4973f9d1..409cb14f 100644 --- a/plans/tier2.md +++ b/plans/tier2.md @@ -61,8 +61,6 @@ checking lifetimes. Treat this as builder organization, not a throughput claim Keep raw transaction spill unchanged when existing pending-value forms suffice Version transaction or deferred-load formats only if their representation changes -Container encoding need not wait for complete removal of cells from load and -TOAST paths Give the walk its own `CodecError` variants for a rejected array header and an uncovered element type, so a malformed body is distinguishable from a type the @@ -74,9 +72,9 @@ Choose encoding from source type, element codec, and target shape together Admitting an array OID must not enable local encoding for a `String` override, unsupported nested target, or `QBit`. Keep those pairs on oracle until covered -Column-wide admission also affects owned load and TOAST rows. Heap values can -retain raw bodies in existing pending cells; COPY currently casts containers to -text. Support that text form or introduce an explicit oracle fallback before +Column-wide admission also affects resolved TOAST values and COPY rows. +Resolved values retain raw bodies in existing pending cells; COPY casts +containers to text. Support that text form or introduce an explicit oracle fallback before opening gates. Fast defaults need equivalent handling. An unsupported input must not enter a local-only buffer merely because WAL encoding works diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index 56165bff..b64d9032 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -155,7 +155,7 @@ pub fn tap_filenode_set( /// /// ```ignore /// let (rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); -/// let drained = tokio::spawn(drain_backfill(rx, observer)); +/// let drained = tokio::spawn(drain_backfill(rx, &catalog, observer)); /// let outcome = pump.await?.context("bootstrap pump")?; /// let shipped = drained.await?; /// ``` @@ -262,6 +262,7 @@ pub async fn run_greenfield_bootstrap( /// release CH state before the daemon swaps to the shadow-catalog emitter. pub async fn drain_backfill( mut rx: mpsc::Receiver>, + catalog: &CatalogMap, observer: &mut O, ) -> Result { let mut shipped: u64 = 0; @@ -278,7 +279,12 @@ pub async fn drain_backfill( } last_rfn = Some(tuple.rfn); last_lsn = tuple.source_lsn; - let committed = tuple.into_committed_insert(); + let rel = catalog + .get(tuple.rfn.db_node, tuple.rfn.rel_node) + .with_context(|| format!("bootstrap drain: no descriptor for {:?}", tuple.rfn))?; + let committed = tuple + .into_committed_insert(&rel) + .map_err(|e| anyhow::anyhow!("bootstrap drain: {e}"))?; observer .on_tuple(&committed) .await @@ -465,7 +471,9 @@ fn one_char(s: String, what: &str) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::backfill::backup_page_walk::{PAGE_BYTES, make_rel, synth_single_tuple_page}; + use crate::backfill::backup_page_walk::{ + PAGE_BYTES, TupleData, make_rel, synth_single_tuple_page, + }; use crate::backfill::backup_source::{BackupSink, BackupSource, FileMeta}; use crate::decode::heap_decoder::HeapOp; use async_trait::async_trait; @@ -635,11 +643,11 @@ mod tests { let tuple = &tuples[0]; assert_eq!(tuple.source_lsn, 0xDEAD_BEEF); assert_eq!(tuple.xid, 99); - assert_eq!(tuple.columns.len(), 1); - assert!(matches!( - tuple.columns[0], - Some(crate::decode::heap_decoder::ColumnValue::Int4(42)) - )); + let columns = tuple.cells(&make_rel()).unwrap(); + assert_eq!( + columns, + [Some(crate::decode::heap_decoder::ColumnValue::Int4(42))] + ); } #[tokio::test] @@ -662,9 +670,10 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: vec![Some(crate::decode::heap_decoder::ColumnValue::Int4( - v as i32, - ))], + data: TupleData::from_cells(&[Some( + crate::decode::heap_decoder::ColumnValue::Int4(v as i32), + )]), + pending: None, }]) .await .unwrap(); @@ -672,7 +681,9 @@ mod tests { drop(tx); let mut observer = CollectingTupleObserver::default(); - let shipped = drain_backfill(rx, &mut observer).await.unwrap(); + let shipped = drain_backfill(rx, &int4_catalog(), &mut observer) + .await + .unwrap(); assert_eq!(shipped, 3); assert_eq!(observer.tuples.len(), 3); for (i, c) in observer.tuples.iter().enumerate() { @@ -689,6 +700,12 @@ mod tests { } } + fn int4_catalog() -> CatalogMap { + let mut catalog = CatalogMap::new(); + catalog.insert(Arc::new(make_rel())); + catalog + } + #[derive(Default)] struct CountingObserver { on_tuple_calls: u32, @@ -751,9 +768,10 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: vec![Some(crate::decode::heap_decoder::ColumnValue::Int4( - v as i32, - ))], + data: TupleData::from_cells(&[Some( + crate::decode::heap_decoder::ColumnValue::Int4(v as i32), + )]), + pending: None, }]) .await .unwrap(); @@ -761,7 +779,7 @@ mod tests { drop(tx); let mut obs = CountingObserver::default(); - let shipped = drain_backfill(rx, &mut obs).await.unwrap(); + let shipped = drain_backfill(rx, &int4_catalog(), &mut obs).await.unwrap(); assert_eq!(shipped, 4); assert_eq!(obs.on_tuple_calls, 4); // One synthetic xact closes the backfill @@ -783,7 +801,8 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: Vec::new(), + data: TupleData::from_cells(&[Some(crate::decode::heap_decoder::ColumnValue::Int4(1))]), + pending: None, }; let (tx, rx) = mpsc::channel::>(1); tx.send(vec![tuple(16400), tuple(16401)]).await.unwrap(); @@ -792,7 +811,9 @@ mod tests { reject_xact_end: true, ..Default::default() }; - let err = drain_backfill(rx, &mut obs).await.unwrap_err(); + let err = drain_backfill(rx, &int4_catalog(), &mut obs) + .await + .unwrap_err(); assert!(err.to_string().contains("mid-table xact end"), "{err}"); assert_eq!(obs.on_tuple_calls, 1); } @@ -804,7 +825,7 @@ mod tests { let (tx, rx) = mpsc::channel::>(64); drop(tx); let mut obs = CountingObserver::default(); - let shipped = drain_backfill(rx, &mut obs).await.unwrap(); + let shipped = drain_backfill(rx, &int4_catalog(), &mut obs).await.unwrap(); assert_eq!(shipped, 0); assert_eq!(obs.on_tuple_calls, 0); assert_eq!(obs.on_xact_end_calls, 1); diff --git a/src/backfill/backup_backfill.rs b/src/backfill/backup_backfill.rs index dee7cd53..5c4e3034 100644 --- a/src/backfill/backup_backfill.rs +++ b/src/backfill/backup_backfill.rs @@ -1122,6 +1122,7 @@ impl RecordSink for PrescanSink { #[cfg(test)] mod tests { use super::*; + use crate::backfill::backup_page_walk::TupleData; use crate::decode::visibility::{ HEAP_XMAX_INVALID, HEAP_XMAX_IS_MULTI, HEAP_XMIN_COMMITTED, HEAP_XMIN_INVALID, PgMultiXactAccum, PgXactAccum, PgXactView, @@ -1613,7 +1614,8 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: Vec::new(), + data: TupleData::Heap(Box::default()), + pending: None, } } diff --git a/src/backfill/backup_page_walk.rs b/src/backfill/backup_page_walk.rs index 1a764f66..e25f054d 100644 --- a/src/backfill/backup_page_walk.rs +++ b/src/backfill/backup_page_walk.rs @@ -1,5 +1,5 @@ //! Page-walk Tap sink. Decodes user-heap tar entries through the same -//! heap decoder the WAL hot path uses (`decode_block_data`). +//! heap decoder the WAL hot path uses (`view_block_data`). //! See [architecture/bootstrap.md](../../architecture/bootstrap.md). //! //! ## V1 limits @@ -27,12 +27,16 @@ use walrus::pg::walparser::{Oid, RelFileNode}; use crate::backfill::backup_source::{ BackupSink, EntrySink, FileAction, FileKind, FileMeta, StartInfo, }; +use crate::backfill::copy_backfill::CopyRow; use crate::backfill::pg_path::{BaseRelFile, RelFork, parse_base_path}; use crate::backfill::walk_barrier::WalkBarrier; use crate::decode::heap_decoder::{ - ColumnValue, CommittedTuple, DecodeError, DecodedHeap, DecodedTuple, HeapOp, decode_block_data, + ColumnValue, CommittedTuple, DecodeError, DecodedHeap, DecodedTuple, HeapOp, TupleView, + decode_block_data, view_block_data, +}; +use crate::decode::visibility::{ + PendingXids, PgMultiXactAccum, PgXactAccum, PgXactView, SealedPatch, }; -use crate::decode::visibility::{PgMultiXactAccum, PgXactAccum, PgXactView, SealedPatch}; use crate::schema::RelDescriptor; use foldhash::{HashMap, HashMapExt, HashSet}; @@ -80,8 +84,8 @@ pub enum PageWalkError { }, } -/// One decoded tuple from a backup page; a synthetic INSERT at -/// `_lsn = start_lsn`. +/// One tuple from a backup page or COPY; a synthetic INSERT at +/// `_lsn = start_lsn`. Carries row bytes, decoded only at encode #[derive(Debug, Clone)] pub struct BackfillTuple { pub rfn: RelFileNode, @@ -100,39 +104,78 @@ pub struct BackfillTuple { /// ([`crate::toast::ToastRow`]) pub blkno: u32, pub offnum: u16, - /// Attnum-1 indexed, matching `RelDescriptor.attributes` - pub columns: Vec>, + pub data: TupleData, + /// Xids an undecided row waits on, set once it enters a pending table + pub pending: Option, +} + +/// Row bytes of a [`BackfillTuple`] +#[derive(Debug, Clone, PartialEq)] +pub enum TupleData { + /// On-page tuple from `t_infomask2` on: the `block.data` shape heap + /// decode reads + Heap(Box<[u8]>), + Copy(CopyRow), +} + +impl TupleData { + /// Heap image of `values`, see + /// [`form_block_data`](crate::decode::heap_decoder::form_block_data) + #[cfg(any(test, feature = "test-support"))] + pub fn from_cells(values: &[Option]) -> Self { + Self::Heap(crate::decode::heap_decoder::form_block_data(values).into()) + } + + pub fn len(&self) -> usize { + match self { + Self::Heap(b) => b.len(), + Self::Copy(r) => r.bytes().len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } impl BackfillTuple { - pub fn has_mapped_external(&self, mapping: &crate::mapping::TableMapping) -> bool { - mapping.columns.iter().any(|c| { - usize::try_from(i32::from(c.src_attnum) - 1) - .ok() - .and_then(|i| self.columns.get(i)) - .is_some_and(|v| matches!(v, Some(ColumnValue::ExternalToast(_)))) - }) + /// Tuple image in place; `None` for COPY rows + pub fn view<'a>(&'a self, rel: &RelDescriptor) -> Result>, DecodeError> { + match &self.data { + TupleData::Heap(b) => view_block_data(b, rel).map(Some), + TupleData::Copy(_) => Ok(None), + } + } + + /// Owned cells, attnum-1 indexed, for observers and tests + pub fn cells(&self, rel: &RelDescriptor) -> Result>, String> { + match &self.data { + TupleData::Heap(b) => decode_block_data(b, rel) + .map(|t| t.columns) + .map_err(|e| e.to_string()), + TupleData::Copy(row) => crate::backfill::copy_backfill::copy_cells(row, rel), + } } /// start_lsn rides as both source_lsn and commit_lsn so /// ReplacingMergeTree(_lsn) collapses duplicates the WAL decoder /// re-emits for records in [start_lsn, end_lsn] - pub fn into_committed_insert(self) -> CommittedTuple { - CommittedTuple { + pub fn into_committed_insert(self, rel: &RelDescriptor) -> Result { + Ok(CommittedTuple { decoded: DecodedHeap { rfn: self.rfn, xid: self.xid, source_lsn: self.source_lsn, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: self.columns, + columns: self.cells(rel)?, partial: false, }), old: None, }, commit_ts: 0, commit_lsn: self.source_lsn, - } + }) } } @@ -321,19 +364,20 @@ impl<'a> PageWalker<'a> { continue; } let tuple_bytes = &page[lp_off..lp_off + lp_len]; - match decode_on_page_tuple(tuple_bytes, self.rel) { - Some((xid, xmax, infomask, columns)) => { + match on_page_tuple(tuple_bytes, self.rel) { + Some((header, data)) => { out.push(BackfillTuple { rfn: self.rel.rfn, - xid, - xmax, - infomask, + xid: header.xmin, + xmax: header.xmax, + infomask: header.infomask, source_lsn: self.source_lsn, page_lsn, blkno: block_no, // OffsetNumber is 1-based (PG off/itemid.h) offnum: (i + 1) as u16, - columns, + data: TupleData::Heap(data.into()), + pending: None, }); tally.tuples_emitted += 1; } @@ -396,42 +440,60 @@ pub(crate) fn toast_rows_from_page( ) -> Vec { let mut rows = Vec::new(); for offnum in 1..=page_max_offnum(page) { - let Some(tuple) = page_tuple_bytes(page, offnum) else { - continue; - }; - let Some((_, _, _, mut columns)) = decode_on_page_tuple(tuple, rel) else { + let Some((_, data)) = page_tuple_bytes(page, offnum).and_then(|t| on_page_tuple(t, rel)) + else { continue; }; - let Some((chunk_id, chunk_seq, chunk_data)) = - crate::decode::heap_decoder::take_toast_chunk_columns(&mut columns) - else { + let Some(row) = toast_row(data, rel, blkno, offnum, lsn) else { continue; }; - rows.push(crate::toast::ToastRow { - db_oid: rel.rfn.db_node, - toast_relid: rel.oid, - blkno, - offnum, - chunk_id, - chunk_seq, - chunk_data: bytes::Bytes::from(chunk_data), - lsn, - }); + rows.push(row); } rows } -/// Decode one on-page tuple into `(xmin, xmax, infomask, columns)`. +/// Chunk row of one `pg_toast_*` tuple's `block.data` image +pub(crate) fn toast_row( + data: &[u8], + rel: &RelDescriptor, + blkno: u32, + offnum: u16, + lsn: u64, +) -> Option { + let view = view_block_data(data, rel).ok()?; + let (chunk_id, chunk_seq, chunk_data) = + crate::decode::heap_decoder::toast_chunk_datums(&view.datums)?; + Some(crate::toast::ToastRow { + db_oid: rel.rfn.db_node, + toast_relid: rel.oid, + blkno, + offnum, + chunk_id, + chunk_seq, + chunk_data: bytes::Bytes::copy_from_slice(chunk_data), + lsn, + }) +} + +/// `HeapTupleHeaderData` fields the gate reads +#[derive(Clone, Copy, Debug)] +pub(crate) struct OnPageHeader { + pub xmin: u32, + pub xmax: u32, + pub infomask: u16, +} + +/// Split one on-page tuple into its header fields and `block.data` image. /// On-disk shape carries a full `HeapTupleHeaderData` (23 bytes); from /// `t_infomask2` on it matches the `xl_heap_header` (5 bytes) shape PG /// strips into WAL via /// `XLogRegisterBufData(0, tup->t_data + SizeofHeapTupleHeader, ...)`, /// so the shared heap decoder reads it in place. `None` on truncated / -/// malformed header. -pub(crate) fn decode_on_page_tuple( - tuple: &[u8], +/// malformed header or attributes. +pub(crate) fn on_page_tuple<'a>( + tuple: &'a [u8], rel: &RelDescriptor, -) -> Option<(u32, u32, u16, Vec>)> { +) -> Option<(OnPageHeader, &'a [u8])> { use crate::decode::heap_decoder::SIZE_OF_HEAP_TUPLE_HEADER; const T_INFOMASK2_OFF: usize = 18; @@ -443,17 +505,18 @@ pub(crate) fn decode_on_page_tuple( // t_infomask2 18..20 t_infomask 20..22 // t_hoff 22 (offset to user data, 8-byte aligned) // t_bits[] 23.. (NULL bitmap, only if HEAP_HASNULL) - let xmin = u32::from_le_bytes(tuple[0..4].try_into().unwrap()); - let xmax = u32::from_le_bytes(tuple[4..8].try_into().unwrap()); - let t_infomask = u16::from_le_bytes(tuple[20..22].try_into().unwrap()); + let header = OnPageHeader { + xmin: u32::from_le_bytes(tuple[0..4].try_into().unwrap()), + xmax: u32::from_le_bytes(tuple[4..8].try_into().unwrap()), + infomask: u16::from_le_bytes(tuple[20..22].try_into().unwrap()), + }; let t_hoff = tuple[22] as usize; if t_hoff < SIZE_OF_HEAP_TUPLE_HEADER || t_hoff > tuple.len() { return None; } - match decode_block_data(&tuple[T_INFOMASK2_OFF..], rel) { - Ok(d) => Some((xmin, xmax, t_infomask, d.columns)), - Err(_) => None, - } + let data = &tuple[T_INFOMASK2_OFF..]; + crate::decode::heap_decoder::scan_block_data(data, rel).ok()?; + Some((header, data)) } /// Tap router: resolves a segment's descriptor at `begin` and hands back a @@ -1183,6 +1246,14 @@ mod tests { assert_eq!(sink.source_lsn(), 0xABCD_1234); } + /// Leading int4 of a tuple walked against [`make_rel`]'s layout + fn int4(t: &BackfillTuple) -> Option { + match t.cells(&make_rel()).unwrap().first() { + Some(Some(ColumnValue::Int4(v))) => Some(*v), + _ => None, + } + } + #[test] fn page_walker_emits_single_tuple() { let rel = make_rel(); @@ -1194,8 +1265,7 @@ mod tests { assert_eq!(out.len(), 1); assert_eq!(out[0].source_lsn, 0xABCD); assert_eq!(out[0].xid, 99); - assert_eq!(out[0].columns.len(), 1); - assert!(matches!(out[0].columns[0], Some(ColumnValue::Int4(42)))); + assert_eq!(int4(&out[0]), Some(42)); assert_eq!(tally.pages_walked, 1); assert_eq!(tally.tuples_emitted, 1); } @@ -1425,10 +1495,7 @@ mod tests { assert_eq!(captured.len(), 1); assert_eq!(captured[0].source_lsn, 0x1234_5678); assert_eq!(captured[0].xid, 99); - assert!(matches!( - captured[0].columns[0], - Some(ColumnValue::Int4(99)) - )); + assert!(matches!(int4(&captured[0]), Some(99))); assert_eq!(ld(&sink.stats.files_seen), 1); assert_eq!(ld(&sink.stats.files_walked), 1); assert_eq!(ld(&sink.stats.pages_walked), 1); @@ -1692,7 +1759,7 @@ mod tests { assert_eq!(ld(&sink.stats.tail_bytes_dropped), 0); for (i, t) in captured.iter().enumerate() { assert_eq!(t.blkno, i as u32, "page {i} kept its block number"); - assert!(matches!(t.columns[0], Some(ColumnValue::Int4(v)) if v == i as i32)); + assert_eq!(int4(t), Some(i as i32)); } } @@ -1821,18 +1888,15 @@ mod tests { // Each tuple carries its own file's rfn + value; a shared slot // would attribute both to rel B - let mut by_rel: Vec<_> = sink - .captured() - .into_iter() - .map(|t| (t.rfn.rel_node, t.columns)) - .collect(); - by_rel.sort_by_key(|(rel_node, _)| *rel_node); - assert_eq!( - by_rel, - [ - (16400, vec![Some(ColumnValue::Int4(100))]), - (16401, vec![Some(ColumnValue::Int4(200))]), - ] - ); + let captured = sink.captured(); + let mut by_rel: HashMap = HashMap::new(); + for t in &captured { + if let Some(v) = int4(t) { + by_rel.insert(t.rfn.rel_node, v); + } + } + assert_eq!(captured.len(), 2); + assert_eq!(by_rel.get(&16400), Some(&100), "entry A decoded vs rel A"); + assert_eq!(by_rel.get(&16401), Some(&200), "entry B decoded vs rel B"); } } diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index a3413e40..1ceae5d6 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -76,14 +76,17 @@ use walrus::pg::replication::conn::PgConfig; use crate::backfill::backfill_staging::{self, StagingPlan, StagingRel, StagingSession}; use crate::backfill::backfill_types::{BackupRequest, PassContext, PassOutcome}; use crate::backfill::backup_checkpoint::BackupCheckpoint; -use crate::backfill::backup_page_walk::{BOOTSTRAP_TUPLE_CHANNEL_CAP, BackfillTuple, CatalogMap}; +use crate::backfill::backup_page_walk::{ + BOOTSTRAP_TUPLE_CHANNEL_CAP, BackfillTuple, CatalogMap, TupleData, +}; use crate::backfill::opt_in::Backfiller; use crate::backfill::visibility_pending::{self, SharedPendingLedger}; use crate::catalog::shadow_catalog::ShadowCatalog; use crate::config::{DestEmitter, ResolvedConfig}; use crate::decode::codecs::NumericKind; +use crate::decode::datum::ValueRef; use crate::decode::heap_decoder::ColumnValue; -use crate::emit::ch_emitter::{EmitterStats, ToastMode}; +use crate::emit::ch_emitter::{Cell, EmitterStats, Slot, ToastMode}; use crate::emit::pipeline::tail::OwnedTail; use crate::emit::pipeline::{Fatal, bootstrap}; use crate::emit::route::{Routing, freeze_routes, routing}; @@ -430,8 +433,6 @@ enum WireKind { struct ColPlan { attnum: i16, - kind: WireKind, - type_oid: u32, } struct CopyPlan { @@ -464,6 +465,17 @@ fn wire_kind(type_oid: u32) -> Option { }) } +/// Wire form a column selects as; out-of-matrix types cast to `text`. +/// jsonb's send form prefixes a version byte; its text is the document the +/// WAL path renders, so take that instead +fn copy_kind(type_oid: u32) -> WireKind { + wire_kind(type_oid).unwrap_or(match type_oid { + JSONBOID => WireKind::Json, + NUMERICOID => WireKind::NumericText, + _ => WireKind::CastText, + }) +} + fn column_plan(desc: &RelDescriptor) -> CopyPlan { let mut select = String::new(); let mut cols = Vec::with_capacity(desc.attributes.len()); @@ -474,23 +486,16 @@ fn column_plan(desc: &RelDescriptor) -> CopyPlan { continue; } let ident = quote_ident(&a.name); - let (expr, kind) = match wire_kind(a.type_oid) { - Some(k) => (ident, k), - // jsonb's send form prefixes a version byte; its text is the - // document the WAL path renders, so take that instead - None if a.type_oid == JSONBOID => (format!("{ident}::text"), WireKind::Json), - None if a.type_oid == NUMERICOID => (format!("{ident}::text"), WireKind::NumericText), - None => (format!("{ident}::text"), WireKind::CastText), + let expr = if wire_kind(a.type_oid).is_some() { + ident + } else { + format!("{ident}::text") }; if !select.is_empty() { select.push_str(", "); } select.push_str(&expr); - cols.push(ColPlan { - attnum: a.attnum, - kind, - type_oid: a.type_oid, - }); + cols.push(ColPlan { attnum: a.attnum }); } CopyPlan { select, @@ -556,23 +561,14 @@ pub async fn copy_rows_into( let mut slab_bytes = 0usize; while let Some(row) = stream.next().await { let row = row.context("backfill: COPY stream")?; - let mut payload_bytes = 0u64; - let mut columns: Vec> = vec![None; plan.natts]; + let mut fields: Vec> = vec![None; plan.natts]; for (i, cp) in plan.cols.iter().enumerate() { - let raw: Option<&[u8]> = row.try_get(i).context("backfill: COPY field")?; - payload_bytes += raw.map_or(0, |bytes| bytes.len() as u64); - let v = raw - .map(|raw| decode_field(cp.kind, cp.type_oid, raw)) - .transpose() - .map_err(anyhow::Error::msg)? - .unwrap_or(ColumnValue::Null); - columns[(cp.attnum - 1).max(0) as usize] = Some(v); - } - slab_bytes += columns - .iter() - .flatten() - .map(ColumnValue::approx_bytes) - .sum::(); + fields[(cp.attnum - 1).max(0) as usize] = + row.try_get(i).context("backfill: COPY field")?; + } + let payload_bytes = fields.iter().flatten().map(|f| f.len() as u64).sum::(); + let data = CopyRow::new(&fields); + slab_bytes += data.bytes().len(); slab.push(BackfillTuple { rfn: desc.rfn, xid: 0, @@ -583,7 +579,8 @@ pub async fn copy_rows_into( // COPY rows have no on-page TID blkno: 0, offnum: 0, - columns, + data: TupleData::Copy(data), + pending: None, }); rows += 1; stats.backfill_copy_rows.fetch_add(1, Ordering::Relaxed); @@ -684,54 +681,149 @@ fn fixed(raw: &[u8], what: &str) -> Result<[u8; N], String> { .map_err(|_| format!("{what}: expected {N} bytes, got {}", raw.len())) } -fn text_or_bytea(raw: &[u8], wrap: fn(String) -> ColumnValue) -> ColumnValue { - simdutf8::basic::from_utf8(raw) - .map_or_else(|_| ColumnValue::Bytea(raw.to_vec()), |s| wrap(s.into())) +/// One binary COPY row: per attribute, `typsend` bytes or NULL. Laid out +/// `[natts: u16][end: u32 per attribute][field bytes]`, an end's high bit +/// marking NULL +#[derive(Clone, Debug, PartialEq)] +pub struct CopyRow(Box<[u8]>); + +const COPY_NULL: u32 = 1 << 31; + +impl CopyRow { + /// `fields` indexed by attnum - 1 + pub fn new(fields: &[Option<&[u8]>]) -> Self { + let head = 2 + 4 * fields.len(); + let body = fields.iter().flatten().map(|f| f.len()).sum::(); + let mut out = Vec::with_capacity(head + body); + out.extend_from_slice(&(fields.len() as u16).to_le_bytes()); + let mut end = 0u32; + for f in fields { + end += f.map_or(0, |f| f.len() as u32); + let mark = if f.is_none() { COPY_NULL } else { 0 }; + out.extend_from_slice(&(end | mark).to_le_bytes()); + } + for f in fields.iter().flatten() { + out.extend_from_slice(f); + } + Self(out.into()) + } + + /// Spooled bytes, see [`Self::from_bytes`] + pub fn bytes(&self) -> &[u8] { + &self.0 + } + + /// Row [`Self::bytes`] wrote; `None` when the header overruns them + pub fn from_bytes(bytes: Box<[u8]>) -> Option { + let row = Self(bytes); + let natts = row.natts()?; + let last = natts.checked_sub(1).map_or(Some(0), |i| row.end(i))?; + (row.0.len() == 2 + 4 * natts + last as usize).then_some(row) + } + + fn natts(&self) -> Option { + Some(u16::from_le_bytes(self.0.get(..2)?.try_into().ok()?) as usize) + } + + fn end(&self, i: usize) -> Option { + let at = 2 + 4 * i; + let word = u32::from_le_bytes(self.0.get(at..at + 4)?.try_into().ok()?); + Some(word & !COPY_NULL) + } + + /// Field of attribute index `i`; `None` past the row or NULL + pub fn field(&self, i: usize) -> Option<&[u8]> { + let natts = self.natts()?; + if i >= natts { + return None; + } + let at = 2 + 4 * i; + let word = u32::from_le_bytes(self.0[at..at + 4].try_into().ok()?); + if word & COPY_NULL != 0 { + return None; + } + let start = if i == 0 { 0 } else { self.end(i - 1)? }; + let base = 2 + 4 * natts; + self.0 + .get(base + start as usize..base + (word & !COPY_NULL) as usize) + } +} + +fn text_or_bytea<'a>(raw: &'a [u8], wrap: fn(&'a str) -> ValueRef<'a>) -> ValueRef<'a> { + simdutf8::basic::from_utf8(raw).map_or(ValueRef::Bytea(raw), wrap) +} + +/// Value of attribute index `i` in heap-decoder shape; NULL past the row +pub(crate) fn copy_slot<'a>( + row: &'a CopyRow, + rel: &RelDescriptor, + i: usize, +) -> Result, String> { + let (Some(att), Some(raw)) = (rel.attributes.get(i), row.field(i)) else { + return Ok(Slot::Cell(Cell::Value(ValueRef::Null))); + }; + decode_field(copy_kind(att.type_oid), att.type_oid, raw) +} + +/// Owned cells of a COPY row, attnum-1 indexed +pub(crate) fn copy_cells( + row: &CopyRow, + rel: &RelDescriptor, +) -> Result>, String> { + (0..rel.attributes.len()) + .map(|i| copy_slot(row, rel, i).map(|v| Some(owned(v)))) + .collect() +} + +fn owned(slot: Slot<'_>) -> ColumnValue { + match slot { + Slot::Cell(Cell::Value(v)) => v.to_owned_value(), + Slot::Owned(v) => v, + Slot::Cell(_) => unreachable!("COPY fields decode to values"), + } } /// Decode non-NULL wire field into heap-decoder value shape -fn decode_field(kind: WireKind, type_oid: u32, raw: &[u8]) -> Result { - Ok(match kind { - WireKind::Bool => ColumnValue::Bool(fixed::<1>(raw, "bool")?[0] != 0), - WireKind::Char => ColumnValue::Char(fixed::<1>(raw, "char")?[0] as i8), - WireKind::Int2 => ColumnValue::Int2(i16::from_be_bytes(fixed(raw, "int2")?)), - WireKind::Int4 => ColumnValue::Int4(i32::from_be_bytes(fixed(raw, "int4")?)), - WireKind::Int8 => ColumnValue::Int8(i64::from_be_bytes(fixed(raw, "int8")?)), - WireKind::Oid => ColumnValue::Oid(u32::from_be_bytes(fixed(raw, "oid")?)), - WireKind::Oid8 => ColumnValue::Oid8(u64::from_be_bytes(fixed(raw, "oid8")?)), - WireKind::Float4 => ColumnValue::Float4(f32::from_be_bytes(fixed(raw, "float4")?)), - WireKind::Float8 => ColumnValue::Float8(f64::from_be_bytes(fixed(raw, "float8")?)), - WireKind::Date => ColumnValue::Date(i32::from_be_bytes(fixed(raw, "date")?)), - WireKind::Time => ColumnValue::Time(i64::from_be_bytes(fixed(raw, "time")?)), - WireKind::Timestamp => ColumnValue::Timestamp(i64::from_be_bytes(fixed(raw, "timestamp")?)), +fn decode_field(kind: WireKind, type_oid: u32, raw: &[u8]) -> Result, String> { + let v = match kind { + WireKind::Bool => ValueRef::Bool(fixed::<1>(raw, "bool")?[0] != 0), + WireKind::Char => ValueRef::Char(fixed::<1>(raw, "char")?[0] as i8), + WireKind::Int2 => ValueRef::Int2(i16::from_be_bytes(fixed(raw, "int2")?)), + WireKind::Int4 => ValueRef::Int4(i32::from_be_bytes(fixed(raw, "int4")?)), + WireKind::Int8 => ValueRef::Int8(i64::from_be_bytes(fixed(raw, "int8")?)), + WireKind::Oid => ValueRef::Oid(u32::from_be_bytes(fixed(raw, "oid")?)), + WireKind::Oid8 => ValueRef::Oid8(u64::from_be_bytes(fixed(raw, "oid8")?)), + WireKind::Float4 => ValueRef::Float4(f32::from_be_bytes(fixed(raw, "float4")?)), + WireKind::Float8 => ValueRef::Float8(f64::from_be_bytes(fixed(raw, "float8")?)), + WireKind::Date => ValueRef::Date(i32::from_be_bytes(fixed(raw, "date")?)), + WireKind::Time => ValueRef::Time(i64::from_be_bytes(fixed(raw, "time")?)), + WireKind::Timestamp => ValueRef::Timestamp(i64::from_be_bytes(fixed(raw, "timestamp")?)), WireKind::TimestampTz => { - ColumnValue::TimestampTz(i64::from_be_bytes(fixed(raw, "timestamptz")?)) + ValueRef::TimestampTz(i64::from_be_bytes(fixed(raw, "timestamptz")?)) } - WireKind::Uuid => ColumnValue::Uuid(fixed(raw, "uuid")?), - WireKind::Bytea => ColumnValue::Bytea(raw.to_vec()), - WireKind::Text => text_or_bytea(raw, ColumnValue::Text), + WireKind::Uuid => ValueRef::Uuid(fixed(raw, "uuid")?), + WireKind::Bytea => ValueRef::Bytea(raw), + WireKind::Text => text_or_bytea(raw, ValueRef::Text), // Preserve source type for typinput WireKind::CastText => match simdutf8::basic::from_utf8(raw) { - Ok(s) => ColumnValue::PgPendingText { - type_oid, - text: s.to_owned(), - }, - Err(_) => ColumnValue::Bytea(raw.to_vec()), + Ok(text) => ValueRef::PgPendingText { type_oid, text }, + Err(_) => ValueRef::Bytea(raw), }, - WireKind::Name => text_or_bytea(raw, ColumnValue::Name), - WireKind::Json => text_or_bytea(raw, ColumnValue::Json), + WireKind::Name => text_or_bytea(raw, ValueRef::Name), + WireKind::Json => text_or_bytea(raw, ValueRef::Json), // numeric_out text form; specials carry their flag WireKind::NumericText => { let s = simdutf8::basic::from_utf8(raw) .map_err(|_| "numeric::text not utf8".to_string())?; - ColumnValue::Numeric(match s { + return Ok(Slot::Owned(ColumnValue::Numeric(match s { "NaN" => NumericKind::NaN, "Infinity" => NumericKind::PInf, "-Infinity" => NumericKind::NInf, _ => NumericKind::Finite(s.into()), - }) + }))); } - }) + }; + Ok(Slot::Cell(Cell::Value(v))) } // --------------------------------------------------------------------------- @@ -2114,58 +2206,58 @@ mod tests { ); assert_eq!(plan.natts, 5); assert_eq!(plan.cols.len(), 4, "dropped column not selected"); - assert_eq!(plan.cols[0].kind, WireKind::Int8); - assert_eq!(plan.cols[1].kind, WireKind::NumericText); - assert_eq!(plan.cols[2].kind, WireKind::CastText); assert_eq!(plan.cols[2].attnum, 4); + assert_eq!(copy_kind(INT8OID), WireKind::Int8); + assert_eq!(copy_kind(NUMERICOID), WireKind::NumericText); + assert_eq!(copy_kind(1009), WireKind::CastText); // jsonb_out text, the document shape the WAL path yields - assert_eq!(plan.cols[3].kind, WireKind::Json); + assert_eq!(copy_kind(JSONBOID), WireKind::Json); } #[test] fn decode_matches_wal_variants() { assert_eq!( - decode_field(WireKind::Int8, 0, &7i64.to_be_bytes()).unwrap(), + owned(decode_field(WireKind::Int8, 0, &7i64.to_be_bytes()).unwrap()), ColumnValue::Int8(7), ); assert_eq!( - decode_field(WireKind::Bool, 0, &[1]).unwrap(), + owned(decode_field(WireKind::Bool, 0, &[1]).unwrap()), ColumnValue::Bool(true), ); assert_eq!( - decode_field(WireKind::Date, 0, &8000i32.to_be_bytes()).unwrap(), + owned(decode_field(WireKind::Date, 0, &8000i32.to_be_bytes()).unwrap()), ColumnValue::Date(8000), ); assert_eq!( - decode_field(WireKind::Text, 0, b"abc").unwrap(), + owned(decode_field(WireKind::Text, 0, b"abc").unwrap()), ColumnValue::Text("abc".into()), ); assert_eq!( - decode_field(WireKind::Name, 0, b"orders").unwrap(), + owned(decode_field(WireKind::Name, 0, b"orders").unwrap()), ColumnValue::Name("orders".into()), ); assert_eq!( - decode_field(WireKind::Json, 0, b"{\"a\": 1}").unwrap(), + owned(decode_field(WireKind::Json, 0, b"{\"a\": 1}").unwrap()), ColumnValue::Json("{\"a\": 1}".into()), ); assert_eq!( - decode_field(WireKind::NumericText, 0, b"12.50").unwrap(), + owned(decode_field(WireKind::NumericText, 0, b"12.50").unwrap()), ColumnValue::Numeric(NumericKind::Finite("12.50".into())), ); assert_eq!( - decode_field(WireKind::NumericText, 0, b"NaN").unwrap(), + owned(decode_field(WireKind::NumericText, 0, b"NaN").unwrap()), ColumnValue::Numeric(NumericKind::NaN), ); assert_eq!( - decode_field(WireKind::NumericText, 0, b"Infinity").unwrap(), + owned(decode_field(WireKind::NumericText, 0, b"Infinity").unwrap()), ColumnValue::Numeric(NumericKind::PInf), ); assert_eq!( - decode_field(WireKind::NumericText, 0, b"-Infinity").unwrap(), + owned(decode_field(WireKind::NumericText, 0, b"-Infinity").unwrap()), ColumnValue::Numeric(NumericKind::NInf), ); assert_eq!( - decode_field(WireKind::CastText, 1007, b"{1,2}").unwrap(), + owned(decode_field(WireKind::CastText, 1007, b"{1,2}").unwrap()), ColumnValue::PgPendingText { type_oid: 1007, text: "{1,2}".into(), @@ -2175,11 +2267,11 @@ mod tests { assert!(decode_field(WireKind::Int4, 0, &[0, 1]).is_err()); // Invalid UTF-8 degrades to Bytea like the heap decoder assert_eq!( - decode_field(WireKind::Text, 0, &[0xFF, 0xFE]).unwrap(), + owned(decode_field(WireKind::Text, 0, &[0xFF, 0xFE]).unwrap()), ColumnValue::Bytea(vec![0xFF, 0xFE]), ); assert_eq!( - decode_field(WireKind::CastText, 1007, &[0xFF]).unwrap(), + owned(decode_field(WireKind::CastText, 1007, &[0xFF]).unwrap()), ColumnValue::Bytea(vec![0xFF]), ); assert!(decode_field(WireKind::NumericText, 0, &[0xFF]).is_err()); @@ -2295,7 +2387,11 @@ mod tests { ), ] { assert_eq!(wire_kind(oid), Some(kind), "oid {oid}"); - assert_eq!(decode_field(kind, oid, &raw).unwrap(), want, "oid {oid}"); + assert_eq!( + owned(decode_field(kind, oid, &raw).unwrap()), + want, + "oid {oid}" + ); let variable = matches!( kind, WireKind::Bytea | WireKind::Text | WireKind::Name | WireKind::Json diff --git a/src/backfill/spool.rs b/src/backfill/spool.rs index b01adcd3..d8a57c83 100644 --- a/src/backfill/spool.rs +++ b/src/backfill/spool.rs @@ -21,15 +21,17 @@ use std::path::PathBuf; use tokio::fs::OpenOptions; use tokio::io::AsyncSeekExt; -use crate::backfill::backup_page_walk::BackfillTuple; +use crate::backfill::backup_page_walk::{BackfillTuple, TupleData}; +use crate::backfill::copy_backfill::CopyRow; +use crate::decode::visibility::PendingXids; use crate::xact::spill::{ - ChunkReader, ChunkWriter, Cursor, IO_CHUNK, SpillError, decode_value, encode_value, push_u8, - push_u16, push_u32, push_u64, + ChunkReader, ChunkWriter, Cursor, IO_CHUNK, SpillError, push_u8, push_u16, push_u32, push_u64, }; use walrus::pg::walparser::RelFileNode; const SPOOL_MAGIC: [u8; 2] = *b"WD"; -const SPOOL_VERSION: u16 = 2; +/// v3 carries tuple bytes instead of decoded cells +const SPOOL_VERSION: u16 = 3; /// Default in-memory prefix budget before records spill to file pub const DEFERRED_SPOOL_MEM_MAX: usize = 8 << 20; @@ -468,15 +470,12 @@ fn truncated() -> SpillError { } pub(crate) fn approx_bytes(value: &BackfillTuple) -> usize { - std::mem::size_of::() - + value - .columns - .iter() - .flatten() - .map(crate::decode::heap_decoder::ColumnValue::approx_bytes) - .sum::() + std::mem::size_of::() + value.data.len() } +const DATA_HEAP: u8 = 0; +const DATA_COPY: u8 = 1; + fn encode_record(value: &BackfillTuple, out: &mut Vec) { push_u32(out, value.rfn.spc_node); push_u32(out, value.rfn.db_node); @@ -488,16 +487,21 @@ fn encode_record(value: &BackfillTuple, out: &mut Vec) { push_u64(out, value.page_lsn); push_u32(out, value.blkno); push_u16(out, value.offnum); - push_u32(out, value.columns.len() as u32); - for col in &value.columns { - match col { - None => push_u8(out, 0), - Some(v) => { - push_u8(out, 1); - encode_value(out, v); - } + match value.pending { + None => push_u8(out, 0), + Some(p) => { + push_u8(out, 1); + push_u32(out, p.insert); + push_u32(out, p.delete); } } + let (tag, bytes) = match &value.data { + TupleData::Heap(b) => (DATA_HEAP, &b[..]), + TupleData::Copy(r) => (DATA_COPY, r.bytes()), + }; + push_u8(out, tag); + push_u32(out, bytes.len() as u32); + out.extend_from_slice(bytes); } fn decode_record(buf: &[u8]) -> Result { @@ -514,14 +518,33 @@ fn decode_record(buf: &[u8]) -> Result { let page_lsn = c.u64()?; let blkno = c.u32()?; let offnum = c.u16()?; - let ncols = c.u32()? as usize; - let mut columns = Vec::with_capacity(ncols); - for _ in 0..ncols { - columns.push(match c.u8()? { - 0 => None, - _ => Some(decode_value(c)?), - }); - } + let pending = match c.u8()? { + 0 => None, + _ => Some(PendingXids { + insert: c.u32()?, + delete: c.u32()?, + }), + }; + let tag = c.u8()?; + let len = c.u32()? as usize; + let bytes: Box<[u8]> = c.need(len)?.into(); + let data = match tag { + DATA_HEAP => TupleData::Heap(bytes), + DATA_COPY => { + TupleData::Copy( + CopyRow::from_bytes(bytes).ok_or_else(|| SpillError::Format { + offset: 0, + detail: "malformed COPY row".into(), + })?, + ) + } + tag => { + return Err(SpillError::Format { + offset: 0, + detail: format!("unknown tuple data tag {tag}"), + }); + } + }; Ok(BackfillTuple { rfn, xid, @@ -531,7 +554,8 @@ fn decode_record(buf: &[u8]) -> Result { page_lsn, blkno, offnum, - columns, + data, + pending, }) } @@ -635,14 +659,43 @@ mod tests { page_lsn: lsn + 0x100, blkno: 3, offnum: 7, - columns: vec![ + data: TupleData::from_cells(&[ Some(ColumnValue::Int4(1)), None, Some(ColumnValue::Bytea(payload.to_vec())), - ], + ]), + pending: None, } } + /// COPY rows and pending xids survive the file round trip + #[tokio::test] + async fn file_records_keep_copy_rows_and_pending_xids() { + let dir = tempdir().unwrap(); + let mut spool = DeferredSpool::new(dir.path().join("copy.bin"), 0); + let row = CopyRow::new(&[Some(&7i32.to_be_bytes()[..]), None, Some(b"abc")]); + let mut copy = tuple(1, b""); + copy.data = TupleData::Copy(row.clone()); + let mut pending = tuple(2, b"xyz"); + pending.pending = Some(PendingXids { + insert: 11, + delete: 12, + }); + spool.push(copy).await.unwrap(); + spool.push(pending.clone()).await.unwrap(); + assert_eq!(spool.resident_bytes(), 0); + let out = drain_all(spool).await; + assert_eq!(out[0].data, TupleData::Copy(row.clone())); + assert_eq!(out[0].pending, None); + assert_eq!(out[1].data, pending.data); + assert_eq!(out[1].pending, pending.pending); + assert_eq!(row.field(0), Some(&7i32.to_be_bytes()[..])); + assert_eq!( + (row.field(1), row.field(2), row.field(3)), + (None, Some(&b"abc"[..]), None) + ); + } + #[tokio::test] async fn checkpoint_then_reopen_keeps_every_record() { let dir = tempdir().unwrap(); @@ -781,7 +834,7 @@ mod tests { let out = drain_all(spool).await; assert_eq!(out.len(), 10); assert_eq!(out[3].source_lsn, 0x1003); - assert_eq!(out[3].columns[2], Some(ColumnValue::Bytea(b"abc".to_vec()))); + assert_eq!(out[3].data, tuple(0, b"abc").data); } #[tokio::test(flavor = "current_thread")] diff --git a/src/backfill/visibility_gate.rs b/src/backfill/visibility_gate.rs index 88b82eb8..541d4c0a 100644 --- a/src/backfill/visibility_gate.rs +++ b/src/backfill/visibility_gate.rs @@ -540,7 +540,7 @@ pub async fn resolve_greenfield( mod tests { use super::*; use crate::backfill::backup_page_walk::{ - PAGE_BYTES, PageWalkSink, make_rel, make_rel_named, synth_single_tuple_page, + PAGE_BYTES, PageWalkSink, TupleData, make_rel, make_rel_named, synth_single_tuple_page, }; use crate::backfill::backup_source::testing::expect_tap; use crate::backfill::backup_source::{BackupSink, FileMeta, StartInfo}; @@ -573,7 +573,8 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: Vec::new(), + data: TupleData::Heap(Box::default()), + pending: None, } } @@ -852,7 +853,7 @@ mod tests { }; let row = |id| BackfillTuple { - columns: vec![Some(ColumnValue::Int4(id))], + data: TupleData::from_cells(&[Some(ColumnValue::Int4(id))]), ..tuple(100, 0, HEAP_XMIN_COMMITTED) }; let mut tails = Vec::new(); @@ -883,8 +884,11 @@ mod tests { for (want_seq, (ack, ack_task, mut msg_rx)) in [3u64, 7].into_iter().zip(tails) { let mut seqs = Vec::new(); while let Some(msg) = msg_rx.recv().await { - let BatcherMsg::Rows(c) = msg else { panic!() }; - seqs.extend(c.rows.iter().map(|r| r.seq)); + let BatcherMsg::Segments(segments) = msg else { + panic!() + }; + let rows = crate::emit::pipeline::batcher::segment_rows(&segments); + seqs.extend(rows.iter().map(|r| r.0)); } assert_eq!(seqs, vec![want_seq; 2], "rows rode their own lane's tail"); drop(ack); @@ -1001,9 +1005,18 @@ mod tests { tees: Vec::new(), }; let row = |value, infomask| BackfillTuple { - columns: vec![Some(ColumnValue::Int4(1)), Some(value)], + data: TupleData::from_cells(&[Some(ColumnValue::Int4(1)), Some(value)]), ..tuple(100, 0, infomask) }; + let has_mapped_external = |t: &BackfillTuple| { + let cells = t.cells(&desc).unwrap(); + mapping.columns.iter().any(|c| { + matches!( + cells[c.src_attnum as usize - 1], + Some(ColumnValue::ExternalToast(_)) + ) + }) + }; let (stats, passed) = run_stream_phase( &catalog, vec![ @@ -1015,8 +1028,8 @@ mod tests { .await; assert_eq!(stats.gated, 1); assert_eq!(passed.len(), 2); - assert!(!passed[0].has_mapped_external(&mapping)); - assert_eq!(passed[1].has_mapped_external(&mapping), body_mapped); + assert!(!has_mapped_external(&passed[0])); + assert_eq!(has_mapped_external(&passed[1]), body_mapped); } } diff --git a/src/backfill/visibility_pending.rs b/src/backfill/visibility_pending.rs index 1415bd1d..f04036ec 100644 --- a/src/backfill/visibility_pending.rs +++ b/src/backfill/visibility_pending.rs @@ -159,17 +159,22 @@ fn pending_table_mapping( } } -fn inject_metadata(tuple: &mut BackfillTuple, rel: &RelDescriptor, pending: PendingXids) { +/// Values a pending row adds past `rel`'s attributes, see +/// [`pending_table_mapping`] +pub(crate) fn metadata_patch( + tuple: &BackfillTuple, + rel: &RelDescriptor, + out: &mut Vec<(usize, ColumnValue)>, +) { + let Some(pending) = tuple.pending else { + return; + }; let base = metadata_base(rel); - let infomask = tuple.infomask as i16; - debug_assert!( - tuple.columns.len() <= base, - "walk indexes columns by attnum-1 against this descriptor", - ); - tuple.columns.resize(base, None); - tuple.columns.push(Some(ColumnValue::Oid(pending.insert))); - tuple.columns.push(Some(ColumnValue::Oid(pending.delete))); - tuple.columns.push(Some(ColumnValue::Int2(infomask))); + out.extend([ + (base, ColumnValue::Oid(pending.insert)), + (base + 1, ColumnValue::Oid(pending.delete)), + (base + 2, ColumnValue::Int2(tuple.infomask as i16)), + ]); } #[derive(Debug)] @@ -223,7 +228,7 @@ impl PendingSpool { } tally.start_lsn = tally.start_lsn.min(tuple.source_lsn); tally.rows += 1; - inject_metadata(&mut tuple, &rel, pending); + tuple.pending = Some(pending); self.spool .push(tuple) .await @@ -936,7 +941,7 @@ fn xid_list(xids: &[u32]) -> String { #[cfg(test)] mod tests { use super::*; - use crate::backfill::backup_page_walk::make_rel_named; + use crate::backfill::backup_page_walk::{TupleData, make_rel_named}; fn manifest(rel: &str, xids: Vec) -> PendingManifest { PendingManifest { @@ -986,7 +991,7 @@ mod tests { let pending = pending_table_mapping(&m, &desc, &rel); assert_eq!(pending.target.table, "t__wspending"); - let mut tuple = BackfillTuple { + let tuple = BackfillTuple { rfn: desc.rfn, xid: 100, xmax: 200, @@ -995,26 +1000,40 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: vec![Some(ColumnValue::Int4(1))], - }; - inject_metadata( - &mut tuple, - &desc, - PendingXids { + data: TupleData::from_cells(&[Some(ColumnValue::Int4(1))]), + pending: Some(PendingXids { insert: 100, delete: 200, - }, + }), + }; + let mut patch = Vec::new(); + metadata_patch(&tuple, &desc, &mut patch); + let route = RouteSnapshot::freeze(Arc::new(pending), Arc::default(), Default::default()); + let view = tuple.view(&desc).unwrap().unwrap(); + let stamp = crate::emit::ch_emitter::RowStamp { + source_lsn: tuple.source_lsn, + xid: tuple.xid, + commit_ts: 0, + op_code: crate::emit::ch_emitter::OP_INSERT, + leaf: None, + }; + let mut chunks = crate::emit::pipeline::plan_spool::ChunkBuilder::new(1, usize::MAX); + chunks + .append((), &desc, &route, |enc| { + enc.append_view(&desc, &view.datums, &patch, &stamp) + }) + .unwrap(); + let chunks = chunks.take().unwrap(); + // UInt32 xids and UInt16 infomask read back as same-width ints + assert_eq!( + chunks[0].1.cells(), + [[ + ColumnValue::Int4(1), + ColumnValue::Int4(100), + ColumnValue::Int4(200), + ColumnValue::Int2(0x0100), + ]], ); - for c in &pending.columns[1..] { - let idx = (c.src_attnum - 1) as usize; - let got = tuple.columns[idx].as_ref().expect("metadata value"); - let want = match c.target_name.as_str() { - XMIN_COLUMN => ColumnValue::Oid(100), - XMAX_COLUMN => ColumnValue::Oid(200), - _ => ColumnValue::Int2(0x0100), - }; - assert_eq!(got, &want, "{}", c.target_name); - } } #[tokio::test] diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index ea7203a3..0e8c2e02 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -16,8 +16,8 @@ use crate::budget::{MemoryBudget, MemoryPermit, acquire_opt}; use crate::catalog::desc_log::{DescriptorLog, DescriptorLogs}; use crate::config::ResolvedConfig; use crate::decode::heap_decoder::{ - CommittedTuple, XLH_DELETE_IS_SUPER, XLOG_HEAP_DELETE, XLOG_HEAP_HOT_UPDATE, XLOG_HEAP_INSERT, - XLOG_HEAP_OPMASK, XLOG_HEAP_UPDATE, take_key_change_tombstone, + ColumnValue, DescribedHeap, HeapOp, XLH_DELETE_IS_SUPER, XLOG_HEAP_DELETE, + XLOG_HEAP_HOT_UPDATE, XLOG_HEAP_INSERT, XLOG_HEAP_OPMASK, XLOG_HEAP_UPDATE, view_heap_record, }; use crate::decode::visibility::PgXactPatch; use crate::decode::wal_xact::{ @@ -26,7 +26,10 @@ use crate::decode::wal_xact::{ }; use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::ack::{AckHandle, BufferedRowSender, OpenSeq, Publish, SeqAlloc}; -use crate::emit::pipeline::batcher::{BatcherClosed, BatcherMsg, RowBuffer}; +use crate::emit::pipeline::batcher::{ + BatcherClosed, BatcherMsg, DECODE_CHUNK_BYTES, PlacedSegment, +}; +use crate::emit::pipeline::plan_spool::{ChunkBuilder, ViewChange}; use crate::emit::route::{RouteSnapshot, freeze_routes}; use crate::mapping::MappingSnapshot; use crate::pos::Pos; @@ -34,10 +37,10 @@ use crate::record::{Record, RecordSink, SegmentSink, SinkError, WAL_SEG_SIZE}; use crate::schema::{FIRST_NORMAL_OBJECT_ID, RelDescriptor}; use crate::source::wal_stream::WalStream; use crate::toast::{ChunkRefMap, ToastResolver, ToastRow, ToastRowRef}; -use crate::xact::spill::BodySpoolFile; +use crate::xact::spill::{BodySpoolFile, DescribedInput}; use crate::xact::xact_buffer::{ - BufferingDecoderSink, DrainEntry, DrainedBatch, SubxactTracker, WalkStep, XactBuffer, - detoast_heap, heap_reads_toast, resolve_stash, + BufferingDecoderSink, DrainEntry, DrainHeap, DrainedBatch, SubxactTracker, WalkStep, + XactBuffer, detoast_heap, detoast_sites, external_sites, heap_reads_toast, resolve_stash, }; use foldhash::{HashMap, HashSet}; @@ -182,8 +185,11 @@ pub struct WalReplaySink { batch_rows: usize, batch_bytes: usize, msg_tx: mpsc::Sender, - /// Rows not yet on the batcher channel, shipped with held seq events - row_buffer: RowBuffer, + /// Open commit's encoded rows, keyed by descriptor identity + chunks: ChunkBuilder, + /// Segments not yet on the batcher channel, shipped with held seq events + staged: Vec, + staged_rows: usize, patch: Option>>, seqs: SeqAlloc, /// Current seq and its routed rows, registered on first row @@ -212,6 +218,21 @@ fn tail_closed(_: BatcherClosed) -> SinkError { SinkError::Other("wal_replay: tail closed".into()) } +fn encode_err(e: crate::ch::EmitterError) -> SinkError { + SinkError::Other(format!("wal_replay: encode: {e}")) +} + +/// Rows staged before a channel hop +const STAGED_ROWS: usize = 65536; + +/// Commit a drained batch belongs to, with its TOAST sources +struct Commit<'a> { + ts: i64, + lsn: u64, + spool: Option<&'a BodySpoolFile>, + ref_maps: &'a [&'a ChunkRefMap], +} + impl WalReplaySink { pub fn new(inputs: WalReplayInputs) -> Self { let pending_cap = inputs @@ -257,7 +278,9 @@ impl WalReplaySink { batch_rows: inputs.batch_rows, batch_bytes: inputs.batch_bytes, msg_tx: inputs.msg_tx, - row_buffer: RowBuffer::default(), + chunks: ChunkBuilder::new(STAGED_ROWS, DECODE_CHUNK_BYTES), + staged: Vec::new(), + staged_rows: 0, patch: inputs.patch, seqs: inputs.ack.seqs(inputs.next_seq).buffer_events(), open: None, @@ -299,7 +322,7 @@ impl WalReplaySink { } /// Transactions needing replay below `lsn` to rebuild buffered prefix - pub async fn xacts_opened_below(&self, lsn: u64) -> Vec<(u32, u64)> { + pub async fn xacts_opened_below(&mut self, lsn: u64) -> Vec<(u32, u64)> { self.buffer .lock() .await @@ -327,7 +350,7 @@ impl WalReplaySink { /// Lowest first-record LSN still buffered. A resume above it would drop /// the prefix of a transaction that commits later - pub async fn oldest_inflight(&self) -> Option { + pub async fn oldest_inflight(&mut self) -> Option { self.buffer .lock() .await @@ -344,19 +367,59 @@ impl WalReplaySink { Ok(self.stats()) } - async fn emit( + /// Seq of the open commit, registered on its first routed row; counts + /// `rows` against it + fn place_rows(&mut self, commit_lsn: u64, rows: usize) -> u64 { + let (open, placed) = self + .open + .get_or_insert_with(|| (self.seqs.open(commit_lsn, Publish::Commit), 0)); + *placed += rows as u64; + open.seq() + } + + /// Stage encoded rows as a segment of `seq` once due, or at once when a + /// leaf permit rides them: ship it before another acquisition can + /// wait on it + async fn encoded( &mut self, seq: u64, - table: (&Arc, &Arc), - committed: CommittedTuple, - value_permit: Option>, + permit: Option, ) -> std::result::Result<(), SinkError> { - if self.row_buffer.stage(seq, table, committed, value_permit) { + if permit.is_none() && !self.chunks.due() { + return Ok(()); + } + let ship = permit.is_some(); + self.stage(seq, permit.map(Arc::new))?; + if ship || self.staged_rows >= STAGED_ROWS { self.flush_pending().await?; } Ok(()) } + fn stage( + &mut self, + seq: u64, + permit: Option>, + ) -> std::result::Result<(), SinkError> { + let chunks: Vec<_> = self + .chunks + .take() + .map_err(encode_err)? + .into_iter() + .map(|(_, c)| c) + .collect(); + if chunks.is_empty() { + return Ok(()); + } + self.staged_rows += chunks.iter().map(|c| c.rows()).sum::(); + self.staged.push(PlacedSegment { + seq, + chunks, + permit, + }); + Ok(()) + } + /// Mirror all chunks from restored TOAST page image. Bypass transaction /// stash because neighboring tuples may belong to other transactions. /// @@ -538,7 +601,11 @@ impl WalReplaySink { } drain.finish().await.map_err(SinkError::from)?; if let Some((seq, rows)) = self.open.take() { + self.stage(seq.seq(), None)?; self.seqs.place(seq, rows); + if self.staged_rows >= STAGED_ROWS { + self.flush_pending().await?; + } } self.subxact_tracker.forget_tree(xid); Ok(()) @@ -582,77 +649,44 @@ impl WalReplaySink { .map_err(SinkError::ctx("toast rewrite barrier"))?; } WalkStep::Heap(drained) => { - for mut heap in drained.into_cells().map_err(SinkError::from)? { - let rfn = heap.decoded.rfn; - // Decode TOAST chunks, route through parent row - let Some(target) = targets.get(&(rfn.db_node, rfn.rel_node)) else { - continue; - }; - if commit_lsn <= self.from_lsn { - // Walked pages cover commits through from_lsn - self.replay.commits_below_from += 1; - continue; - } - if commit_lsn > target.through { - self.replay.commits_past_through += 1; - continue; - } - let Some(route) = &target.route else { - self.stats - .unsupported_relations - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - }; - // Skip deletes for append-only destinations - if route.drops_deletes() - && matches!( - heap.decoded.op, - crate::decode::heap_decoder::HeapOp::Delete - ) - { - self.stats - .deletes_discarded - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - continue; - } - let rel = &target.rel; - // Make earlier commits available before resolving values. - // Inline-only heaps read nothing: flushing for them - // shreds the batch on a row-heavy segment. Gate stays - // as wide as detoast's leaf acquire: narrowing it to - // store-bound pointers would hold pending leaf permits - // across that acquire, which can wait on them - if heap_reads_toast(&heap) { - self.flush_rows().await?; + let rel = drained.descriptor().clone(); + let rfn = match &drained { + DrainHeap::Cells(h) => h.decoded.rfn, + DrainHeap::Input(_) => rel.rfn, + }; + let rows = drained.rows() as u64; + // Decode TOAST chunks, route through parent row + let Some(target) = targets.get(&(rfn.db_node, rfn.rel_node)) else { + continue; + }; + if commit_lsn <= self.from_lsn { + // Walked pages cover commits through from_lsn + self.replay.commits_below_from += rows; + continue; + } + if commit_lsn > target.through { + self.replay.commits_past_through += rows; + continue; + } + let Some(route) = &target.route else { + self.stats + .unsupported_relations + .fetch_add(rows, std::sync::atomic::Ordering::Relaxed); + continue; + }; + let at = Commit { + ts: commit_ts, + lsn: commit_lsn, + spool, + ref_maps: &ref_maps, + }; + match drained { + DrainHeap::Input(input) => { + Box::pin(self.replay_views(&input, &rel, route, &at)).await?; } - let value_permit = - detoast_heap(&mut heap, spool, &ref_maps, &self.resolver) - .await - .map_err(SinkError::from)?; - let (open, rows) = self.open.get_or_insert_with(|| { - (self.seqs.open(commit_lsn, Publish::Commit), 0) - }); - *rows += 1; - let seq = open.seq(); - if !route.drops_deletes() - && let Some(tomb) = take_key_change_tombstone(&mut heap.decoded, rel) - { - *rows += 1; - let tomb = CommittedTuple { - decoded: tomb, - commit_ts, - commit_lsn, - }; - self.emit(seq, (rel, route), tomb, None).await?; + DrainHeap::Cells(heap) => { + Box::pin(self.replay_cells(heap, route, &at)).await?; } - let committed = CommittedTuple { - decoded: heap.decoded, - commit_ts, - commit_lsn, - }; - self.emit(seq, (rel, route), committed, value_permit.map(Arc::new)) - .await?; - self.replay.rows_replayed += 1; } } } @@ -660,6 +694,102 @@ impl WalReplaySink { Ok(()) } + /// Whether `op` lands; append-only destinations drop deletes + fn lands(&self, route: &RouteSnapshot, op: HeapOp) -> bool { + let dropped = route.drops_deletes() && op == HeapOp::Delete; + if dropped { + self.stats + .deletes_discarded + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + !dropped + } + + /// Encode an input's tuple images in place, external values resolved + /// into per-image patches + async fn replay_views( + &mut self, + input: &DescribedInput, + rel: &Arc, + route: &Arc, + at: &Commit<'_>, + ) -> std::result::Result<(), SinkError> { + let lsn = input.input.source_lsn; + let images = view_heap_record(input.input.record_bytes(), rel) + .map_err(|e| SinkError::Other(format!("wal_replay: decode at {lsn:#X}: {e}")))?; + for h in &images { + if !self.lands(route, h.op) { + continue; + } + let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); + let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); + let mut permit = None; + if !sites.is_empty() { + // Make earlier commits available before resolving values. + // Inline-only heaps read nothing: flushing for them shreds + // the batch on a row-heavy segment + self.flush_rows().await?; + let db = rel.rfn.db_node; + let (values, leaf) = + detoast_sites(&sites, rel, db, lsn, at.spool, at.ref_maps, &self.resolver) + .await + .map_err(SinkError::from)?; + for (site, v) in sites.iter().zip(values) { + patches[site.image].push((site.idx, v)); + } + permit = leaf; + } + let change = ViewChange { + op: h.op, + xid: input.input.xid, + source_lsn: lsn, + new: h.new.as_ref(), + old: h.old.as_ref(), + new_patch: &patches[0], + old_patch: &patches[1], + }; + let key = Arc::as_ptr(rel) as usize; + let rows = self + .chunks + .push_views(key, rel, route, change, at.ts) + .map_err(encode_err)?; + let seq = self.place_rows(at.lsn, rows); + self.replay.rows_replayed += 1; + self.encoded(seq, permit).await?; + } + Ok(()) + } + + /// [`Self::replay_views`] for owned cells + async fn replay_cells( + &mut self, + mut heap: DescribedHeap, + route: &Arc, + at: &Commit<'_>, + ) -> std::result::Result<(), SinkError> { + if !self.lands(route, heap.decoded.op) { + return Ok(()); + } + // Gate stays as wide as detoast's leaf acquire: narrowing it to + // store-bound pointers would hold pending leaf permits across that + // acquire, which can wait on them + if heap_reads_toast(&heap) { + self.flush_rows().await?; + } + let permit = detoast_heap(&mut heap, at.spool, at.ref_maps, &self.resolver) + .await + .map_err(SinkError::from)?; + let rel = &heap.descriptor; + let key = Arc::as_ptr(rel) as usize; + let rows = self + .chunks + .push_cells(key, rel, route, heap.decoded, at.ts) + .map_err(encode_err)?; + let seq = self.place_rows(at.lsn, rows); + self.replay.rows_replayed += 1; + self.encoded(seq, permit).await + } + async fn handle(&mut self, record: &Record<'_>) -> std::result::Result<(), SinkError> { let rm = record.parsed.header.resource_manager_id; // Capture TOAST page images from Heap and XLOG resource managers @@ -733,10 +863,15 @@ impl BufferedRowSender for WalReplaySink { } async fn send_rows(&mut self) -> std::result::Result<(), SinkError> { - self.row_buffer - .flush(&self.msg_tx) + if self.staged.is_empty() { + return Ok(()); + } + self.staged_rows = 0; + let staged = std::mem::take(&mut self.staged); + self.msg_tx + .send(BatcherMsg::Segments(staged)) .await - .map_err(tail_closed) + .map_err(|_| tail_closed(BatcherClosed)) } } @@ -750,6 +885,7 @@ impl RecordSink for WalReplaySink { return Ok(()); }; // Failed commit's seq stays unplaced, pinning ack below it + let _ = self.chunks.take(); self.flush_pending().await?; Err(e) }) @@ -1611,7 +1747,10 @@ mod tests { abort.parsed.main_data = std::borrow::Cow::Owned(vec![0u8; 10]); assert!(sink.on_record(&abort).await.is_err()); let shipped = msg_rx.try_recv(); - assert!(matches!(&shipped, Ok(BatcherMsg::Rows(chunk)) if chunk.rows.len() == 1)); + assert!(matches!( + &shipped, + Ok(BatcherMsg::Segments(s)) if crate::emit::pipeline::batcher::segment_rows(s).len() == 1 + )); // Rows staged past a closed tail fail the replay buffer.lock().await.on_heap(insert(903, 30)).await.unwrap(); diff --git a/src/bin/stream/bootstrap.rs b/src/bin/stream/bootstrap.rs index 14f31ca7..aa4dc985 100644 --- a/src/bin/stream/bootstrap.rs +++ b/src/bin/stream/bootstrap.rs @@ -983,7 +983,8 @@ pub(crate) async fn run_bootstrap( } else { // Metrics-only skips destination convergence let mut observer = MetricsTupleObserver::default(); - let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &mut observer), pump); + let (drain_res, pump_res) = + tokio::join!(drain_backfill(rx, &drain_catalog, &mut observer), pump); let shipped = drain_res.context("bootstrap drain")?; let outcome: BootstrapOutcome = pump_res .context("bootstrap pump join")? diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index f7146713..a67d6def 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -455,7 +455,7 @@ impl<'a> TupleBuild<'a> for DecodedTuple { /// Tuple image borrowing its record. Indexed by attnum-1 like /// [`DecodedTuple::columns`]; inline up to 32 attributes -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct TupleView<'a> { pub datums: SmallVec<[DatumRef<'a>; 32]>, pub partial: bool, @@ -698,7 +698,8 @@ fn decode_multi_insert<'a, T: TupleBuild<'a>>( have: data.len().saturating_sub(cursor), }); } - let tup = walk_tuple(&data[..cursor + datalen], header, cursor, rel, 0)?; + let mut tup = TupleBuild::with_capacity(rel.attributes.len()); + walk_tuple(&data[..cursor + datalen], header, cursor, rel, 0, &mut tup)?; cursor += datalen; out.push(HeapImages { op: HeapOp::Insert, @@ -830,6 +831,95 @@ pub fn view_block_data<'a>( decode_tuple_payload(block_data, 0, rel, 0) } +/// [`view_block_data`] refilling `out`, sparing a bulk loop the return copy +pub fn view_block_data_into<'a>( + block_data: &'a [u8], + rel: &RelDescriptor, + out: &mut TupleView<'a>, +) -> Result<(), DecodeError> { + out.datums.clear(); + out.partial = false; + decode_tuple_payload_into(block_data, 0, rel, 0, out) +} + +/// Validate a `block.data`-shaped buffer without building attributes +pub fn scan_block_data(block_data: &[u8], rel: &RelDescriptor) -> Result { + decode_tuple_payload(block_data, 0, rel, 0) +} + +/// `block.data` image of `values` as `heap_fill_tuple` lays them out, each +/// stored per its variant's PostgreSQL type. Covers fixed-width, text, +/// bytea and external pointer cells +#[cfg(any(test, feature = "test-support"))] +pub fn form_block_data(values: &[Option]) -> Vec { + let natts = values.len(); + let has_null = values + .iter() + .any(|v| matches!(v, None | Some(ColumnValue::Null))); + let bitmap_len = if has_null { natts.div_ceil(8) } else { 0 }; + let hoff = (SIZE_OF_HEAP_TUPLE_HEADER + bitmap_len).next_multiple_of(8); + let mut bitmap = vec![0u8; bitmap_len]; + let mut data = Vec::new(); + let varlena = |body: &[u8]| { + if body.len() < 127 { + let mut v = vec![(((body.len() + 1) << 1) | 1) as u8]; + v.extend_from_slice(body); + (v, 1) + } else { + let mut v = (((body.len() + 4) as u32) << 2).to_le_bytes().to_vec(); + v.extend_from_slice(body); + (v, 4) + } + }; + for (i, v) in values.iter().enumerate() { + let (bytes, align) = match v { + None | Some(ColumnValue::Null) => continue, + Some(ColumnValue::Bool(b)) => (vec![u8::from(*b)], 1), + Some(ColumnValue::Char(c)) => (vec![*c as u8], 1), + Some(ColumnValue::Int2(n)) => (n.to_le_bytes().to_vec(), 2), + Some(ColumnValue::Int4(n) | ColumnValue::Date(n)) => (n.to_le_bytes().to_vec(), 4), + Some(ColumnValue::Oid(n)) => (n.to_le_bytes().to_vec(), 4), + Some(ColumnValue::Float4(f)) => (f.to_le_bytes().to_vec(), 4), + Some( + ColumnValue::Int8(n) + | ColumnValue::Time(n) + | ColumnValue::Timestamp(n) + | ColumnValue::TimestampTz(n), + ) => (n.to_le_bytes().to_vec(), 8), + Some(ColumnValue::Oid8(n)) => (n.to_le_bytes().to_vec(), 8), + Some(ColumnValue::Float8(f)) => (f.to_le_bytes().to_vec(), 8), + Some(ColumnValue::Uuid(u)) => (u.to_vec(), 1), + Some(ColumnValue::Text(t) | ColumnValue::Json(t)) => varlena(t.as_bytes()), + Some(ColumnValue::Bytea(b)) => varlena(b), + Some(ColumnValue::ExternalToast(p)) => { + let mut v = vec![0x01, 18]; + for word in [ + p.va_rawsize as u32, + p.va_extinfo, + p.va_valueid, + p.va_toastrelid, + ] { + v.extend_from_slice(&word.to_le_bytes()); + } + (v, 1) + } + Some(other) => unimplemented!("form_block_data: {other:?}"), + }; + if has_null { + bitmap[i / 8] |= 1 << (i % 8); + } + data.resize(data.len().next_multiple_of(align), 0); + data.extend_from_slice(&bytes); + } + let mut out = (natts as u16).to_le_bytes().to_vec(); + out.extend_from_slice(&if has_null { HEAP_HASNULL } else { 0 }.to_le_bytes()); + out.push(hoff as u8); + out.extend_from_slice(&bitmap); + out.resize(SIZE_OF_HEAP_HEADER + hoff - SIZE_OF_HEAP_TUPLE_HEADER, 0); + out.extend_from_slice(&data); + out +} + /// Fields PG strips into `xl_heap_header` or `xl_multi_insert_tuple` #[derive(Clone, Copy)] struct TupleHeader { @@ -846,12 +936,25 @@ struct TupleHeader { /// it or past end of `buf` surface absent and caller flips `partial`. /// Bitmap + xl_heap_header precede prefix elision so are always present at /// `header_off`. +#[inline] fn decode_tuple_payload<'a, T: TupleBuild<'a>>( buf: &'a [u8], header_off: usize, rel: &RelDescriptor, prefixlen: usize, ) -> Result { + let mut out = T::with_capacity(rel.attributes.len()); + decode_tuple_payload_into(buf, header_off, rel, prefixlen, &mut out)?; + Ok(out) +} + +fn decode_tuple_payload_into<'a, T: TupleBuild<'a>>( + buf: &'a [u8], + header_off: usize, + rel: &RelDescriptor, + prefixlen: usize, + out: &mut T, +) -> Result<(), DecodeError> { if buf.len() < header_off + SIZE_OF_HEAP_HEADER { return Err(DecodeError::Truncated { offset: header_off, @@ -871,6 +974,7 @@ fn decode_tuple_payload<'a, T: TupleBuild<'a>>( header_off + SIZE_OF_HEAP_HEADER, rel, prefixlen, + out, ) } @@ -882,7 +986,8 @@ fn walk_tuple<'a, T: TupleBuild<'a>>( bitmap_off: usize, rel: &RelDescriptor, prefixlen: usize, -) -> Result { + out: &mut T, +) -> Result<(), DecodeError> { let t_hoff = usize::from(header.hoff); if t_hoff < SIZE_OF_HEAP_TUPLE_HEADER { return Err(DecodeError::BadHoff { @@ -910,7 +1015,6 @@ fn walk_tuple<'a, T: TupleBuild<'a>>( let col_data_off = bitmap_off + (t_hoff - SIZE_OF_HEAP_TUPLE_HEADER); let attrs = &rel.attributes; - let mut out = T::with_capacity(attrs.len()); // Mark this + every later col absent let absent_from = |out: &mut T, idx: usize| { for att in &attrs[idx..] { @@ -959,8 +1063,8 @@ fn walk_tuple<'a, T: TupleBuild<'a>>( // accounting only; varlena width needs its bytes so later cols // stay absent too let Some(n) = decoded_size_or_skip(att) else { - absent_from(&mut out, idx); - return Ok(out); + absent_from(out, idx); + return Ok(()); }; out.push(att, DatumRef::Absent); cur += n; @@ -977,13 +1081,13 @@ fn walk_tuple<'a, T: TupleBuild<'a>>( } } let Ok((datum, consumed)) = read_datum(att, buf, abs) else { - absent_from(&mut out, idx); - return Ok(out); + absent_from(out, idx); + return Ok(()); }; out.push(att, datum); cur += consumed; } - Ok(out) + Ok(()) } /// Resolve a column's fast default ([`RelAttr::missing_default`]) to a value; @@ -1587,6 +1691,25 @@ fn read_cstring(buf: &[u8], abs: usize) -> Result<(DatumRef<'_>, usize), Truncat Ok((DatumRef::Cstring(&buf[abs..abs + end]), end + 1)) } +/// `(chunk_id, chunk_seq, chunk_data)` of a `pg_toast_*` tuple's 3 datums; +/// `None` for shapes that don't fit +pub(crate) fn toast_chunk_datums<'a>(datums: &[DatumRef<'a>]) -> Option<(u32, u32, &'a [u8])> { + let [ + DatumRef::Fixed(id), + DatumRef::Fixed(seq), + DatumRef::Varlena(data), + .., + ] = *datums + else { + return None; + }; + Some(( + u32::from_le_bytes(id.try_into().ok()?), + u32::from_le_bytes(seq.try_into().ok()?), + data, + )) +} + /// Take `(chunk_id, chunk_seq, chunk_data)` out of a `pg_toast_*` tuple's /// 3 columns (`chunk_id oid`, `chunk_seq int4`, `chunk_data bytea`); /// `None` for shapes that don't fit diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index d62bf04f..5a4f9b69 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -1432,7 +1432,8 @@ pub(crate) struct ColumnPlan { pub(crate) datetime64_scale: u32, pub(crate) non_finite: NonFinite, pub(crate) encoding: ColumnEncoding, - /// Source attribute feeding this column, `None` when descriptor lacks it + /// Source attribute feeding this column, `None` when descriptor holds + /// it dropped pub(crate) source: Option, } @@ -1792,12 +1793,19 @@ impl TablePlan { if plan.encoding != ColumnEncoding::Local && plan.non_finite != NonFinite::default() { return Err(config_err("substitution needs local encoding")); } - plan.source = att.and_then(|a| { - Some(SourceColumn { - index: usize::try_from(c.src_attnum - 1).ok()?, - direct: Direct::choose(a, &plan), - }) - }); + // Attnums past the descriptor still source patched values + plan.source = usize::try_from(c.src_attnum - 1) + .ok() + .and_then(|index| match att { + Some(a) => Some(SourceColumn { + index, + direct: Direct::choose(a, &plan), + }), + None => (index >= rel.attributes.len()).then_some(SourceColumn { + index, + direct: Direct::Value, + }), + }); columns.push(plan); col_sql.push(quote_ident(&c.target_name)); } @@ -2401,71 +2409,87 @@ impl TableEncoder { op_code, leaf, }; - self.append_cells(&cells, &stamp) + self.append_cells(|i| cells[i], &stamp) } - /// [`Self::append_routed`] reading one image of `rel` in place. Values - /// whose decode allocates, and PostGIS renders, build in row scratch + /// [`Self::append_routed`] reading one image of `rel` in place. `patch` + /// replaces attributes by index: resolved TOAST values, and values past + /// `rel`'s attributes. Values whose decode allocates, and PostGIS + /// renders, build in row scratch pub(crate) fn append_view( &mut self, rel: &RelDescriptor, datums: &[DatumRef<'_>], + patch: &[(usize, ColumnValue)], stamp: &RowStamp, ) -> Result { - enum Slot<'a> { - Cell(Cell<'a>), - Owned(usize), - } - let mut owned: SmallVec<[ColumnValue; 2]> = SmallVec::new(); - let mut own = |v: ColumnValue| { - owned.push(v); - Slot::Owned(owned.len() - 1) - }; - let slots: SmallVec<[Slot<'_>; 16]> = self - .plan - .columns - .iter() - .map(|c| { - let Some((src, &datum)) = - c.source.and_then(|src| Some((src, datums.get(src.index)?))) - else { - return Slot::Cell(Cell::Absent); - }; - let att = &rel.attributes[src.index]; - match (src.direct, datum) { - (Direct::Copy, DatumRef::Fixed(le)) => Slot::Cell(Cell::Le(le)), - (Direct::Wkt, _) => match datum_value(att, datum) { - None => Slot::Cell(Cell::Absent), - Some(mut v) => { - crate::ops::oracle::render_ext_value(&mut v); - own(v) - } - }, - _ => match datum_view(att, datum) { - Ok(v) => Slot::Cell(v.map_or(Cell::Absent, Cell::Value)), - Err(_) => { - own(datum_value(att, datum).expect("allocating datum is present")) - } - }, + let mut row = RowScratch::new(self.plan.columns.len()); + for c in &self.plan.columns { + let Some(src) = c.source else { + row.cell(Cell::Absent); + continue; + }; + let wkt = src.direct == Direct::Wkt; + if let Some((_, v)) = patch.iter().find(|(i, _)| *i == src.index) { + if wkt { + let mut v = v.clone(); + crate::ops::oracle::render_ext_value(&mut v); + row.own(v); + } else { + row.cell(Cell::Value(v.view())); } - }) - .collect(); - let cells: SmallVec<[Cell<'_>; 16]> = slots - .iter() - .map(|s| match s { - Slot::Cell(c) => *c, - Slot::Owned(i) => Cell::Value(owned[*i].view()), - }) - .collect(); - self.append_cells(&cells, stamp) + continue; + } + let (Some(&datum), Some(att)) = (datums.get(src.index), rel.attributes.get(src.index)) + else { + row.cell(Cell::Absent); + continue; + }; + match (src.direct, datum) { + (Direct::Copy, DatumRef::Fixed(le)) => row.cell(Cell::Le(le)), + (Direct::Wkt, _) => match datum_value(att, datum) { + None => row.cell(Cell::Absent), + Some(mut v) => { + crate::ops::oracle::render_ext_value(&mut v); + row.own(v); + } + }, + _ => match datum_view(att, datum) { + Ok(v) => row.cell(v.map_or(Cell::Absent, Cell::Value)), + Err(_) => { + row.own(datum_value(att, datum).expect("allocating datum is present")) + } + }, + } + } + self.append_cells(|i| row.get(i), stamp) } - /// Append one row of cells in `plan.columns` order - fn append_cells( + /// Append one row whose mapped sources `slot` yields; columns without a + /// source attribute stay absent + pub(crate) fn append_slots<'a>( &mut self, - cells: &[Cell<'_>], + mut slot: impl FnMut(SourceColumn) -> Result, EmitterError>, stamp: &RowStamp, ) -> Result { + let mut row = RowScratch::new(self.plan.columns.len()); + for c in &self.plan.columns { + match c.source.map(&mut slot).transpose()? { + None => row.cell(Cell::Absent), + Some(Slot::Cell(cell)) => row.cell(cell), + Some(Slot::Owned(v)) => row.own(v), + } + } + self.append_cells(|i| row.get(i), stamp) + } + + /// Append one row, `cell(i)` yielding `plan.columns[i]`'s cell + fn append_cells<'c>( + &mut self, + cell: impl Fn(usize) -> Cell<'c>, + stamp: &RowStamp, + ) -> Result { + let columns = self.plan.columns.len(); // Size oracle cells before mutating column buffers let mut oracle = std::mem::take(&mut self.oracle_cells); oracle.clear(); @@ -2473,9 +2497,9 @@ impl TableEncoder { // Literals bound for a String target never reach the request, so they // owe it nothing until a cell needing conversion strands them let mut stranded_bytes = 0usize; - for (i, cell) in cells.iter().enumerate() { + for i in 0..columns { if let ColumnBuf::Oracle(o) = &self.buffers[i] { - let cell = oracle_cell(*cell, o.source_type_oid) + let cell = oracle_cell(cell(i), o.source_type_oid) .map_err(|e| name_column(e, &self.plan.columns[i].name))?; if !o.resolves_locally(&cell) { row_oracle_bytes += cell.wire_bytes(); @@ -2498,7 +2522,7 @@ impl TableEncoder { ))); } let mut oracle_iter = oracle.iter_mut(); - for (i, cell) in cells.iter().enumerate() { + for i in 0..columns { let col_plan = &self.plan.columns[i]; let buf = &mut self.buffers[i]; if let ColumnBuf::Oracle(o) = buf { @@ -2506,7 +2530,7 @@ impl TableEncoder { o.push(std::mem::replace(cell, OracleCell::Default)); continue; } - match *cell { + match cell(i) { // Absent coerces like NULL, covers key-only delete // tombstones under non-FULL replica identity Cell::Absent => { @@ -2527,7 +2551,7 @@ impl TableEncoder { self.oracle_cells = oracle; self.oracle_bytes += row_oracle_bytes + stranded_bytes; // Synthetic columns: lsn, xid, commit_ts (unix micros), delete marker - let off = cells.len(); + let off = columns; push_fixed(&mut self.buffers[off], &stamp.source_lsn.to_le_bytes())?; push_fixed(&mut self.buffers[off + 1], &stamp.xid.to_le_bytes())?; let unix_us = stamp.commit_ts.saturating_add(DATETIME64_PG_EPOCH_US); @@ -2550,6 +2574,50 @@ impl TableEncoder { } } +/// One row's cells in `plan.columns` order, values whose decode allocates +/// held aside so cells can borrow them +struct RowScratch<'a> { + marks: SmallVec<[Mark<'a>; 16]>, + owned: SmallVec<[ColumnValue; 2]>, +} + +/// Cell, or index into [`RowScratch::owned`] +enum Mark<'a> { + Cell(Cell<'a>), + Owned(usize), +} + +impl<'a> RowScratch<'a> { + fn new(columns: usize) -> Self { + Self { + marks: SmallVec::with_capacity(columns), + owned: SmallVec::new(), + } + } + + fn cell(&mut self, cell: Cell<'a>) { + self.marks.push(Mark::Cell(cell)); + } + + fn own(&mut self, v: ColumnValue) { + self.owned.push(v); + self.marks.push(Mark::Owned(self.owned.len() - 1)); + } + + fn get(&self, i: usize) -> Cell<'_> { + match &self.marks[i] { + Mark::Cell(c) => *c, + Mark::Owned(j) => Cell::Value(self.owned[*j].view()), + } + } +} + +/// One mapped source value, borrowed or built in row scratch +pub(crate) enum Slot<'a> { + Cell(Cell<'a>), + Owned(ColumnValue), +} + /// One mapped cell ready to encode #[derive(Clone, Copy, Debug)] pub(crate) enum Cell<'a> { @@ -3296,6 +3364,47 @@ pub(crate) enum Wire<'a> { Oracle(&'a [OracleCell]), } +#[cfg(test)] +impl EncodedChunk { + /// Mapped cells per row read back from wire buffers: fixed widths as + /// signed ints, strings as bytes + pub(crate) fn cells(&self) -> Vec> { + let fixed = |b: &[u8]| match b.len() { + 1 => ColumnValue::Char(b[0] as i8), + 2 => ColumnValue::Int2(i16::from_le_bytes(b.try_into().unwrap())), + 4 => ColumnValue::Int4(i32::from_le_bytes(b.try_into().unwrap())), + 8 => ColumnValue::Int8(i64::from_le_bytes(b.try_into().unwrap())), + _ => ColumnValue::Bytea(b.to_vec()), + }; + let string = |offsets: &[u64], data: &[u8], r: usize| { + let start = r.checked_sub(1).map_or(0, |p| offsets[p] as usize); + ColumnValue::Bytea(data[start..offsets[r] as usize].to_vec()) + }; + let mapped = self.route.mapping.columns.len(); + let mut rows = vec![Vec::with_capacity(mapped); self.rows]; + for buf in &self.buffers[..mapped] { + for (r, row) in rows.iter_mut().enumerate() { + row.push(match buf.wire() { + Wire::Fixed { width, bytes } => fixed(&bytes[r * width..(r + 1) * width]), + Wire::NullableFixed { null_map, .. } + | Wire::NullableString { null_map, .. } + if null_map[r] != 0 => + { + ColumnValue::Null + } + Wire::NullableFixed { width, inner, .. } => { + fixed(&inner[r * width..(r + 1) * width]) + } + Wire::String { offsets, data, .. } + | Wire::NullableString { offsets, data, .. } => string(offsets, data, r), + Wire::Oracle(cells) => ColumnValue::Text(format!("{:?}", cells[r])), + }); + } + } + rows + } +} + #[cfg(test)] impl ColumnBuf { pub(crate) fn wire(&self) -> Wire<'_> { diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs index 6eb72840..c577f5a5 100644 --- a/src/emit/direct_parity.rs +++ b/src/emit/direct_parity.rs @@ -97,8 +97,12 @@ fn lz4(content: &[u8]) -> Vec { } fn external() -> Vec { + external_id(7) +} + +fn external_id(value_id: u32) -> Vec { let mut v = vec![0x01, 18]; - for word in [100i32 as u32, 96, 7, 16400] { + for word in [100i32 as u32, 96, value_id, 16400] { v.extend_from_slice(&word.to_le_bytes()); } v @@ -213,7 +217,7 @@ fn assert_parity( .append_routed(&committed, &mapping, op, None) .map_err(|e| e.to_string()); let b = direct - .append_view(rel, &view.datums, &stamp) + .append_view(rel, &view.datums, &[], &stamp) .map_err(|e| e.to_string()); assert_eq!(a, b, "row {i}: {values:?}"); if a.is_err() { @@ -805,7 +809,7 @@ fn predeclared_columns_match() { .append_routed(&committed, &mapping, OP_INSERT, None) .unwrap(); let view = view_block_data(&data, &rel).unwrap(); - direct.append_view(&rel, &view.datums, &stamp).unwrap(); + direct.append_view(&rel, &view.datums, &[], &stamp).unwrap(); let wire = |e: &mut TableEncoder| { let (bufs, _) = e.take_block().unwrap(); bufs.iter() @@ -835,8 +839,21 @@ fn view_matches_owned_decode_for_records() { /// Operation with new and old tuple bytes type Change = (HeapOp, Option>, Option>); -/// Plan row changes both ways; segments must carry identical rows -fn plan_both(rel: &RelDescriptor, targets: &[&str], changes: &[Change]) { +/// Plan row changes both ways; segments must carry identical rows. +/// External values resolve through `values` by value id, as detoast would +fn plan_both( + rel: &RelDescriptor, + targets: &[&str], + changes: &[Change], + values: &[(u32, ColumnValue)], +) { + let resolved = |p: crate::decode::heap_decoder::ToastPointer| { + values + .iter() + .find(|(id, _)| *id == p.va_valueid) + .map(|(_, v)| v.clone()) + .expect("value for every pointer") + }; use crate::emit::pipeline::plan_spool::{PlanItem, PlanWriter, ViewChange}; use crate::emit::route::RouteSnapshot; let rel = std::sync::Arc::new(rel.clone()); @@ -856,22 +873,37 @@ fn plan_both(rel: &RelDescriptor, targets: &[&str], changes: &[Change]) { if views { let new = new.as_ref().map(|d| view_block_data(d, &rel).unwrap()); let old = old.as_ref().map(|d| view_block_data(d, &rel).unwrap()); + let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); + for site in crate::xact::xact_buffer::external_sites([new.as_ref(), old.as_ref()]) { + patches[site.image].push((site.idx, resolved(site.p))); + } let change = ViewChange { op: *op, xid: 5, source_lsn: lsn, new: new.as_ref(), old: old.as_ref(), + new_patch: &patches[0], + old_patch: &patches[1], }; w.push_views(change, table).unwrap(); } else { + let decode = |d: &Vec| { + let mut t = decode_block_data(d, &rel).unwrap(); + for c in t.columns.iter_mut() { + if let Some(ColumnValue::ExternalToast(p)) = *c { + *c = Some(resolved(p)); + } + } + t + }; let decoded = DecodedHeap { rfn: rel.rfn, xid: 5, source_lsn: lsn, op: *op, - new: new.as_ref().map(|d| decode_block_data(d, &rel).unwrap()), - old: old.as_ref().map(|d| decode_block_data(d, &rel).unwrap()), + new: new.as_ref().map(decode), + old: old.as_ref().map(decode), }; w.push_cells(decoded, table).unwrap(); } @@ -947,6 +979,37 @@ fn plan_views_match_cells_for_row_changes() { (HeapOp::Delete, None, Some(row(3, 3.0, "c"))), (HeapOp::Delete, None, None), ], + &[], ); } } + +/// Resolved external values patch tuple views as detoast patches cells; +/// key changes compare resolved values, not pointers +#[test] +fn plan_views_match_cells_with_resolved_externals() { + let mut r = rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, TEXTOID, -1, 'i', "text"), + ]); + r.replident = ReplIdent::Full { pk_attnums: None }; + let attrs = r.attributes.clone(); + let row = |id: i32, value_id: u32| { + let ext = external_id(value_id); + tuple(&attrs, 2, &[Some(&id.to_le_bytes()[..]), Some(&ext[..])]) + }; + let text = |s: &str| ColumnValue::Text(s.into()); + plan_both( + &r, + &["Int32", "String"], + &[ + (HeapOp::Insert, Some(row(1, 7)), None), + // Other pointer, same value: no key change + (HeapOp::Update, Some(row(1, 8)), Some(row(1, 7))), + // Other value: delete marker plus new row + (HeapOp::Update, Some(row(1, 9)), Some(row(1, 8))), + (HeapOp::Delete, None, Some(row(1, 9))), + ], + &[(7, text("same")), (8, text("same")), (9, text("other"))], + ); +} diff --git a/src/emit/pipeline/ack.rs b/src/emit/pipeline/ack.rs index c11f5ecb..d2aee30e 100644 --- a/src/emit/pipeline/ack.rs +++ b/src/emit/pipeline/ack.rs @@ -362,6 +362,11 @@ impl AckHandle { buffered_events: None, } } + + /// [`SeqAlloc::place`] from a task other than the allocator's + pub fn place(&self, OpenSeq(seq): OpenSeq, rows: u64) { + let _ = self.tx.send(AckEvent::Placed { seq, rows }); + } } /// Whether a done seq advances `emitter_ack` to its `commit_lsn` diff --git a/src/emit/pipeline/batcher.rs b/src/emit/pipeline/batcher.rs index e7df3e19..95825b07 100644 --- a/src/emit/pipeline/batcher.rs +++ b/src/emit/pipeline/batcher.rs @@ -1,10 +1,9 @@ //! Insert batcher — per-table accumulation stage. //! -//! Coalesces decoded rows ([`RoutedRow`]) per destination table into -//! budget-sized ClickHouse Native blocks (`InsertBatch`). Encoding happens -//! here, not in decoders, so rows from all M decoders and all xacts merge -//! into one part per flush window per table instead of one part per decoder -//! per xact. +//! Merges encoded column chunks ([`PlacedSegment`]) per destination table +//! into budget-sized ClickHouse Native blocks (`InsertBatch`), so rows from +//! every producer and xact share one part per flush window per table +//! instead of one part per producer per xact. //! //! Single hub task owns one `TableEncoder` per table; per-table task //! sharding / hash(pk) splitting is the plan's later optimization. @@ -30,10 +29,10 @@ use tokio::task::JoinHandle; use tokio::time::Instant; use crate::config::ResolvedConfig; -use crate::decode::heap_decoder::{CommittedTuple, HeapOp}; +#[cfg(any(test, feature = "test-support"))] +use crate::decode::heap_decoder::CommittedTuple; use crate::emit::ch_emitter::{ - Append, ColumnBuf, ColumnEncoding, EmitterStats, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, - TableEncoder, TablePlan, + Append, ColumnBuf, ColumnEncoding, EmitterStats, EncodedChunk, TableEncoder, TablePlan, }; use crate::emit::pipeline::{DEFAULT_PIPELINE_FLUSH, Fatal}; use crate::emit::route::RouteSnapshot; @@ -51,126 +50,54 @@ pub struct RowTable { pub route: Arc, } -/// One decoded row routed to its destination -pub struct RoutedRow { +/// Planned segment placed under its seq +pub struct PlacedSegment { pub seq: u64, - /// Index into containing chunk's `tables` - pub table: u32, - pub committed: CommittedTuple, - /// Detoast leaf permit shrunk to this row's decoded TOAST bytes, so - /// decoded values and their encoder slab copy stay covered to insert - /// ack. Per row, unlike the slice permit on [`RowChunk`] - pub value_permit: Option>, -} - -/// Share table metadata and a memory permit across decoded rows -#[derive(Default)] -pub struct RowChunk { - pub tables: Vec, - pub rows: Vec, + pub chunks: Vec, /// Released post-insert-ack, when every covering `InsertBatch` drops pub permit: Option>, } -impl RowChunk { - fn table(&mut self, rel: &Arc, route: &Arc) -> u32 { - if !self - .tables - .last() - .is_some_and(|t| Arc::ptr_eq(&t.rel, rel) && Arc::ptr_eq(&t.route, route)) - { - self.tables.push(RowTable { - rel: rel.clone(), - route: route.clone(), - }); - } - (self.tables.len() - 1) as u32 - } -} - -/// Rows per chunk a [`RowBuffer`] ships. Fixed rather than config-driven: -/// backfill rows are uniform, and the byte trigger covers the fat-row case -const ROW_BUFFER_MAX_ROWS: usize = 1024; - -/// Coalesces routed rows into one `BatcherMsg::Rows` per `ROW_BUFFER_MAX_ROWS` -/// rows or [`DECODE_CHUNK_BYTES`], the same amortization streaming placement -/// gets. Rows of different seqs may share a chunk; the batcher routes each -/// independently -#[derive(Default)] -pub struct RowBuffer { - chunk: RowChunk, - bytes: usize, -} - -impl RowBuffer { - /// Share `permit` across all buffered and sent chunks - pub fn new(permit: Option>) -> Self { - Self { - chunk: RowChunk { - permit, - ..Default::default() - }, - bytes: 0, - } - } - - pub async fn push( - &mut self, - msg_tx: &mpsc::Sender, +#[cfg(any(test, feature = "test-support"))] +impl PlacedSegment { + /// Segment of owned rows under one table, each encoded as its own op + pub fn of_rows( seq: u64, - table: (&Arc, &Arc), - committed: CommittedTuple, - value_permit: Option>, - ) -> Result<(), BatcherClosed> { - if self.stage(seq, table, committed, value_permit) { - self.flush(msg_tx).await?; + rel: &Arc, + route: &Arc, + rows: &[CommittedTuple], + ) -> Result { + use crate::emit::pipeline::plan_spool::{ChunkBuilder, op_code}; + let mut builder = ChunkBuilder::new(usize::MAX, usize::MAX); + let leaf = route.partition.as_ref().map(|p| p.leaf); + for r in rows { + let op = op_code(r.decoded.op)?; + builder.append((), rel, route, |enc| { + enc.append_routed(r, &route.mapping, op, leaf) + })?; } - Ok(()) - } - - /// Buffer a row, true once buffer is due to flush - pub fn stage( - &mut self, - seq: u64, - table: (&Arc, &Arc), - committed: CommittedTuple, - value_permit: Option>, - ) -> bool { - // Flush leaf permits before another acquisition can wait on them. - let has_value_permit = value_permit.is_some(); - self.bytes += committed.decoded.approx_bytes(); - let table = self.chunk.table(table.0, table.1); - self.chunk.rows.push(RoutedRow { + Ok(Self { seq, - table, - committed, - value_permit, - }); - has_value_permit - || self.chunk.rows.len() >= ROW_BUFFER_MAX_ROWS - || self.bytes >= DECODE_CHUNK_BYTES - } - - pub async fn flush(&mut self, msg_tx: &mpsc::Sender) -> Result<(), BatcherClosed> { - if self.chunk.rows.is_empty() { - return Ok(()); - } - self.bytes = 0; - let next = Self::new(self.chunk.permit.clone()).chunk; - let chunk = std::mem::replace(&mut self.chunk, next); - msg_tx - .send(BatcherMsg::Rows(chunk)) - .await - .map_err(|_| BatcherClosed) + chunks: builder.take()?.into_iter().map(|(_, c)| c).collect(), + permit: None, + }) } } -/// Planned segment placed under its seq -pub struct PlacedSegment { - pub seq: u64, - pub chunks: Vec, - /// Released post-insert-ack, when every covering `InsertBatch` drops - pub permit: Option>, +/// `(seq, mapped cells)` per row of `segments`, see [`EncodedChunk::cells`] +#[cfg(test)] +pub(crate) fn segment_rows( + segments: &[PlacedSegment], +) -> Vec<(u64, Vec)> { + segments + .iter() + .flat_map(|s| { + s.chunks + .iter() + .flat_map(|c| c.cells()) + .map(move |cells| (s.seq, cells)) + }) + .collect() } #[derive(Debug, thiserror::Error)] @@ -266,12 +193,9 @@ pub(crate) struct InsertBatch { /// process ahead of rows enqueued before it — else flush seals a partial set /// and falsely signals "earlier data sealed", pinning the durability wait. pub enum BatcherMsg { - /// Chunk of rows from one placement (see [`DECODE_CHUNK_BYTES`]), - /// amortizing the per-row channel-send + cross-thread wakeup — the - /// dominant coordination cost under sustained load. Rows may carry - /// different `seq`s; batcher routes each independently. - Rows(RowChunk), - /// Planned segments staged across commits, one hop for the lot + /// Segments staged across commits, one hop for the lot, amortizing the + /// channel send + cross-thread wakeup that dominates coordination + /// under sustained load Segments(Vec), /// Seal every open table, push to inserters, reply. Barrier (drain before /// DDL/TRUNCATE) and shutdown. @@ -299,7 +223,6 @@ struct Table { meta: Arc, seq_counts: Vec<(u64, u64)>, slice_permits: Vec>, - value_permits: Vec>, deadline: Option, } @@ -352,7 +275,6 @@ impl Tables { meta, seq_counts: Vec::new(), slice_permits: Vec::new(), - value_permits: Vec::new(), deadline: None, }); Ok(*e.insert(self.open.len() - 1)) @@ -425,14 +347,6 @@ pub(crate) fn spawn( } tokio::select! { msg = msg_rx.recv() => match msg { - Some(BatcherMsg::Rows(chunk)) => { - let live = effective_cfg(&cfg, snapshot(config_rx.as_ref()).as_deref()); - let ctx = RowCtx { cfg: live, out: &out, alloc, epoch, stats }; - if let Err(e) = handle_rows(&mut tables, &ctx, chunk).await { - fatal.set(format!("batcher: {e}")); - break; - } - } Some(BatcherMsg::Segments(segments)) => { let live = effective_cfg(&cfg, snapshot(config_rx.as_ref()).as_deref()); let ctx = RowCtx { cfg: live, out: &out, alloc, epoch, stats }; @@ -513,63 +427,6 @@ async fn config_changed(rx: &mut Option>>) { } } -/// Process a decoder's row chunk in order. Chunk only amortizes the channel -/// hop, not the coalescing; budget/deadline trips behave per-row. -async fn handle_rows(tables: &mut Tables, ctx: &RowCtx<'_>, chunk: RowChunk) -> Result<(), String> { - ctx.stats - .insertbatch_rows_in - .fetch_add(chunk.rows.len() as u64, Ordering::Relaxed); - let slots = chunk - .tables - .iter() - .map(|t| tables.slot(ctx, t)) - .collect::, _>>()?; - for row in chunk.rows { - let i = row.table as usize; - let route = &chunk.tables[i].route; - handle_row(tables, ctx, slots[i], route, row, chunk.permit.as_ref()).await?; - } - Ok(()) -} - -async fn handle_row( - tables: &mut Tables, - ctx: &RowCtx<'_>, - slot: usize, - route: &RouteSnapshot, - row: RoutedRow, - slice_permit: Option<&Arc>, -) -> Result<(), String> { - let t = &mut tables.open[slot]; - let op = match row.committed.decoded.op { - HeapOp::Insert => OP_INSERT, - HeapOp::Update | HeapOp::HotUpdate => OP_UPDATE, - HeapOp::Delete => OP_DELETE, - // TRUNCATE is a reorder barrier; must never route here - HeapOp::Truncate => return Err("TRUNCATE routed to batcher".into()), - }; - // Seal before appending row that exceeds oracle frame threshold - let leaf = route.partition.as_ref().map(|p| p.leaf); - let append = t - .enc - .append_routed(&row.committed, &route.mapping, op, leaf) - .map_err(|e| e.to_string())?; - if append == Append::Full { - t.seal_block()?; - t.enc - .append_routed(&row.committed, &route.mapping, op, leaf) - .map_err(|e| e.to_string())?; - } - if let Some(p) = row.value_permit - && t.value_permits - .last() - .is_none_or(|held| !Arc::ptr_eq(held, &p)) - { - t.value_permits.push(p); - } - appended(tables, ctx, slot, row.seq, 1, slice_permit).await -} - /// Merge each chunk of a planned segment into its destination's batch async fn handle_segment( tables: &mut Tables, @@ -686,12 +543,10 @@ async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> R if n_rows == 0 { t.seq_counts.clear(); t.slice_permits.clear(); - t.value_permits.clear(); return Ok(()); } let per_seq = std::mem::take(&mut t.seq_counts); - let mut permits = std::mem::take(&mut t.slice_permits); - permits.append(&mut t.value_permits); + let permits = std::mem::take(&mut t.slice_permits); // Separate pool avoids waiting on decoded payload held by this batch let encoded = out.budget.acquire(bytes).await; let batch = InsertBatch { @@ -750,7 +605,9 @@ async fn flush_all( #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, HeapOp}; + use crate::decode::heap_decoder::{ + ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp, + }; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; use tokio::sync::oneshot; @@ -813,46 +670,20 @@ mod tests { one(rel_named(table), route_named(table), tuple(seq, id)) } - fn one(rel: Arc, route: Arc, row: RoutedRow) -> BatcherMsg { - BatcherMsg::Rows(RowChunk { - tables: vec![RowTable { rel, route }], - rows: vec![row], - permit: None, - }) - } - - #[tokio::test] - async fn row_buffer_flushes_leaf_permit_with_buffered_rows() { - let (tx, mut rx) = mpsc::channel(1); - let budget = crate::budget::MemoryBudget::new(256); - let mut row_buffer = RowBuffer::default(); - let (rel, route) = (rel_named("t"), route_named("t")); - for id in 0..2 { - let row = tuple(0, id); - let permit = if id == 1 { - Some(Arc::new(budget.acquire(256).await)) - } else { - None - }; - row_buffer - .push(&tx, row.seq, (&rel, &route), row.committed, permit) - .await - .unwrap(); - } - let msg = rx.try_recv().expect("flush pinned row"); - assert!(matches!(&msg, BatcherMsg::Rows(c) if c.rows.len() == 2)); - assert_eq!(budget.resident_bytes(), 256); - drop(msg); - assert_eq!(budget.resident_bytes(), 0); - row_buffer.flush(&tx).await.unwrap(); - assert!(rx.try_recv().is_err()); + fn one( + rel: Arc, + route: Arc, + (seq, row): (u64, CommittedTuple), + ) -> BatcherMsg { + BatcherMsg::Segments(vec![ + PlacedSegment::of_rows(seq, &rel, &route, &[row]).unwrap(), + ]) } - fn tuple(seq: u64, id: i32) -> RoutedRow { - RoutedRow { + fn tuple(seq: u64, id: i32) -> (u64, CommittedTuple) { + ( seq, - table: 0, - committed: CommittedTuple { + CommittedTuple { decoded: DecodedHeap { rfn: RelFileNode { spc_node: 1663, @@ -871,8 +702,7 @@ mod tests { commit_ts: 0, commit_lsn: (seq + 1) * 100, }, - value_permit: None, - } + ) } fn text_table(target_type: &str) -> (Arc, Arc) { @@ -943,62 +773,6 @@ mod tests { assert!(fatal.message().is_none(), "no fatal: {:?}", fatal.message()); } - /// A mixed-seq `Rows` chunk trips the budget mid-chunk yet reconciles - /// per-seq same as the per-row path — chunk boundary is purely a - /// channel-hop amortization (point of `DECODE_CHUNK_ROWS`). - #[tokio::test] - async fn rows_chunk_trips_budget_and_tracks_per_seq() { - let (msg_tx, msg_rx) = mpsc::channel(64); - let (batches_tx, batches_rx) = async_channel::bounded(64); - let fatal = Fatal::new(); - let handle = spawn( - msg_rx, - batches_tx, - BatcherConfig { - row_budget: 2, - inserters: 1, - byte_budget: 1 << 30, - flush_timeout: Duration::from_secs(3600), - }, - Allocator::global(&mimalloc::MiMalloc), - fatal.clone(), - Arc::new(EmitterStats::default()), - None, - ); - let chunk = RowChunk { - tables: vec![RowTable { - rel: rel_named("t"), - route: route_named("t"), - }], - rows: vec![ - tuple(0, 0), - tuple(0, 1), - tuple(0, 2), - tuple(1, 0), - tuple(1, 1), - ], - permit: None, - }; - msg_tx - .send(BatcherMsg::Rows(chunk)) - .await - .expect("send chunk"); - drop(msg_tx); - - let mut total = 0u64; - let mut per_seq = std::collections::BTreeMap::new(); - while let Ok(b) = batches_rx.recv().await { - total += b.n_rows as u64; - for (seq, n) in b.per_seq { - *per_seq.entry(seq).or_insert(0) += n; - } - } - handle.await.expect("batcher task"); - assert_eq!(total, 5, "all rows sealed exactly once"); - assert_eq!(per_seq, [(0, 3), (1, 2)].into(), "rows per seq"); - assert!(fatal.message().is_none(), "no fatal: {:?}", fatal.message()); - } - /// Route-carried `config_column` override reaches the encoder plan /// unchanged with no config watch wired — the snapshot is the only /// override source (spec §Tests / Route and config) @@ -1383,7 +1157,7 @@ mod tests { for i in 0..2 { let (rel, route) = text_table("String"); let mut r = tuple(i, 1); - r.committed.decoded.new.as_mut().unwrap().columns = + r.1.decoded.new.as_mut().unwrap().columns = vec![Some(ColumnValue::Text("x".repeat(64 << 10)))]; tx.send(one(rel, route, r)).await.unwrap(); } @@ -1412,40 +1186,22 @@ mod tests { seq: u64, rel: &Arc, route: &Arc, - rows: Vec, + rows: Vec<(u64, CommittedTuple)>, ) -> BatcherMsg { - let plan = - TablePlan::for_route(Allocator::global(&mimalloc::MiMalloc), rel, route).unwrap(); - let mut enc = TableEncoder::new(plan).unwrap(); - for r in &rows { - assert_eq!( - enc.append_routed(&r.committed, &route.mapping, OP_INSERT, None) - .unwrap(), - Append::Done - ); - } - let (buffers, rows) = enc.take_block().unwrap(); - BatcherMsg::Segments(vec![PlacedSegment { - seq, - chunks: vec![EncodedChunk { - rel: rel.clone(), - route: route.clone(), - buffers, - rows, - }], - permit: None, - }]) + let rows: Vec<_> = rows.into_iter().map(|(_, r)| r).collect(); + let segment = PlacedSegment::of_rows(seq, rel, route, &rows).unwrap(); + assert_eq!(segment.chunks.len(), 1, "one chunk per segment"); + BatcherMsg::Segments(vec![segment]) } - fn text_row(seq: u64, id: i32, text: &str) -> RoutedRow { + fn text_row(seq: u64, id: i32, text: &str) -> (u64, CommittedTuple) { let mut r = tuple(seq, id); - r.committed.decoded.new.as_mut().unwrap().columns = - vec![Some(ColumnValue::Text(text.into()))]; + r.1.decoded.new.as_mut().unwrap().columns = vec![Some(ColumnValue::Text(text.into()))]; r } /// Chunks of several transactions merge into one batch with string - /// offsets rebased, matching rows batched one at a time + /// offsets rebased, matching one-row chunks #[tokio::test] async fn segments_merge_like_rows() { let (rel, route) = text_table("Nullable(String)"); @@ -1514,32 +1270,6 @@ mod tests { assert!(!fatal.is_set()); } - /// Row that would push oracle cells past the seal threshold opens the - /// next block of the same INSERT - #[tokio::test] - async fn oracle_seal_moves_row_to_next_block() { - let (tx, rx, fatal, handle) = - spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); - let big = "x".repeat(crate::ops::oracle::ORACLE_BATCH_SEAL_BYTES * 3 / 5); - for i in 0..2 { - let (rel, route) = text_table("Array(Nullable(String))"); - let mut r = tuple(i, 1); - r.committed.decoded.new.as_mut().unwrap().columns = - vec![Some(ColumnValue::Text(big.clone()))]; - tx.send(one(rel, route, r)).await.unwrap(); - } - drop(tx); - let b = rx.recv().await.unwrap(); - assert!(rx.recv().await.is_err(), "one INSERT"); - assert_eq!(b.per_seq, [(0, 1), (1, 1)]); - assert_eq!( - b.blocks.iter().map(|b| b.n_rows).collect::>(), - [1, 1] - ); - handle.await.unwrap(); - assert!(!fatal.is_set()); - } - /// Byte budget wins over an unreached deadline: no clock time passes. #[tokio::test(start_paused = true)] async fn byte_budget_flushes_before_deadline() { @@ -1656,12 +1386,6 @@ mod tests { fatal.message().expect("batcher trips fatal") } - fn with_op(op: HeapOp) -> RoutedRow { - let mut row = tuple(0, 1); - row.committed.decoded.op = op; - row - } - fn date_table(target_type: &str) -> (Arc, Arc) { let mut rel = rel_named("t"); Arc::make_mut(&mut rel).attributes[0].type_oid = crate::schema::DATEOID; @@ -1671,28 +1395,26 @@ mod tests { (rel, route) } - #[tokio::test(start_paused = true)] - async fn row_faults_trip_fatal_and_stop_the_batcher() { - let chunk = |row| RowChunk { - tables: vec![RowTable { - rel: rel_named("t"), - route: route_named("t"), - }], - rows: vec![row], - permit: None, + #[test] + fn row_faults_fail_segment_encoding() { + let fault = |rel, route, row: CommittedTuple| { + let Err(e) = PlacedSegment::of_rows(0, &rel, &route, &[row]) else { + panic!("faulty row encodes"); + }; + e.to_string() }; - let truncate = chunk(with_op(HeapOp::Truncate)); - let msg = batcher_fatal(vec![BatcherMsg::Rows(truncate)], true, false).await; - assert_eq!(msg, "batcher: TRUNCATE routed to batcher"); + let (_, mut truncate) = tuple(0, 1); + truncate.decoded.op = HeapOp::Truncate; + let msg = fault(rel_named("t"), route_named("t"), truncate); + assert!(msg.contains("TRUNCATE planned as a row"), "{msg}"); let (rel, route) = date_table("Date32"); - let mut overflow = tuple(0, 1); - overflow.committed.decoded.new.as_mut().unwrap().columns[0] = - Some(ColumnValue::Date(i32::MAX - 1)); - let msg = batcher_fatal(vec![one(rel, route, overflow)], true, false).await; + let (_, mut overflow) = tuple(0, 1); + overflow.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Date(i32::MAX - 1)); + let msg = fault(rel, route, overflow); assert!(msg.contains("Date32 epoch"), "{msg}"); let (rel, route) = date_table("Int32"); - let msg = batcher_fatal(vec![one(rel, route, tuple(0, 1))], true, false).await; + let msg = fault(rel, route, tuple(0, 1).1); assert!(msg.contains("local temporal encoding"), "{msg}"); } diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index f533ed71..db5a528c 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -7,26 +7,33 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; use walrus::pg::walparser::RelFileNode; -use crate::backfill::backup_page_walk::{BackfillTuple, CatalogMap}; +use crate::backfill::backup_page_walk::{BackfillTuple, CatalogMap, TupleData, toast_row}; +use crate::backfill::copy_backfill::copy_slot; use crate::backfill::spool::{DeferredReader, DeferredSpool, Replayable, SpoolMark}; +use crate::backfill::visibility_pending::metadata_patch; use crate::backfill::wal_replay::HeapTid; use crate::backfill::walk_barrier::{WALK_CHECKPOINT_PERIOD, WalkBarrier}; +use crate::budget::MemoryPermit; +use crate::ch::EmitterError; use crate::config::ResolvedConfig; -use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; -use crate::emit::ch_emitter::EmitterStats; +use crate::decode::datum::DatumRef; +use crate::decode::heap_decoder::{ColumnValue, ToastPointer, TupleView, view_block_data_into}; +use crate::emit::ch_emitter::{EmitterStats, OP_INSERT, RowStamp}; use crate::emit::pipeline::ack::{AckHandle, OpenSeq, Publish, SeqAlloc}; -use crate::emit::pipeline::batcher::{BatcherClosed, BatcherMsg, RowBuffer, RowTable}; +use crate::emit::pipeline::batcher::{ + BatcherClosed, BatcherMsg, DECODE_CHUNK_BYTES, PlacedSegment, RowTable, +}; +use crate::emit::pipeline::plan_spool::ChunkBuilder; use crate::emit::route::{FrozenRoutes, RouteSnapshot, RowPolicy, freeze_routes}; use crate::mapping::{MappingSnapshot, TableMapping}; -use crate::ops::oracle::render_ext_columns; use crate::schema::{RelDescriptor, RelName}; use crate::ticker::Ticker; use crate::toast::{ - ChunkStoreError, FetchedValue, ToastResolver, ToastRow, check_value_caps, detoasted_sized, - finish_value, pointer_extsize, + ChunkStoreError, FetchedValue, ToastResolver, check_value_caps, detoasted_sized, finish_value, + pointer_extsize, }; use crate::xact::xact_buffer::{MissPolicy, StoreMiss, fill_store_miss}; use foldhash::{HashMap, HashMapExt, HashSet}; @@ -199,53 +206,74 @@ pub async fn drain( Deferral::Handback(spool) => (Some(spool), true), }; let mut footprint = DeferredFootprint::new(&stats); - let mut seqs = ack.seqs(0); let mut rows_routed = 0; let mut open: Option = None; let mut puts = resolver.chunk_puts(); - let mut row_buffer = RowBuffer::default(); + let mut out = LoadOut::new(&msg_tx, &ack, 0); let mut consumed = 0u64; let mut ticker = Ticker::new( barrier .as_ref() .map_or(WALK_CHECKPOINT_PERIOD, |b| b.period()), ); - while let Some(batch) = rx.recv().await { + loop { + if rx.is_empty() { + out.send().await?; + } + let Some(batch) = rx.recv().await else { + break; + }; consumed += batch.len() as u64; - for mut tuple in batch { + for tuple in batch { let rfn = tuple.rfn; let run = match &mut open { Some(run) if run.rfn == rfn => run, slot => { if let Some(prev) = slot.take() { - // Send all rows before publishing their expected count - row_buffer.flush(&msg_tx).await.map_err(closed)?; - seqs.place(prev.seq, prev.rows); + out.place(prev.seq, prev.rows); } slot.insert(Run { rfn, - seq: seqs.open(tuple.source_lsn, Publish::Commit), + seq: out.seqs.open(tuple.source_lsn, Publish::Commit), rows: 0, target: Target::resolve(&catalog, &routes, &skip_initial, rfn), }) } }; - let (rel, route) = match &run.target { - Target::Routed(rel, route) => (rel, route), + let (rel, route, varlena) = match &run.target { + Target::Routed { + rel, + route, + varlena, + } => (rel, route, *varlena), Target::Unsupported => { stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); continue; } Target::Skipped => continue, - Target::Toast(toast_relid) => { - if let Some(row) = row_from_columns(tuple, rfn.db_node, *toast_relid) { - puts.push(row).await.map_err(put_err)?; + Target::Toast(toast) => { + if let TupleData::Heap(data) = &tuple.data { + debug_assert_ne!(tuple.offnum, 0, "walked toast tuple without TID"); + // Weakest evidence for a TID: a backup copy can be + // torn, so any WAL-sourced row for the same TID outranks it + if let Some(row) = toast_row(data, toast, tuple.blkno, tuple.offnum, 0) { + out.send().await?; + puts.push(row).await.map_err(put_err)?; + } } continue; } }; - if tuple.has_mapped_external(&route.mapping) { + let sites = if varlena { + let view = tuple.view(rel).map_err(|e| decode_err(rel, e))?; + view.as_ref() + .map_or_else(Vec::new, |v| mapped_pointers(v, &route.mapping)) + } else { + Vec::new() + }; + let mut patch = Vec::new(); + if !sites.is_empty() { // Defer until page walk has populated or started backing store if !resolver.fill_on_miss() { let spool = deferred.as_mut().ok_or_else(|| { @@ -258,14 +286,9 @@ pub async fn drain( footprint.publish(spool); continue; } - fill_toast(&mut tuple, &route.mapping, &resolver)?; + patch = fill_toast(sites, &resolver)?; } - render_ext_columns(&rel.attributes, &mut tuple.columns); - let committed = tuple.into_committed_insert(); - row_buffer - .push(&msg_tx, run.seq.seq(), (rel, route), committed, None) - .await - .map_err(closed)?; + out.push(run.seq.seq(), rel, route, tuple, patch).await?; run.rows += 1; rows_routed += 1; } @@ -276,10 +299,10 @@ pub async fn drain( { // Durable files must include their chunks puts.flush().await.map_err(put_err)?; - row_buffer.flush(&msg_tx).await.map_err(closed)?; if let Some(run) = open.take() { - seqs.place(run.seq, run.rows); + out.place(run.seq, run.rows); } + out.flush().await?; let mut mark = SpoolMark::default(); if let Some(spool) = deferred.as_mut() { mark = spool @@ -288,14 +311,14 @@ pub async fn drain( .map_err(|e| format!("bootstrap: deferred spool checkpoint: {e}"))?; footprint.publish(spool); } - b.publish_drain(consumed, seqs.next(), mark).await; + b.publish_drain(consumed, out.seqs.next(), mark).await; } } - row_buffer.flush(&msg_tx).await.map_err(closed)?; if let Some(run) = open.take() { - seqs.place(run.seq, run.rows); + out.place(run.seq, run.rows); } - let next_seq = seqs.next(); + let next_seq = out.seqs.next(); + out.finish().await?; puts.flush().await.map_err(put_err)?; @@ -399,7 +422,6 @@ async fn resolve_spooled( let mut seq = None; let mut placed = 0u64; let total_bytes = spool.spooled_bytes(); - let mut seqs = ack.seqs(first_seq); let mut total_placed = 0; let mut ticker = Ticker::new(WALK_CHECKPOINT_PERIOD); let mut replay = spool @@ -408,6 +430,7 @@ async fn resolve_spooled( .map_err(|e| format!("bootstrap: deferred spool seal: {e}"))?; let mut progress = DeferredReplayProgress::new(stats, total_bytes); progress.advance(replay.remaining_file_bytes()); + let mut out = LoadOut::new(msg_tx, ack, first_seq); // Read and fetch the next batch while this one routes: spool reads and // inserts otherwise leave the store pool idle let mut ready = prepare_batch(&mut replay, routes, catalog, stats, resolver, coverage).await?; @@ -415,18 +438,18 @@ async fn resolve_spooled( let remaining = replay.remaining_file_bytes(); let (next, routed) = tokio::join!( prepare_batch(&mut replay, routes, catalog, stats, resolver, coverage), - route_batch(batch, msg_tx, &mut seqs, &mut seq, &mut placed), + route_batch(batch, &mut out, &mut seq, &mut placed), ); routed?; progress.advance(remaining); if let Some(c) = checkpoint.as_mut() { if let Some(s) = seq.take() { - seqs.place(s, placed); + out.place(s, placed); } total_placed += placed; placed = 0; if ticker.fire() { - c.tail.checkpoint(seqs.next()).await?; + c.tail.checkpoint(out.seqs.next()).await?; c.state.offset = total_bytes - remaining; c.state.rows += total_placed; total_placed = 0; @@ -437,7 +460,7 @@ async fn resolve_spooled( } let rows_routed = match checkpoint.as_mut() { Some(c) => { - c.tail.checkpoint(seqs.next()).await?; + c.tail.checkpoint(out.seqs.next()).await?; c.state.offset = total_bytes; c.state.rows += total_placed; c.state.save(c.dir).await.map_err(|e| e.to_string())?; @@ -453,8 +476,10 @@ async fn resolve_spooled( } }; if let Some(s) = seq { - seqs.place(s, placed); + out.place(s, placed); } + let next_seq = out.seqs.next(); + out.finish().await?; let covered = stats.bootstrap_replay_covered_rows.load(Ordering::Relaxed) - covered_before; if covered > 0 { tracing::info!( @@ -465,7 +490,7 @@ async fn resolve_spooled( ); } Ok(BootstrapDrainOutcome { - next_seq: seqs.next(), + next_seq, rows_routed, deferred: None, }) @@ -475,6 +500,347 @@ fn closed(e: BatcherClosed) -> String { format!("bootstrap: {e}") } +fn decode_err(rel: &RelDescriptor, e: impl std::fmt::Display) -> String { + format!("bootstrap: {} tuple decode: {e}", rel.rel_name) +} + +/// Rows per load segment; the byte limit bounds wide rows +const LOAD_SEGMENT_ROWS: usize = 65536; + +/// Slabs queued for a lane's encoder +const LOAD_QUEUE: usize = 1; + +/// Routed load rows on their way to the encoder +#[derive(Default)] +struct Slab { + tables: Vec<(Arc, Arc)>, + rows: Vec, + bytes: usize, +} + +struct LoadRow { + seq: u64, + table: u32, + tuple: BackfillTuple, + /// Resolved external values + patch: Vec<(usize, ColumnValue)>, +} + +impl Slab { + fn push( + &mut self, + seq: u64, + rel: &Arc, + route: &Arc, + tuple: BackfillTuple, + patch: Vec<(usize, ColumnValue)>, + ) { + let table = match self + .tables + .iter() + .rposition(|(r, t)| Arc::ptr_eq(r, rel) && Arc::ptr_eq(t, route)) + { + Some(i) => i, + None => { + self.tables.push((rel.clone(), route.clone())); + self.tables.len() - 1 + } + }; + self.bytes += crate::backfill::spool::approx_bytes(&tuple) + + patch.iter().map(|(_, v)| v.approx_bytes()).sum::(); + self.rows.push(LoadRow { + seq, + table: table as u32, + tuple, + patch, + }); + } + + fn due(&self) -> bool { + self.rows.len() >= LOAD_SEGMENT_ROWS || self.bytes >= DECODE_CHUNK_BYTES + } +} + +/// `view` is scratch, refilled per heap row +fn encode_row<'r>( + chunks: &mut ChunkBuilder, + view: &mut TupleView<'r>, + tuple: &'r BackfillTuple, + rel: &Arc, + route: &Arc, + mut patch: Vec<(usize, ColumnValue)>, +) -> Result<(), String> { + let key = Arc::as_ptr(rel) as usize; + let stamp = RowStamp { + source_lsn: tuple.source_lsn, + xid: tuple.xid, + commit_ts: 0, + op_code: OP_INSERT, + leaf: route.partition.as_ref().map(|p| p.leaf), + }; + let appended = match &tuple.data { + TupleData::Heap(data) => { + view_block_data_into(data, rel, view).map_err(|e| decode_err(rel, e))?; + metadata_patch(tuple, rel, &mut patch); + chunks.append(key, rel, route, |enc| { + enc.append_view(rel, &view.datums, &patch, &stamp) + }) + } + TupleData::Copy(row) => chunks.append(key, rel, route, |enc| { + enc.append_slots( + |src| copy_slot(row, rel, src.index).map_err(EmitterError::Type), + &stamp, + ) + }), + }; + appended.map_err(|e| format!("bootstrap: encode {}: {e}", rel.rel_name)) +} + +enum LoadMsg { + /// Rows, then placements whose rows all precede them + Rows(Slab, Option>, Vec<(OpenSeq, u64)>), + /// Ship everything received, then reply + Flush(oneshot::Sender<()>), +} + +/// Routed load rows go to a lane encoder task, which ships them whatever +/// the router is waiting on and places each seq after its rows. Router +/// sends its slab before waiting on anything but the encoder +struct LoadOut { + tx: mpsc::Sender, + encoder: tokio::task::JoinHandle>, + seqs: SeqAlloc, + slab: Slab, + places: Vec<(OpenSeq, u64)>, + /// Leaf permit riding rows sent while held + permit: Option>, +} + +impl LoadOut { + fn new(msg_tx: &mpsc::Sender, ack: &AckHandle, first_seq: u64) -> Self { + let (tx, rx) = mpsc::channel(LOAD_QUEUE); + let encoder = Encoder { + msg_tx: msg_tx.clone(), + ack: ack.clone(), + state: Some(EncodeState::default()), + places: Vec::new(), + permit: None, + }; + Self { + tx, + encoder: tokio::spawn(encoder.run(rx)), + seqs: ack.seqs(first_seq), + slab: Slab::default(), + places: Vec::new(), + permit: None, + } + } + + fn hold(&mut self, permit: Option>) { + self.permit = permit; + } + + /// Queue `tuple` as an insert under `seq` + async fn push( + &mut self, + seq: u64, + rel: &Arc, + route: &Arc, + tuple: BackfillTuple, + patch: Vec<(usize, ColumnValue)>, + ) -> Result<(), String> { + self.slab.push(seq, rel, route, tuple, patch); + if self.slab.due() { + self.send().await?; + } + Ok(()) + } + + /// Hand queued rows and placements to the encoder + async fn send(&mut self) -> Result<(), String> { + if self.slab.rows.is_empty() && self.places.is_empty() { + return Ok(()); + } + let slab = std::mem::take(&mut self.slab); + let places = std::mem::take(&mut self.places); + let permit = self.permit.clone(); + self.post(LoadMsg::Rows(slab, permit, places)).await + } + + /// Report `rows` for `seq`, every one already pushed + fn place(&mut self, seq: OpenSeq, rows: u64) { + self.places.push((seq, rows)); + } + + /// Wait until every pushed row reached the batcher and every seq placed + async fn flush(&mut self) -> Result<(), String> { + self.send().await?; + let (reply, done) = oneshot::channel(); + self.post(LoadMsg::Flush(reply)).await?; + match done.await { + Ok(()) => Ok(()), + Err(_) => Err(self.failure().await), + } + } + + /// [`Self::flush`], then stop the encoder + async fn finish(mut self) -> Result<(), String> { + self.send().await?; + drop(self.tx); + self.encoder + .await + .map_err(|e| format!("bootstrap: encoder: {e}"))? + } + + async fn post(&mut self, msg: LoadMsg) -> Result<(), String> { + match self.tx.send(msg).await { + Ok(()) => Ok(()), + Err(_) => Err(self.failure().await), + } + } + + async fn failure(&mut self) -> String { + match (&mut self.encoder).await { + Ok(Err(e)) => e, + Ok(Ok(())) => "bootstrap: encoder stopped early".into(), + Err(e) => format!("bootstrap: encoder: {e}"), + } + } +} + +/// Builder state crossing to blocking encode jobs and back +struct EncodeState { + chunks: ChunkBuilder, + /// Seq of rows open in `chunks` + seq: u64, + /// Sealed segments awaiting shipment + ready: Vec, + ready_rows: usize, +} + +impl Default for EncodeState { + fn default() -> Self { + Self { + chunks: ChunkBuilder::new(LOAD_SEGMENT_ROWS, DECODE_CHUNK_BYTES), + seq: 0, + ready: Vec::new(), + ready_rows: 0, + } + } +} + +impl EncodeState { + /// Encode rows as inserts at their load version, sealing a segment at + /// each seq change and at `chunks`' limits + fn encode(&mut self, mut slab: Slab) -> Result<(), String> { + let mut view = TupleView::default(); + for row in &mut slab.rows { + if row.seq != self.seq { + self.seal()?; + self.seq = row.seq; + } + let patch = std::mem::take(&mut row.patch); + let (rel, route) = &slab.tables[row.table as usize]; + encode_row(&mut self.chunks, &mut view, &row.tuple, rel, route, patch)?; + if self.chunks.due() { + self.seal()?; + } + } + Ok(()) + } + + fn seal(&mut self) -> Result<(), String> { + let rows = self.chunks.rows(); + if rows == 0 { + return Ok(()); + } + let chunks = self + .chunks + .take() + .map_err(|e| format!("bootstrap: encode: {e}"))? + .into_iter() + .map(|(_, c)| c) + .collect(); + self.chunks.clear(); + self.ready.push(PlacedSegment { + seq: self.seq, + chunks, + permit: None, + }); + self.ready_rows += rows; + Ok(()) + } +} + +struct Encoder { + msg_tx: mpsc::Sender, + ack: AckHandle, + /// `None` while a blocking job holds it + state: Option, + places: Vec<(OpenSeq, u64)>, + permit: Option>, +} + +impl Encoder { + async fn run(mut self, mut rx: mpsc::Receiver) -> Result<(), String> { + while let Some(msg) = rx.recv().await { + match msg { + LoadMsg::Rows(slab, permit, places) => { + // A flush separates rows holding one permit from the next + self.permit = permit; + let mut state = self.state.take().expect("encode state home"); + let (state, encoded) = tokio::task::spawn_blocking(move || { + let encoded = state.encode(slab); + (state, encoded) + }) + .await + .map_err(|e| format!("bootstrap: encode job: {e}"))?; + encoded?; + self.places.extend(places); + let due = state.ready_rows >= LOAD_SEGMENT_ROWS; + self.state = Some(state); + if due { + self.ship().await?; + } + } + LoadMsg::Flush(reply) => { + self.ship().await?; + let _ = reply.send(()); + } + } + // Caught up: router may be waiting on something besides us + if rx.is_empty() { + self.ship().await?; + } + } + self.ship().await + } + + /// Send sealed and open rows, `permit` riding to insert ack, then place + /// seqs whose rows all shipped + async fn ship(&mut self) -> Result<(), String> { + let state = self.state.as_mut().expect("encode state home"); + state.seal()?; + // Every row it covered ships now; holding it longer stalls next acquire + let permit = self.permit.take(); + if !state.ready.is_empty() { + let mut segments = std::mem::take(&mut state.ready); + state.ready_rows = 0; + for s in &mut segments { + s.permit = permit.clone(); + } + self.msg_tx + .send(BatcherMsg::Segments(segments)) + .await + .map_err(|_| closed(BatcherClosed))?; + } + for (seq, rows) in self.places.drain(..) { + self.ack.place(seq, rows); + } + Ok(()) + } +} + struct Run { rfn: RelFileNode, seq: OpenSeq, @@ -487,9 +853,14 @@ enum Target { Unsupported, /// Already loaded by caller Skipped, - /// Store TOAST chunks under this relation OID - Toast(u32), - Routed(Arc, Arc), + /// Store TOAST chunks of this relation + Toast(Arc), + Routed { + rel: Arc, + route: Arc, + /// Some mapped column is varlena, so a tuple may hold external values + varlena: bool, + }, } impl Target { @@ -506,11 +877,22 @@ impl Target { return Self::Skipped; } if catalog.is_toast(rfn.db_node, rfn.rel_node) { - return Self::Toast(rel.oid); + return Self::Toast(rel); + } + let Some(route) = routes.route(&rel) else { + return Self::Unsupported; + }; + let varlena = route.mapping.columns.iter().any(|c| { + usize::try_from(c.src_attnum as i32 - 1) + .ok() + .and_then(|i| rel.attributes.get(i)) + .is_some_and(|a| a.type_len == -1) + }); + Self::Routed { + rel, + route, + varlena, } - routes - .route(&rel) - .map_or(Self::Unsupported, |route| Self::Routed(rel, route)) } } @@ -540,6 +922,8 @@ struct ReplayRow { /// Index into containing batch's `tables` table: u32, pointers: Vec, + /// Values settled for external attributes + patch: Vec<(usize, ColumnValue)>, replay_covered: bool, } @@ -552,28 +936,34 @@ struct PointerSite { } /// Check only columns routed to ClickHouse -fn mapped_pointers(tuple: &BackfillTuple, mapping: &TableMapping) -> Vec { +fn mapped_pointers(view: &TupleView<'_>, mapping: &TableMapping) -> Vec { mapping .columns .iter() .enumerate() .filter_map(|(col, c)| { let idx = usize::try_from(c.src_attnum as i32 - 1).ok()?; - let Some(ColumnValue::ExternalToast(p)) = tuple.columns.get(idx)? else { + let DatumRef::External(p) = *view.datums.get(idx)? else { return None; }; - Some(PointerSite { idx, col, p: *p }) + Some(PointerSite { idx, col, p }) }) .collect() } -/// Replace oversized values with NULL and remove them from fetch list +/// Settle oversized values as NULL and remove them from fetch list fn fill_oversize( - tuple: &mut BackfillTuple, sites: &mut Vec, + patch: &mut Vec<(usize, ColumnValue)>, resolver: &ToastResolver, ) { - sites.retain(|site| !resolver.fill_oversize(&mut tuple.columns[site.idx], &site.p)); + sites.retain(|site| { + let filled = resolver.fills_oversize(&site.p); + if filled { + patch.push((site.idx, ColumnValue::Null)); + } + !filled + }); } /// Batch whose values are resolved, holding the leaf permit its rows ride @@ -615,8 +1005,7 @@ async fn prepare_batch( /// batch's leaf permit, which the next batch's acquire may wait on async fn route_batch( batch: ResolvedReplayBatch, - msg_tx: &mpsc::Sender, - seqs: &mut SeqAlloc, + out: &mut LoadOut, seq: &mut Option, placed: &mut u64, ) -> Result<(), String> { @@ -625,30 +1014,21 @@ async fn route_batch( rows, permit, } = batch; - let mut row_buffer = RowBuffer::new(permit); + out.hold(permit); for row in rows { if row.replay_covered { continue; } - let mut tuple = row.tuple; - let at = seq - .get_or_insert_with(|| seqs.open(tuple.source_lsn, Publish::Commit)) + let s = seq + .get_or_insert_with(|| out.seqs.open(row.tuple.source_lsn, Publish::Commit)) .seq(); let RowTable { rel, route } = &tables[row.table as usize]; - render_ext_columns(&rel.attributes, &mut tuple.columns); - row_buffer - .push( - msg_tx, - at, - (rel, route), - tuple.into_committed_insert(), - None, - ) - .await - .map_err(closed)?; + out.push(s, rel, route, row.tuple, row.patch).await?; *placed += 1; } - row_buffer.flush(msg_tx).await.map_err(closed) + out.flush().await?; + out.hold(None); + Ok(()) } /// Read referrers until the batch seals or the spool ends, dropping rows no @@ -668,7 +1048,7 @@ async fn next_batch( }; let mut run: Option<(RelFileNode, Option)> = None; while batch.rows.len() < REPLAY_BATCH_ROWS && batch.need + batch.resident < REPLAY_FETCH_BYTES { - let Some(mut tuple) = replay + let Some(tuple) = replay .next() .await .map_err(|e| format!("bootstrap: deferred spool replay: {e}"))? @@ -694,20 +1074,25 @@ async fn next_batch( stats.unsupported_relations.fetch_add(1, Ordering::Relaxed); continue; }; - let route = &batch.tables[table as usize].route; - let mut pointers = mapped_pointers(&tuple, &route.mapping); + let RowTable { rel, route } = &batch.tables[table as usize]; + let mut pointers = tuple + .view(rel) + .map_err(|e| decode_err(rel, e))? + .map_or_else(Vec::new, |v| mapped_pointers(&v, &route.mapping)); batch.need += check_value_caps( pointers.iter().map(|site| site.p), resolver.inline_value_max(), resolver.overflow(), ) .map_err(|e| format!("bootstrap: {e}"))?; - fill_oversize(&mut tuple, &mut pointers, resolver); + let mut patch = Vec::new(); + fill_oversize(&mut pointers, &mut patch, resolver); batch.resident += crate::backfill::spool::approx_bytes(&tuple); batch.rows.push(ReplayRow { tuple, table, pointers, + patch, replay_covered: false, }); } @@ -761,6 +1146,7 @@ async fn resolve_batch( tuple, table, pointers, + patch, replay_covered, }, fetched, @@ -779,7 +1165,7 @@ async fn resolve_batch( let target = &route.mapping.columns[site.col].target_name; let (column, bytes) = apply_fetched(fetched, &site.p, type_oid, rel, target, resolver)?; retained += bytes; - tuple.columns[site.idx] = Some(column); + patch.push((site.idx, column)); } } Ok(retained) @@ -899,48 +1285,28 @@ fn apply_fetched( /// No store to consult: NULL-fill mapped TOAST pointers after the value /// cap check, counting each fill fn fill_toast( - tuple: &mut BackfillTuple, - mapping: &TableMapping, + mut sites: Vec, resolver: &ToastResolver, -) -> Result<(), String> { - let mut sites = mapped_pointers(tuple, mapping); +) -> Result, String> { check_value_caps( sites.iter().map(|site| site.p), resolver.inline_value_max(), resolver.overflow(), ) .map_err(|e| format!("bootstrap: {e}"))?; - fill_oversize(tuple, &mut sites, resolver); + let mut patch = Vec::with_capacity(sites.len()); + fill_oversize(&mut sites, &mut patch, resolver); for site in &sites { resolver.note_filled_default(); - tuple.columns[site.idx] = Some(ColumnValue::Null); + patch.push((site.idx, ColumnValue::Null)); } - Ok(()) + Ok(patch) } fn put_err(e: ChunkStoreError) -> String { format!("bootstrap: toast store put: {e}") } -/// Convert PostgreSQL TOAST tuple shape into mirror row -fn row_from_columns(mut tuple: BackfillTuple, db_oid: u32, toast_relid: u32) -> Option { - let (chunk_id, chunk_seq, chunk_data) = - crate::decode::heap_decoder::take_toast_chunk_columns(&mut tuple.columns)?; - debug_assert_ne!(tuple.offnum, 0, "walked toast tuple without TID"); - Some(ToastRow { - db_oid, - toast_relid, - blkno: tuple.blkno, - offnum: tuple.offnum, - chunk_id, - chunk_seq, - chunk_data: bytes::Bytes::from(chunk_data), - // Weakest evidence for a TID: a backup copy can be torn, so any - // WAL-sourced row for the same TID outranks it - lsn: 0, - }) -} - #[cfg(test)] mod tests { #[test] @@ -998,18 +1364,33 @@ mod tests { ); } - /// Flatten the drain's coalesced `Rows` chunks back to a row list - async fn collect_rows(rx: &mut mpsc::Receiver) -> Vec { + /// Flatten the drain's segments back to `(seq, mapped cells)` rows + async fn collect_rows(rx: &mut mpsc::Receiver) -> Vec<(u64, Vec)> { let mut rows = Vec::new(); // Any other message ends collection short of the expected rows - while let Some(BatcherMsg::Rows(chunk)) = rx.recv().await { - rows.extend(chunk.rows); + while let Some(BatcherMsg::Segments(segments)) = rx.recv().await { + rows.extend(segment_rows(&segments)); } rows } + /// `_lsn` per row of `chunk` + fn lsn_column(chunk: &crate::emit::ch_emitter::EncodedChunk) -> Vec { + let lsn = &chunk.buffers[chunk.route.mapping.columns.len()]; + let crate::emit::ch_emitter::Wire::Fixed { bytes, .. } = lsn.wire() else { + panic!("UInt64 _lsn"); + }; + bytes + .as_chunks::<8>() + .0 + .iter() + .map(|b| u64::from_le_bytes(*b)) + .collect() + } + use super::*; - use crate::emit::pipeline::batcher::RoutedRow; + use crate::emit::pipeline::batcher::segment_rows; + use crate::toast::ToastRow; /// Distinguish missing data in walk-seeded store from superseded value in /// read-only end-of-backup store @@ -1163,7 +1544,8 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: vec![Some(ColumnValue::Int4(id))], + data: TupleData::from_cells(&[Some(ColumnValue::Int4(id))]), + pending: None, } } @@ -1207,7 +1589,7 @@ mod tests { columns: vec![ColumnMapping { src_attnum: 1, target_name: "b".into(), - target_type: "String".into(), + target_type: "Nullable(String)".into(), type_pinned: false, }], tees: Vec::new(), @@ -1215,10 +1597,9 @@ mod tests { } /// `pg_toast` rel so [`CatalogMap::is_toast`] fires; `oid` matches the - /// referring pointer's `va_toastrelid`. Attributes unread — the drain - /// reinterprets a toast tuple's columns positionally (`chunk_from_columns`). + /// referring pointer's `va_toastrelid` fn toast_rel(rel_node: u32) -> Arc { - let name = format!("pg_toast_{rel_node}"); + let base = crate::backfill::backup_page_walk::toast_chunk_rel(); Arc::new(RelDescriptor { rfn: RelFileNode { spc_node: 1663, @@ -1226,14 +1607,8 @@ mod tests { rel_node, }, oid: rel_node, - toast_oid: 0, - namespace_oid: 99, - rel_name: RelName::new("pg_toast", &name), - kind: 't', - persistence: 'p', - ancestors: Vec::new(), - replident: ReplIdent::Default { pk_attnums: None }, - attributes: vec![], + rel_name: RelName::new("pg_toast", &format!("pg_toast_{rel_node}")), + ..base }) } @@ -1241,6 +1616,24 @@ mod tests { /// into `toast_relid`/`value_id`, uncompressed (`va_extinfo` high bits /// clear so reassembly returns the concatenated chunks verbatim). fn bytea_toast_tuple(rel_node: u32, toast_relid: u32, value_id: u32) -> BackfillTuple { + referrer_tuple(rel_node, &[pointer(toast_relid, value_id)]) + } + + fn pointer(toast_relid: u32, value_id: u32) -> ToastPointer { + ToastPointer { + va_rawsize: 9, + va_extinfo: 5, + va_valueid: value_id, + va_toastrelid: toast_relid, + } + } + + /// Main-rel tuple whose columns are on-disk TOAST pointers + fn referrer_tuple(rel_node: u32, pointers: &[ToastPointer]) -> BackfillTuple { + let cells: Vec<_> = pointers + .iter() + .map(|p| Some(ColumnValue::ExternalToast(*p))) + .collect(); BackfillTuple { rfn: RelFileNode { spc_node: 1663, @@ -1254,12 +1647,8 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - columns: vec![Some(ColumnValue::ExternalToast(ToastPointer { - va_rawsize: 9, - va_extinfo: 5, - va_valueid: value_id, - va_toastrelid: toast_relid, - }))], + data: TupleData::from_cells(&cells), + pending: None, } } @@ -1279,20 +1668,21 @@ mod tests { page_lsn: 0, blkno: 1, offnum: 1, - columns: vec![ + data: TupleData::from_cells(&[ Some(ColumnValue::Oid(value_id)), Some(ColumnValue::Int4(seq)), Some(ColumnValue::Bytea(body.to_vec())), - ], + ]), + pending: None, } } #[tokio::test] async fn detoasted_input_rejects_external_pointers() { let mut catalog = CatalogMap::new(); - catalog.insert(rel(16400)); + catalog.insert(bytea_rel(16400)); let mapping = Arc::new( - [(RelName::new("public", "t16400"), mapping_for(16400))] + [(RelName::new("public", "t16400"), bytea_mapping_for(16400))] .into_iter() .collect(), ); @@ -1363,7 +1753,7 @@ mod tests { let mut by_seq: HashMap = HashMap::new(); for r in collect_rows(&mut msg_rx).await { - *by_seq.entry(r.seq).or_default() += 1; + *by_seq.entry(r.0).or_default() += 1; } let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 2, "one seq per rfn"); @@ -1416,7 +1806,7 @@ mod tests { let seqs: Vec = collect_rows(&mut msg_rx) .await .iter() - .map(|r| r.seq) + .map(|r| r.0) .collect(); let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.next_seq, 3, "three distinct rfn runs"); @@ -1431,7 +1821,10 @@ mod tests { tuples: Vec, skip_initial: HashSet, batcher_open: bool, - ) -> (Result, Vec) { + ) -> ( + Result, + Vec<(u64, Vec)>, + ) { let mut catalog = CatalogMap::new(); catalog.insert(rel(16400)); catalog.insert(toast_rel(16500)); @@ -1468,7 +1861,9 @@ mod tests { #[tokio::test] async fn skipped_relations_and_malformed_chunks_route_nothing() { let skip = [RelName::new("public", "t16400")].into_iter().collect(); - let tuples = vec![tuple(16400, 1), tuple(16500, 2)]; + let mut malformed = tuple(16500, 2); + malformed.offnum = 1; + let tuples = vec![tuple(16400, 1), malformed]; let (outcome, rows) = drain_plain(tuples, skip, true).await; assert_eq!(outcome.unwrap().rows_routed, 0); assert!(rows.is_empty()); @@ -1613,12 +2008,8 @@ mod tests { assert_eq!(outcome.next_seq, 1); assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); - let cols = &rows[0].committed.decoded.new.as_ref().unwrap().columns; - assert_eq!( - cols[0], - Some(ColumnValue::Null), - "unresolved toast NULL-filled" - ); + let cols = &rows[0].1; + assert_eq!(cols[0], ColumnValue::Null, "unresolved toast NULL-filled"); assert_eq!(stats.toast_values_filled_default.load(Ordering::Relaxed), 1); drop(ack); collector.await.unwrap(); @@ -1666,8 +2057,8 @@ mod tests { let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); - let cols = &rows[0].committed.decoded.new.as_ref().unwrap().columns; - assert_eq!(cols[0], Some(ColumnValue::Null)); + let cols = &rows[0].1; + assert_eq!(cols[0], ColumnValue::Null); assert_eq!( stats.toast_values_filled_oversize.load(Ordering::Relaxed), 1 @@ -1732,9 +2123,9 @@ mod tests { assert_eq!(outcome.next_seq, 3, "toast seq, main seq, deferred seq"); assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); - assert_eq!(rows[0].seq, 2, "deferred row routed under the trailing seq"); - let cols = &rows[0].committed.decoded.new.as_ref().unwrap().columns; - assert_eq!(cols[0], Some(ColumnValue::Bytea(b"hello".to_vec()))); + assert_eq!(rows[0].0, 2, "deferred row routed under the trailing seq"); + let cols = &rows[0].1; + assert_eq!(cols[0], ColumnValue::Bytea(b"hello".to_vec())); assert_eq!(stats.toast_chunks_stored.load(Ordering::Relaxed), 1); assert_eq!(stats.toast_values_fetched.load(Ordering::Relaxed), 1); drop(ack); @@ -1796,7 +2187,7 @@ mod tests { let msg = tokio::time::timeout(std::time::Duration::from_secs(5), msg_rx.recv()) .await .expect("heap rows stuck behind gated put"); - assert!(matches!(&msg, Some(BatcherMsg::Rows(c)) if c.rows.len() == 2)); + assert!(matches!(&msg, Some(BatcherMsg::Segments(s)) if segment_rows(s).len() == 2)); assert_eq!(store.permits.available_permits(), 0, "no put landed yet"); store.permits.add_permits(2); @@ -1949,8 +2340,8 @@ mod tests { let rows = collect_rows(&mut msg_rx).await; assert_eq!(rows.len(), 1); - let cols = &rows[0].committed.decoded.new.as_ref().unwrap().columns; - assert_eq!(cols[0], Some(ColumnValue::Bytea(b"hello".to_vec()))); + let cols = &rows[0].1; + assert_eq!(cols[0], ColumnValue::Bytea(b"hello".to_vec())); drop(ack); collector.await.unwrap(); } @@ -2079,12 +2470,7 @@ mod tests { let (ack, collector) = ack::spawn(Arc::new(crate::pos::Monotone::default()), Fatal::new()); let (msg_tx, mut msg_rx) = mpsc::channel::(64); - let mut referrer = bytea_toast_tuple(16400, 16500, b); - referrer.columns.push(referrer.columns[0].clone()); - let Some(Some(ColumnValue::ExternalToast(p))) = referrer.columns.get_mut(1) else { - unreachable!() - }; - p.va_valueid = c; + let mut referrer = referrer_tuple(16400, &[pointer(16500, b), pointer(16500, c)]); (referrer.blkno, referrer.offnum) = (slot.1, slot.2); referrer.page_lsn = page_lsn; let mut spool = mem_spool(); @@ -2105,25 +2491,30 @@ mod tests { ) .await; drop(msg_tx); - let routed = collect_rows(&mut msg_rx).await; + let mut routed = Vec::new(); + while let Some(msg) = msg_rx.recv().await { + routed.push(msg); + } let covered = stats.bootstrap_replay_covered_rows.load(Ordering::Relaxed); match want { Want::Routed(b, c) => { let resolved = result.unwrap_or_else(|e| panic!("{label}: {e}")); assert_eq!(resolved.rows_routed, 1, "{label}"); - let row = &routed[0].committed; + let BatcherMsg::Segments(segments) = &routed[0] else { + panic!("{label}: expected segments"); + }; + let chunk = &segments[0].chunks[0]; assert_eq!( - (row.decoded.source_lsn, row.commit_lsn), - (0x1000, 0x1000), + lsn_column(chunk), + [0x1000], "{label}: load version moved with lookup bound" ); - let cols = &row.decoded.new.as_ref().unwrap().columns; assert_eq!( - cols[..], - [ - Some(ColumnValue::Bytea(b.to_vec())), - Some(ColumnValue::Bytea(c.to_vec())) - ], + chunk.cells(), + [[ + ColumnValue::Bytea(b.to_vec()), + ColumnValue::Bytea(c.to_vec()) + ]], "{label}" ); assert_eq!(covered, 0, "{label}"); @@ -2273,8 +2664,8 @@ mod tests { let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); - let cols = &rows[0].committed.decoded.new.as_ref().unwrap().columns; - assert_eq!(cols[0], Some(ColumnValue::Bytea(b"hello".to_vec()))); + let cols = &rows[0].1; + assert_eq!(cols[0], ColumnValue::Bytea(b"hello".to_vec())); drop(ack); collector.await.unwrap(); } @@ -2387,14 +2778,8 @@ mod tests { let routed = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed as usize, rows); - let got: Vec<_> = routed - .iter() - .map(|r| r.committed.decoded.new.as_ref().unwrap().columns[0].clone()) - .collect(); - let want: Vec<_> = bodies - .into_iter() - .map(|b| Some(ColumnValue::Bytea(b))) - .collect(); + let got: Vec<_> = routed.iter().map(|r| r.1[0].clone()).collect(); + let want: Vec<_> = bodies.into_iter().map(ColumnValue::Bytea).collect(); assert_eq!(got, want, "replay order kept across the batch boundary"); assert_eq!( stats.toast_value_fetch_batches.load(Ordering::Relaxed), @@ -2433,12 +2818,12 @@ mod tests { tup_tx.send(chunks).await.unwrap(); let referrers = (0..rows) .map(|i| { - let mut t = bytea_toast_tuple(16400, 16500, i + 1); - if let Some(ColumnValue::ExternalToast(p)) = &mut t.columns[0] { - p.va_rawsize = VALUE as i32 + 4; - p.va_extinfo = VALUE as u32; - } - t + let p = ToastPointer { + va_rawsize: VALUE as i32 + 4, + va_extinfo: VALUE as u32, + ..pointer(16500, i + 1) + }; + referrer_tuple(16400, &[p]) }) .collect(); tup_tx.send(referrers).await.unwrap(); @@ -2465,8 +2850,8 @@ mod tests { // Drop rows on receipt, as insert ack would let mut routed = 0; let count = async { - while let Some(BatcherMsg::Rows(chunk)) = msg_rx.recv().await { - routed += chunk.rows.len(); + while let Some(BatcherMsg::Segments(segments)) = msg_rx.recv().await { + routed += segment_rows(&segments).len(); } }; tokio::time::timeout(std::time::Duration::from_secs(5), count) @@ -2618,10 +3003,13 @@ mod tests { for (db_oid, body) in [(1, b"first"), (2, b"other")] { let store = Arc::new(MemChunkStore::new()); let chunk = toast_chunk_tuple(16500, 1, 0, body); - store - .put(&[row_from_columns(chunk, db_oid, 16500).unwrap()]) - .await - .unwrap(); + let TupleData::Heap(data) = &chunk.data else { + unreachable!() + }; + let mut toast = (*toast_rel(16500)).clone(); + toast.rfn.db_node = db_oid; + let row = toast_row(data, &toast, chunk.blkno, chunk.offnum, 0).unwrap(); + store.put(&[row]).await.unwrap(); stores.push((db_oid, store as Arc)); let mut tuple = bytea_toast_tuple(16400, 16500, 1); tuple.rfn.db_node = db_oid; @@ -2630,15 +3018,16 @@ mod tests { Arc::default(), RowPolicy::default(), ); - let pointers = mapped_pointers(&tuple, &route.mapping); - batch.tables.push(RowTable { - rel: bytea_rel(16400), - route, - }); + let rel = bytea_rel(16400); + let view = tuple.view(&rel).unwrap().unwrap(); + let pointers = mapped_pointers(&view, &route.mapping); + drop(view); + batch.tables.push(RowTable { rel, route }); batch.rows.push(ReplayRow { tuple, table: batch.tables.len() as u32 - 1, pointers, + patch: Vec::new(), replay_covered: false, }); } @@ -2648,10 +3037,7 @@ mod tests { .await .unwrap(); for (row, expected) in batch.rows.iter().zip([b"first", b"other"]) { - assert_eq!( - row.tuple.columns[0], - Some(ColumnValue::Bytea(expected.to_vec())) - ); + assert_eq!(row.patch, [(0, ColumnValue::Bytea(expected.to_vec()))]); } } @@ -2729,19 +3115,16 @@ mod tests { let rows = collect_rows(&mut msg_rx).await; let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed, 4); - let bodies: Vec<_> = rows - .iter() - .map(|r| r.committed.decoded.new.as_ref().unwrap().columns[0].clone()) - .collect(); + let bodies: Vec<_> = rows.iter().map(|r| r.1[0].clone()).collect(); assert_eq!( bodies, [b"hello", b"hello", b"world", b"there"] - .map(|b| Some(ColumnValue::Bytea(b.to_vec()))) + .map(|b| ColumnValue::Bytea(b.to_vec())) .to_vec(), "replay order kept, shared value resolved for both referrers", ); assert!( - rows.iter().all(|r| r.seq == rows[0].seq), + rows.iter().all(|r| r.0 == rows[0].0), "one trailing seq for the replay", ); assert_eq!( diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index 6e17426d..a95a5656 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -28,7 +28,7 @@ use thiserror::Error; use crate::ch::EmitterError; use crate::decode::datum::DatumRef; use crate::decode::heap_decoder::{ - DecodedHeap, HeapOp, TupleView, datum_value, take_key_change_tombstone, + ColumnValue, DecodedHeap, HeapOp, TupleView, datum_value, take_key_change_tombstone, }; use crate::emit::ch_emitter::{ Append, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, RowStamp, TableEncoder, TablePlan, @@ -163,9 +163,11 @@ enum Step { }, } -/// Per-table encoder of the open segment, keyed by plan table -struct OpenTable { - table: u32, +/// Per-table encoder of the open segment +struct OpenTable { + key: K, + rel: Arc, + route: Arc, enc: TableEncoder, } @@ -206,6 +208,217 @@ impl EncoderCache { } } +/// Encodes row changes into per-table column chunks of one open segment, +/// sealing a table's chunk early when an oracle frame fills. `K` names the +/// caller's table +pub struct ChunkBuilder { + alloc: Allocator, + open: Vec>, + encoders: EncoderCache, + /// Chunks of the open segment sealed early at an oracle frame limit + sealed: Vec<(K, EncodedChunk)>, + rows: usize, + segment_rows: usize, + segment_bytes: usize, +} + +impl ChunkBuilder { + /// Segment is due once its rows or allocated bytes reach these + pub fn new(segment_rows: usize, segment_bytes: usize) -> Self { + Self { + alloc: Allocator::global(&mimalloc::MiMalloc), + open: Vec::new(), + encoders: EncoderCache::default(), + sealed: Vec::new(), + rows: 0, + segment_rows: segment_rows.max(1), + segment_bytes: segment_bytes.max(1), + } + } + + /// Reuse encoders an earlier builder returned + pub fn with_encoders(mut self, encoders: EncoderCache) -> Self { + self.encoders = encoders; + self + } + + /// Rows in the open segment + pub fn rows(&self) -> usize { + self.rows + } + + /// Whether the open segment reached its row or byte limit + pub fn due(&self) -> bool { + self.rows >= self.segment_rows || self.open_bytes() >= self.segment_bytes + } + + fn open_bytes(&self) -> usize { + self.open.iter().map(|o| o.enc.approx_bytes).sum::() + + self + .sealed + .iter() + .map(|(_, c)| c.allocated_bytes()) + .sum::() + } + + /// Encode one row into `key`'s open chunk, sealing it first when an + /// oracle frame fills + pub(crate) fn append( + &mut self, + key: K, + rel: &Arc, + route: &Arc, + mut row: impl FnMut(&mut TableEncoder) -> std::result::Result, + ) -> std::result::Result<(), EmitterError> { + let i = match self.open.iter().position(|o| o.key == key) { + Some(i) => i, + None => { + let enc = match self.encoders.take(rel, route) { + Some(enc) => enc, + None => TableEncoder::new(TablePlan::for_route(self.alloc, rel, route)?)?, + }; + self.open.push(OpenTable { + key, + rel: rel.clone(), + route: route.clone(), + enc, + }); + self.open.len() - 1 + } + }; + if row(&mut self.open[i].enc)? == Append::Full { + let chunk = self.take_chunk(i)?; + self.sealed.push(chunk); + row(&mut self.open[i].enc)?; + } + self.rows += 1; + Ok(()) + } + + /// Encode one owned row change: drop deletes an append-only route has + /// nowhere to land, render PostGIS cells, split key changes into a + /// delete marker plus the new row. Returns rows appended + pub fn push_cells( + &mut self, + key: K, + rel: &Arc, + route: &Arc, + mut decoded: DecodedHeap, + commit_ts: i64, + ) -> std::result::Result { + if route.drops_deletes() && decoded.op == HeapOp::Delete { + return Ok(0); + } + for t in [decoded.new.as_mut(), decoded.old.as_mut()] + .into_iter() + .flatten() + { + crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); + } + let tomb = (!route.drops_deletes()) + .then(|| take_key_change_tombstone(&mut decoded, rel)) + .flatten(); + let leaf = route.partition.as_ref().map(|p| p.leaf); + let mut rows = 0; + for decoded in tomb.into_iter().chain([decoded]) { + let op = op_code(decoded.op)?; + let committed = crate::decode::heap_decoder::CommittedTuple { + decoded, + commit_ts, + commit_lsn: 0, + }; + self.append(key, rel, route, |enc| { + enc.append_routed(&committed, &route.mapping, op, leaf) + })?; + rows += 1; + } + Ok(rows) + } + + /// [`Self::push_cells`] reading tuple images in place + pub fn push_views( + &mut self, + key: K, + rel: &Arc, + route: &Arc, + change: ViewChange<'_, '_>, + commit_ts: i64, + ) -> std::result::Result { + if route.drops_deletes() && change.op == HeapOp::Delete { + return Ok(0); + } + let leaf = route.partition.as_ref().map(|p| p.leaf); + let stamp = |source_lsn, op_code| RowStamp { + source_lsn, + xid: change.xid, + commit_ts, + op_code, + leaf, + }; + let mut rows = 0; + if !route.drops_deletes() + && let (Some(old), Some(new)) = (change.old, change.new) + && key_moved(rel, &change, old, new) + { + let stamp = stamp(change.source_lsn - 1, OP_DELETE); + self.append(key, rel, route, |enc| { + enc.append_view(rel, &old.datums, change.old_patch, &stamp) + })?; + rows += 1; + } + let (side, patch) = match change.op { + HeapOp::Delete => (change.old, change.old_patch), + _ => (change.new, change.new_patch), + }; + let stamp = stamp(change.source_lsn, op_code(change.op)?); + let datums = side.map_or(&[][..], |t| &t.datums); + self.append(key, rel, route, |enc| { + enc.append_view(rel, datums, patch, &stamp) + })?; + Ok(rows + 1) + } + + fn take_chunk(&mut self, i: usize) -> std::result::Result<(K, EncodedChunk), EmitterError> { + let o = &mut self.open[i]; + let (buffers, rows) = o.enc.take_block()?; + let chunk = EncodedChunk { + rel: o.rel.clone(), + route: o.route.clone(), + buffers, + rows, + }; + Ok((o.key, chunk)) + } + + /// Seal the open segment's chunks, early-sealed ones first + pub fn take(&mut self) -> std::result::Result, EmitterError> { + let mut chunks = std::mem::take(&mut self.sealed); + for i in 0..self.open.len() { + if self.open[i].enc.rows > 0 { + chunks.push(self.take_chunk(i)?); + } + } + self.rows = 0; + Ok(chunks) + } + + /// Park open encoders, emptied by [`Self::take`], so later keys start + /// fresh. A control entry may change routes, and the cache matches + /// route identity + pub fn clear(&mut self) { + debug_assert_eq!(self.rows, 0, "clear after take"); + for o in self.open.drain(..) { + self.encoders.put(o.rel, o.route, o.enc); + } + } + + /// Emptied encoders for the next builder + pub fn into_encoders(mut self) -> EncoderCache { + self.clear(); + self.encoders + } +} + /// Streams one transaction's validated rows into column chunks, in memory /// or on disk, while accumulating the bounded plan header. Dropped unsealed /// (any planning error), the file unlinks and nothing replays @@ -216,7 +429,6 @@ pub struct PlanWriter { cap: u64, /// Resident chunk bytes while in memory, file bytes once spilled bytes: u64, - alloc: Allocator, tables: Vec, /// Tables since the last control entry, which may change routes live_from: usize, @@ -226,14 +438,8 @@ pub struct PlanWriter { /// Arc in `tables` keeps each pointer live and unique. Built once a /// second live table appears, so single-relation plans never hash table_ids: Option>, - open: Vec, - encoders: EncoderCache, - /// Chunks of the open segment sealed early at an oracle frame limit - sealed: Vec<(u32, EncodedChunk)>, + chunks: ChunkBuilder, commit_ts: i64, - open_rows: usize, - segment_rows: usize, - segment_bytes: usize, steps: Vec, row_batches: Vec, routed_rows: u64, @@ -251,18 +457,12 @@ impl PlanWriter { mem_max, cap, bytes: 4, - alloc: Allocator::global(&mimalloc::MiMalloc), tables: Vec::new(), live_from: 0, last_table: 0, table_ids: None, - open: Vec::new(), - encoders: EncoderCache::default(), - sealed: Vec::new(), + chunks: ChunkBuilder::new(usize::MAX, usize::MAX), commit_ts: 0, - open_rows: 0, - segment_rows: usize::MAX, - segment_bytes: usize::MAX, steps: Vec::new(), row_batches: Vec::new(), routed_rows: 0, @@ -273,8 +473,8 @@ impl PlanWriter { /// Seal a segment once its open rows or bytes reach these pub fn with_segment_limits(mut self, rows: usize, bytes: usize) -> Self { - self.segment_rows = rows.max(1); - self.segment_bytes = bytes.max(1); + self.chunks.segment_rows = rows.max(1); + self.chunks.segment_bytes = bytes.max(1); self } @@ -286,7 +486,7 @@ impl PlanWriter { /// Reuse encoders an earlier plan returned through [`Self::seal_keeping`] pub fn with_encoders(mut self, encoders: EncoderCache) -> Self { - self.encoders = encoders; + self.chunks.encoders = encoders; self } @@ -345,138 +545,44 @@ impl PlanWriter { self.tables[table as usize].heaps += rows; } - /// Encode one owned row change: drop deletes an append-only route has - /// nowhere to land, render PostGIS cells, split key changes into a - /// delete marker plus the new row - pub fn push_cells(&mut self, mut decoded: DecodedHeap, table: u32) -> Result<()> { + /// Encode one owned row change, see [`ChunkBuilder::push_cells`] + pub fn push_cells(&mut self, decoded: DecodedHeap, table: u32) -> Result<()> { let t = &mut self.tables[table as usize]; t.heaps += 1; - let (rel, route) = (t.descriptor.clone(), t.route.clone().expect("routed table")); - if route.drops_deletes() && decoded.op == HeapOp::Delete { - self.deletes_discarded += 1; - return Ok(()); - } - for t in [decoded.new.as_mut(), decoded.old.as_mut()] - .into_iter() - .flatten() - { - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); - } - let tomb = (!route.drops_deletes()) - .then(|| take_key_change_tombstone(&mut decoded, &rel)) - .flatten(); - let leaf = route.partition.as_ref().map(|p| p.leaf); - for decoded in tomb.into_iter().chain([decoded]) { - let op = op_code(decoded.op)?; - let committed = crate::decode::heap_decoder::CommittedTuple { - decoded, - commit_ts: self.commit_ts, - commit_lsn: 0, - }; - self.append(table, |enc| { - enc.append_routed(&committed, &route.mapping, op, leaf) - })?; - } - Ok(()) + let route = t.route.as_ref().expect("routed table"); + let rows = self + .chunks + .push_cells(table, &t.descriptor, route, decoded, self.commit_ts)?; + self.pushed(rows) } /// [`Self::push_cells`] reading tuple images in place pub fn push_views(&mut self, change: ViewChange<'_, '_>, table: u32) -> Result<()> { let t = &mut self.tables[table as usize]; t.heaps += 1; - let (rel, route) = (t.descriptor.clone(), t.route.clone().expect("routed table")); - if route.drops_deletes() && change.op == HeapOp::Delete { - self.deletes_discarded += 1; - return Ok(()); - } - let leaf = route.partition.as_ref().map(|p| p.leaf); - let commit_ts = self.commit_ts; - let stamp = |source_lsn, op_code| RowStamp { - source_lsn, - xid: change.xid, - commit_ts, - op_code, - leaf, - }; - if !route.drops_deletes() - && let (Some(old), Some(new)) = (change.old, change.new) - && key_moved(&rel, change.op, old, new) - { - let stamp = stamp(change.source_lsn - 1, OP_DELETE); - self.append(table, |enc| enc.append_view(&rel, &old.datums, &stamp))?; - } - let side = match change.op { - HeapOp::Delete => change.old, - _ => change.new, - }; - let stamp = stamp(change.source_lsn, op_code(change.op)?); - let datums = side.map_or(&[][..], |t| &t.datums); - self.append(table, |enc| enc.append_view(&rel, datums, &stamp)) + let route = t.route.as_ref().expect("routed table"); + let rows = self + .chunks + .push_views(table, &t.descriptor, route, change, self.commit_ts)?; + self.pushed(rows) } - /// Encode into `table`'s open chunk, sealing it first when an oracle - /// frame fills - fn append( - &mut self, - table: u32, - mut row: impl FnMut(&mut TableEncoder) -> std::result::Result, - ) -> Result<()> { - let i = match self.open.iter().position(|o| o.table == table) { - Some(i) => i, - None => { - let t = &self.tables[table as usize]; - let route = t.route.as_ref().expect("routed table"); - let enc = match self.encoders.take(&t.descriptor, route) { - Some(enc) => enc, - None => { - TableEncoder::new(TablePlan::for_route(self.alloc, &t.descriptor, route)?)? - } - }; - self.open.push(OpenTable { table, enc }); - self.open.len() - 1 - } - }; - if row(&mut self.open[i].enc)? == Append::Full { - let chunk = self.take_chunk(i)?; - self.sealed.push(chunk); - row(&mut self.open[i].enc)?; + /// Count `rows` just encoded, none for a dropped delete, and seal a due + /// segment + fn pushed(&mut self, rows: usize) -> Result<()> { + if rows == 0 { + self.deletes_discarded += 1; } - self.open_rows += 1; - self.routed_rows += 1; - let open_bytes = self.open.iter().map(|o| o.enc.approx_bytes).sum::() - + self - .sealed - .iter() - .map(|(_, c)| c.allocated_bytes()) - .sum::(); - if self.open_rows >= self.segment_rows || open_bytes >= self.segment_bytes { + self.routed_rows += rows as u64; + if self.chunks.due() { self.seal_segment()?; } Ok(()) } - fn take_chunk(&mut self, i: usize) -> Result<(u32, EncodedChunk)> { - let o = &mut self.open[i]; - let t = &self.tables[o.table as usize]; - let (buffers, rows) = o.enc.take_block()?; - let chunk = EncodedChunk { - rel: t.descriptor.clone(), - route: t.route.clone().expect("routed table"), - buffers, - rows, - }; - Ok((o.table, chunk)) - } - /// Close the open segment into a step, spilling once memory runs out fn seal_segment(&mut self) -> Result<()> { - let mut chunks = std::mem::take(&mut self.sealed); - for i in 0..self.open.len() { - if self.open[i].enc.rows > 0 { - chunks.push(self.take_chunk(i)?); - } - } - self.open_rows = 0; + let chunks = self.chunks.take()?; if chunks.is_empty() { return Ok(()); } @@ -570,7 +676,7 @@ impl PlanWriter { /// mirror-row position it fences pub fn push_control(&mut self, event: DrainEntry, row_idx: usize) -> Result<()> { self.seal_segment()?; - self.open.clear(); + self.chunks.clear(); self.live_from = self.tables.len(); self.table_ids = None; self.steps.push(Step::Control { event, row_idx }); @@ -599,12 +705,7 @@ impl PlanWriter { commit_ts: i64, ) -> Result<(SealedPlan, EncoderCache)> { self.seal_segment()?; - let mut encoders = std::mem::take(&mut self.encoders); - for o in self.open.drain(..) { - let t = &self.tables[o.table as usize]; - let route = t.route.clone().expect("routed table"); - encoders.put(t.descriptor.clone(), route, o.enc); - } + let encoders = self.chunks.into_encoders(); let file = match self.file.take() { None => None, Some((mut out, file)) => { @@ -627,7 +728,9 @@ impl PlanWriter { } } -/// One row change read from tuple images in place +/// One row change read from tuple images in place. Patches replace image +/// attributes by index: resolved TOAST values, and values past the +/// descriptor's attributes #[derive(Clone, Copy)] pub struct ViewChange<'v, 'a> { pub op: HeapOp, @@ -635,18 +738,41 @@ pub struct ViewChange<'v, 'a> { pub source_lsn: u64, pub new: Option<&'v TupleView<'a>>, pub old: Option<&'v TupleView<'a>>, + pub new_patch: &'v [(usize, ColumnValue)], + pub old_patch: &'v [(usize, ColumnValue)], } /// [`take_key_change_tombstone`]'s test over tuple images in place -fn key_moved(rel: &RelDescriptor, op: HeapOp, old: &TupleView<'_>, new: &TupleView<'_>) -> bool { - let hot = match op { +fn key_moved( + rel: &RelDescriptor, + change: &ViewChange<'_, '_>, + old: &TupleView<'_>, + new: &TupleView<'_>, +) -> bool { + let hot = match change.op { HeapOp::Update => false, HeapOp::HotUpdate => true, _ => return false, }; - let differs = |a: usize| match (old.datums.get(a), new.datums.get(a), rel.attributes.get(a)) { - (Some(&o), Some(&n), Some(att)) => datums_differ(att, o, n), - _ => false, + let patched = |patch: &[(usize, ColumnValue)], a: usize| { + patch.iter().find(|(i, _)| *i == a).map(|(_, v)| v.clone()) + }; + let differs = |a: usize| { + let (Some(&o), Some(&n), Some(att)) = + (old.datums.get(a), new.datums.get(a), rel.attributes.get(a)) + else { + return false; + }; + match (patched(change.old_patch, a), patched(change.new_patch, a)) { + (None, None) => datums_differ(att, o, n), + (po, pn) => values_differ( + att, + [ + po.or_else(|| datum_value(att, o)), + pn.or_else(|| datum_value(att, n)), + ], + ), + } }; let keys = crate::schema::replident_key_attnums(rel); if keys.is_empty() { @@ -673,19 +799,24 @@ fn datums_differ(att: &crate::schema::RelAttr, o: DatumRef<'_>, n: DatumRef<'_>) } (DatumRef::Varlena(a), DatumRef::Varlena(b)) if a == b => false, (DatumRef::Null | DatumRef::Dropped, DatumRef::Null | DatumRef::Dropped) => false, - _ => { - let mut values = [o, n].map(|d| datum_value(att, d)); - if !att.dropped && matches!(att.type_name.as_str(), "geography" | "geometry") { - for v in values.iter_mut().flatten() { - crate::ops::oracle::render_ext_value(v); - } - } - values[0] != values[1] + _ => values_differ(att, [o, n].map(|d| datum_value(att, d))), + } +} + +/// Owned cells compared after PostGIS render; absent never differs +fn values_differ(att: &crate::schema::RelAttr, mut values: [Option; 2]) -> bool { + if values.iter().any(Option::is_none) { + return false; + } + if !att.dropped && matches!(att.type_name.as_str(), "geography" | "geometry") { + for v in values.iter_mut().flatten() { + crate::ops::oracle::render_ext_value(v); } } + values[0] != values[1] } -fn op_code(op: HeapOp) -> std::result::Result { +pub(crate) fn op_code(op: HeapOp) -> std::result::Result { Ok(match op { HeapOp::Insert => OP_INSERT, HeapOp::Update | HeapOp::HotUpdate => OP_UPDATE, @@ -1200,7 +1331,7 @@ mod tests { for (i, route) in routes.into_iter().enumerate() { let mut w = PlanWriter::create(tmp.path().join(format!("{i}.plan")), 1 << 20, 1 << 20) .with_encoders(encoders); - let cached = w.encoders.entries.len(); + let cached = w.chunks.encoders.entries.len(); push( &mut w, &d, @@ -1208,7 +1339,7 @@ mod tests { Some(route.clone()), ) .unwrap(); - let reused = cached - w.encoders.entries.len(); + let reused = cached - w.chunks.encoders.entries.len(); assert_eq!((cached, reused), [(0, 0), (1, 1), (1, 0)][i]); let plan; (plan, encoders) = w.seal_keeping(0x2000, 42).unwrap(); diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 01ae8663..75d9cf1b 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -39,8 +39,7 @@ use std::sync::Arc; -use crate::decode::datum::DatumRef; -use crate::decode::heap_decoder::view_heap_record; +use crate::decode::heap_decoder::{ColumnValue, view_heap_record}; use crate::emit::pipeline::plan_spool::{ DEFAULT_PLAN_MEM_MAX, EncoderCache, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, @@ -48,10 +47,10 @@ use crate::emit::pipeline::plan_spool::{ use crate::emit::route::RouteSnapshot; use crate::schema::RelDescriptor; use crate::toast::{ChunkRefMap, ToastResolver}; -use crate::xact::spill::DescribedInput; +use crate::xact::spill::{BodySpoolFile, DescribedInput}; use crate::xact::xact_buffer::{ DrainEntry, DrainHeap, DrainWalk, DrainedBatch, FailClosedReason, WalkStep, XactBufferError, - detoast_heap, heap_reads_toast, + detoast_heap, detoast_sites, external_sites, heap_reads_toast, }; /// Planner's window onto route state. Implementations resolve from frozen @@ -212,28 +211,24 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { self.writer.discard(table, drained.rows() as u64); continue; } - let drained = match drained { + let mut heap = match drained { DrainHeap::Input(input) => { - if self.plan_views(&input, &rel, table)? { - continue; - } - DrainHeap::Input(input) - } - cells => cells, - }; - for mut heap in drained.into_cells()? { - if heap.decoded.new.as_ref().is_some_and(|t| t.partial) { - return Err(PlanError::PartialUpdate { - lsn: heap.decoded.source_lsn, - }); - } - if heap_reads_toast(&heap) { - // Value permit drops once bytes land in the plan - Box::pin(detoast_heap(&mut heap, spool, &ref_maps, self.resolver)) + Box::pin(self.plan_views(&input, &rel, table, spool, &ref_maps)) .await?; + continue; } - self.writer.push_cells(heap.decoded, table)?; + DrainHeap::Cells(heap) => heap, + }; + if heap.decoded.new.as_ref().is_some_and(|t| t.partial) { + return Err(PlanError::PartialUpdate { + lsn: heap.decoded.source_lsn, + }); + } + if heap_reads_toast(&heap) { + // Value permit drops once bytes land in the plan + Box::pin(detoast_heap(&mut heap, spool, &ref_maps, self.resolver)).await?; } + self.writer.push_cells(heap.decoded, table)?; } } } @@ -242,14 +237,16 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Ok(()) } - /// Encode an input from tuple images in place; `false` when an image - /// holds external values, which only owned cells detoast - fn plan_views( + /// Encode an input from tuple images in place, external values resolved + /// into per-image patches + async fn plan_views( &mut self, input: &DescribedInput, rel: &Arc, table: u32, - ) -> Result { + spool: Option<&BodySpoolFile>, + ref_maps: &[&ChunkRefMap], + ) -> Result<(), PlanError> { let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { XactBufferError::OrdinaryFailClosed { relid: rel.oid, @@ -259,31 +256,36 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { reason: FailClosedReason::Malformed(e.to_string()), } })?; - let external = images.iter().any(|h| { - [&h.new, &h.old] - .into_iter() - .flatten() - .any(|t| t.datums.iter().any(|d| matches!(d, DatumRef::External(_)))) - }); - if external { - return Ok(false); - } for h in &images { if h.new.as_ref().is_some_and(|t| t.partial) { return Err(PlanError::PartialUpdate { lsn: input.input.source_lsn, }); } + let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); + let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); + if !sites.is_empty() { + let db = rel.rfn.db_node; + let lsn = input.input.source_lsn; + // Value permit drops once bytes land in the plan + let (values, _permit) = + detoast_sites(&sites, rel, db, lsn, spool, ref_maps, self.resolver).await?; + for (site, v) in sites.iter().zip(values) { + patches[site.image].push((site.idx, v)); + } + } let change = ViewChange { op: h.op, xid: input.input.xid, source_lsn: input.input.source_lsn, new: h.new.as_ref(), old: h.old.as_ref(), + new_patch: &patches[0], + old_patch: &patches[1], }; self.writer.push_views(change, table)?; } - Ok(true) + Ok(()) } /// Freeze the validated plan for the executor diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index 6bf00b26..26a34f26 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -318,9 +318,7 @@ pub(crate) mod tests { ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp, }; use crate::emit::ch_emitter::Wire; - use crate::emit::pipeline::batcher::{ - BatcherConfig, BatcherMsg, RoutedRow, RowChunk, RowTable, - }; + use crate::emit::pipeline::batcher::{BatcherConfig, BatcherMsg, PlacedSegment, RowTable}; use crate::emit::route::RouteSnapshot; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; @@ -379,11 +377,10 @@ pub(crate) mod tests { RowTable { rel, route } } - pub(crate) fn wkt_row(seq: u64, text: Option<&str>) -> RoutedRow { - RoutedRow { + pub(crate) fn wkt_row(seq: u64, text: Option<&str>) -> (u64, CommittedTuple) { + ( seq, - table: 0, - committed: CommittedTuple { + CommittedTuple { decoded: DecodedHeap { rfn: RelFileNode { spc_node: 1663, @@ -402,14 +399,13 @@ pub(crate) mod tests { commit_ts: 0, commit_lsn: (seq + 1) * 100, }, - value_permit: None, - } + ) } /// One batch of the rows, sealed through the real batcher: `InsertBatch` /// is only reachable that way, and its cell tags are what the encoder /// chose rather than what a test asserted - pub(crate) async fn seal(rows: Vec) -> InsertBatch { + pub(crate) async fn seal(rows: Vec<(u64, CommittedTuple)>) -> InsertBatch { let (msg_tx, msg_rx) = mpsc::channel(64); let (batches_tx, batches_rx) = async_channel::bounded(64); let fatal = Fatal::new(); @@ -427,13 +423,13 @@ pub(crate) mod tests { Arc::new(EmitterStats::default()), None, ); - let chunk = RowChunk { - tables: vec![wkt_table()], - rows, - permit: None, - }; + let RowTable { rel, route } = wkt_table(); + let segments = rows + .into_iter() + .map(|(seq, row)| PlacedSegment::of_rows(seq, &rel, &route, &[row]).unwrap()) + .collect(); msg_tx - .send(BatcherMsg::Rows(chunk)) + .send(BatcherMsg::Segments(segments)) .await .expect("send rows"); let (reply_tx, reply_rx) = oneshot::channel(); @@ -500,7 +496,7 @@ pub(crate) mod tests { async fn one_unrendered_cell_keeps_the_column_remote() { let mut rows = vec![wkt_row(0, Some("POINT(1 2)"))]; let mut raw = wkt_row(0, None); - raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { + raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { type_oid: GEOM_OID, raw: vec![1, 2, 3], }); @@ -547,7 +543,7 @@ pub(crate) mod tests { #[tokio::test] async fn resolver_exits_on_oracle_failure_or_closed_output() { let mut raw = wkt_row(0, None); - raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { + raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { type_oid: GEOM_OID, raw: vec![1], }); @@ -563,7 +559,7 @@ pub(crate) mod tests { async fn unanswered_oracle_column_and_bad_type_fail_seal() { let mut sealer = Sealer::new(CompressionChoice::None).unwrap(); let mut raw = wkt_row(0, None); - raw.committed.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { + raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { type_oid: GEOM_OID, raw: vec![1], }); diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 85f8f003..1989521b 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -206,18 +206,7 @@ pub async fn spawn( .await } -/// Ack every row of `chunks` as done, one count per seq -fn swallow(ack: &AckHandle, chunks: impl IntoIterator) { - let mut counts: HashMap = HashMap::new(); - for chunk in chunks { - for r in &chunk.rows { - *counts.entry(r.seq).or_insert(0) += 1; - } - } - ack.swallowed(counts.into_iter().collect()); -} - -/// [`swallow`] for planned segments +/// Ack every row of `segments` as done, one count per seq fn swallow_segments(ack: &AckHandle, segments: Vec) { let mut counts: HashMap = HashMap::new(); for s in segments { @@ -245,7 +234,6 @@ pub fn spawn_null( let batcher = tokio::spawn(async move { while let Some(msg) = msg_rx.recv().await { match msg { - BatcherMsg::Rows(chunk) => swallow(&swallow_ack, [chunk]), BatcherMsg::Segments(segments) => swallow_segments(&swallow_ack, segments), BatcherMsg::FlushAll(reply) => { let _ = reply.send(()); @@ -265,6 +253,44 @@ pub fn spawn_null( ) } +/// Batcher whose sealed batches drop on arrival, acked as inserted. +/// Measures producers and batching without ClickHouse +#[cfg(any(test, feature = "test-support"))] +pub fn spawn_discard( + cfg: BatcherConfig, + stats: Arc, + fatal: Fatal, +) -> (mpsc::Sender, AckHandle, TailParts) { + let (ack, collector) = ack::spawn(Arc::new(Monotone::default()), fatal.clone()); + let (msg_tx, msg_rx) = mpsc::channel::(256); + let (batches_tx, batches_rx) = async_channel::bounded::(4); + let batcher = batcher::spawn( + msg_rx, + batches_tx, + cfg, + Allocator::global(&mimalloc::MiMalloc), + fatal, + stats, + None, + ); + let discard_ack = ack.clone(); + let discard = tokio::spawn(async move { + while let Ok(batch) = batches_rx.recv().await { + discard_ack.swallowed(batch.per_seq.clone()); + } + }); + ( + msg_tx, + ack, + TailParts { + collector: AbortOnDropHandle::new(collector), + batcher: AbortOnDropHandle::new(batcher), + resolvers: Vec::new(), + inserters: vec![AbortOnDropHandle::new(discard)], + }, + ) +} + /// [`spawn`] plus a live config receiver: the batcher re-reads /// budgets/flush and the inserter pool re-reads compression/retry from it on /// each republish. `None` == boot values only (bootstrap + tests use [`spawn`]). diff --git a/src/toast/resolver.rs b/src/toast/resolver.rs index ec600003..0263a8c9 100644 --- a/src/toast/resolver.rs +++ b/src/toast/resolver.rs @@ -1428,38 +1428,30 @@ impl ToastResolver { finish_value(&p, stored).ok() } - pub async fn fill_ahead( + /// Speculative read of `p` started ahead of planning, its permit + /// absorbed into `permit` + pub async fn take_ahead( &self, db_oid: u32, - t: &mut crate::decode::heap_decoder::DecodedTuple, - rel: &crate::schema::RelDescriptor, + p: &ToastPointer, + type_oid: u32, lsn: u64, permit: &mut Option, - ) { - for (col, attr) in t.columns.iter_mut().zip(&rel.attributes) { - let Some(ColumnValue::ExternalToast(p)) = col else { - continue; - }; - let Some(slot) = self - .pending() - .and_then(|mut c| c.take(db_oid, p, attr.type_oid, lsn)) - else { - continue; - }; - let started = std::time::Instant::now(); - let value = slot.take().await; - self.stats - .toast_value_wait_nanos - .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); - if let Some(ReadValue { - value, - permit: read, - }) = value - { - crate::budget::absorb_into(permit, read); - *col = Some(value); - } - } + ) -> Option { + let slot = self + .pending() + .and_then(|mut c| c.take(db_oid, p, type_oid, lsn))?; + let started = std::time::Instant::now(); + let value = slot.take().await; + self.stats + .toast_value_wait_nanos + .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); + let ReadValue { + value, + permit: read, + } = value?; + crate::budget::absorb_into(permit, read); + Some(value) } pub fn forget_reads<'a>(&self, keys: impl IntoIterator) { @@ -1504,14 +1496,13 @@ impl ToastResolver { pointer_footprint(p) > self.inline_value_max } - /// Replace an oversized value in place, counting the fill. True when the - /// pointer is settled and needs no fetch - pub fn fill_oversize(&self, col: &mut Option, p: &ToastPointer) -> bool { + /// Whether an oversized value fills NULL, counting the fill. True when + /// the pointer is settled and needs no fetch + pub fn fills_oversize(&self, p: &ToastPointer) -> bool { if !self.value_oversize(p) { return false; } self.note_filled_oversize(); - *col = Some(ColumnValue::Null); true } diff --git a/src/xact/spill.rs b/src/xact/spill.rs index b67b12fe..8c187e69 100644 --- a/src/xact/spill.rs +++ b/src/xact/spill.rs @@ -185,6 +185,7 @@ pub struct DescribedInput { } impl DescribedInput { + #[cfg(any(test, feature = "test-support"))] /// Owned cells per tuple, through the cell adapter pub fn decode( &self, diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index ad9e1f27..9a3ffd72 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -77,6 +77,7 @@ use crate::xact::spill::{ SpillReader, SpillStore, SpillWriter, ToastChunk, ToastDelete, }; use foldhash::{HashMap, HashMapExt}; +#[cfg(any(test, feature = "test-support"))] use smallvec::SmallVec; use std::pin::Pin; @@ -208,6 +209,7 @@ impl DrainHeap { } } + #[cfg(any(test, feature = "test-support"))] /// Owned cells per tuple, through the cell adapter pub fn into_cells(self) -> std::result::Result, XactBufferError> { match self { @@ -225,6 +227,7 @@ fn entry_rows(e: &SpillEntry) -> usize { } } +#[cfg(any(test, feature = "test-support"))] /// Cells for every tuple of a buffered input, under its pinned descriptor. /// Record-time scan validated the bytes, so failure means corrupt storage fn input_heaps( @@ -2334,6 +2337,7 @@ pub struct DrainedBatch { } impl DrainedBatch { + #[cfg(any(test, feature = "test-support"))] /// Owned cells of every row change, inputs decoded through the cell /// adapter pub fn cells(&self) -> std::result::Result, XactBufferError> { @@ -2553,13 +2557,74 @@ impl CommittedDrain { } } -/// Returns the leaf permit shrunk to the decoded bytes retained in the -/// heap's tuples; the caller rides it with the routed row to insert ack -/// so decoded values (and their encoder slab copy) stay covered past -/// this call. `None` without budget or external values. -/// Pub for the planner, gap replay, and tests. +/// External value of one tuple image attribute +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ToastSite { + /// Caller's image index + pub image: usize, + /// Attribute index, attnum - 1 + pub idx: usize, + pub p: ToastPointer, +} + +/// Resolve owned cells' external values in place, see [`detoast_sites`]. +/// Pub for the planner, gap replay, and tests pub async fn detoast_heap( heap: &mut DescribedHeap, + spool: Option<&BodySpoolFile>, + chunk_maps: &[&ChunkRefMap], + resolver: &ToastResolver, +) -> std::result::Result, XactBufferError> { + let mut images = [heap.decoded.new.as_mut(), heap.decoded.old.as_mut()]; + let sites: Vec = images + .iter() + .enumerate() + .flat_map(|(image, t)| { + t.iter().flat_map(move |t| { + t.columns + .iter() + .enumerate() + .filter_map(move |(idx, c)| match c { + Some(ColumnValue::ExternalToast(p)) => { + Some(ToastSite { image, idx, p: *p }) + } + _ => None, + }) + }) + }) + .collect(); + let lsn = heap.decoded.source_lsn; + let db = heap.decoded.rfn.db_node; + let (values, permit) = detoast_sites( + &sites, + &heap.descriptor, + db, + lsn, + spool, + chunk_maps, + resolver, + ) + .await?; + for (site, value) in sites.iter().zip(values) { + if let Some(t) = images[site.image].as_mut() { + t.columns[site.idx] = Some(value); + } + } + Ok(permit) +} + +/// Resolve every site of one heap change's images, values aligned with +/// `sites`. Returns the leaf permit shrunk to the decoded bytes retained; +/// the caller rides it with the routed row to insert ack so decoded values +/// (and their encoder slab copy) stay covered past this call. `None` +/// without budget or external values +pub async fn detoast_sites( + sites: &[ToastSite], + // Attached at decode: same descriptor interpretation from decode to + // detoast regardless of captures landing in between + rel: &RelDescriptor, + db: u32, + lsn: u64, // Xact body spool backing `File` refs; None while memory-resident spool: Option<&BodySpoolFile>, // Ref-map generations, oldest first; a value lives in exactly one @@ -2567,25 +2632,26 @@ pub async fn detoast_heap( // parallel path) chunk_maps: &[&ChunkRefMap], resolver: &ToastResolver, -) -> std::result::Result, XactBufferError> { - // Attached at decode: same descriptor interpretation from decode to - // detoast regardless of captures landing in between - let rel = heap.descriptor.clone(); - let db = heap.decoded.rfn.db_node; +) -> std::result::Result<(Vec, Option), XactBufferError> { + let type_oid = |site: &ToastSite| rel.attributes.get(site.idx).map_or(0, |a| a.type_oid); // Transfer speculative values before admitting foreground resolution - let lsn = heap.decoded.source_lsn; let mut ahead = None; - for t in [heap.decoded.new.as_mut(), heap.decoded.old.as_mut()] - .into_iter() - .flatten() - { - resolver.fill_ahead(db, t, &rel, lsn, &mut ahead).await; + let mut values: Vec> = Vec::with_capacity(sites.len()); + for site in sites { + values.push( + resolver + .take_ahead(db, &site.p, type_oid(site), lsn, &mut ahead) + .await, + ); } - let mut pointers: Vec = Vec::new(); - collect_toast_pointers(heap.decoded.new.as_ref(), &mut pointers); - collect_toast_pointers(heap.decoded.old.as_ref(), &mut pointers); + let mut pointers: Vec = sites + .iter() + .zip(&values) + .filter(|(_, v)| v.is_none()) + .map(|(site, _)| site.p) + .collect(); if pointers.is_empty() { - return Ok(ahead); + return Ok((values.into_iter().flatten().collect(), ahead)); } let leaf_need = check_value_caps( pointers.iter().copied(), @@ -2611,19 +2677,36 @@ pub async fn detoast_heap( cache, retained: 0, }; - if let Some(t) = heap.decoded.new.as_mut() { - res.resolve_tuple(t, &rel)?; - } - if let Some(t) = heap.decoded.old.as_mut() { - res.resolve_tuple(t, &rel)?; - } + let values = sites + .iter() + .zip(values) + .map(|(site, v)| match v { + Some(v) => Ok(v), + None => res.resolve(&site.p, type_oid(site)), + }) + .collect::, _>>()?; if let Some(p) = leaf.as_mut() { p.shrink(res.retained as u64); } if let Some(ahead) = ahead { crate::budget::absorb_into(&mut leaf, ahead); } - Ok(leaf) + Ok((values, leaf)) +} + +/// External datums of tuple images, image index by position in `images` +pub fn external_sites<'v, 'a: 'v>( + images: impl IntoIterator>>, +) -> Vec { + let mut sites = Vec::new(); + for (image, t) in images.into_iter().enumerate() { + for (idx, d) in t.iter().flat_map(|t| t.datums.iter().enumerate()) { + if let DatumRef::External(p) = *d { + sites.push(ToastSite { image, idx, p }); + } + } + } + sites } /// Whether [`detoast_heap`] resolves anything, ie whether it reads the @@ -2637,22 +2720,6 @@ pub fn heap_reads_toast(heap: &DescribedHeap) -> bool { .any(|c| matches!(c, Some(ColumnValue::ExternalToast(_)))) } -/// Append every on-disk toast pointer; carries `va_extinfo`/`va_rawsize` -/// so store fetch can cap allocation -fn collect_toast_pointers( - t: Option<&crate::decode::heap_decoder::DecodedTuple>, - out: &mut Vec, -) { - let Some(t) = t else { - return; - }; - for c in &t.columns { - if let Some(ColumnValue::ExternalToast(p)) = c { - out.push(*p); - } - } -} - /// Fetch every pointer the xact's chunk maps can't cover in one round /// trip per toast rel, so resolution never waits a value at a time. Ids /// dedup: a key used by both tuples fetches once. @@ -2812,61 +2879,52 @@ struct ValueResolution<'a> { } impl ValueResolution<'_> { - fn resolve_tuple( + fn resolve( &mut self, - t: &mut crate::decode::heap_decoder::DecodedTuple, - rel: &RelDescriptor, - ) -> std::result::Result<(), XactBufferError> { - for (idx, col) in t.columns.iter_mut().enumerate() { - let Some(ColumnValue::ExternalToast(p)) = col else { - continue; - }; - // `ToastPointer: Copy` frees the borrow on `col` before reassign - let p: ToastPointer = *p; - if self.resolver.fill_oversize(col, &p) { - continue; + p: &ToastPointer, + type_oid: u32, + ) -> std::result::Result { + if self.resolver.fills_oversize(p) { + return Ok(ColumnValue::Null); + } + let key = (p.va_toastrelid, p.va_valueid); + let Some(v) = self + .xact_maps + .iter() + .find_map(|m| m.get(&key)) + .filter(|v| !observed_mid_value(v)) + else { + return self.resolve_store(p, type_oid); + }; + match reassemble_value_ref(p, self.spool, v)? { + Reassembled::Bytes(raw) => { + let (value, held) = detoasted_sized(raw, type_oid); + self.retained += held; + Ok(value) } - let type_oid = rel.attributes.get(idx).map(|a| a.type_oid).unwrap_or(0); - let key = (p.va_toastrelid, p.va_valueid); - if let Some(v) = self.xact_maps.iter().find_map(|m| m.get(&key)) - && !observed_mid_value(v) - { - match reassemble_value_ref(&p, self.spool, v)? { - Reassembled::Bytes(raw) => { - let (value, held) = detoasted_sized(raw, type_oid); - self.retained += held; - *col = Some(value); - } - // Disabled mode: surface the unresolvable value as - // NULL/default downstream (`append_default`), counted, - // never an error. - Reassembled::Missing if self.resolver.fill_on_miss() => { - self.resolver.note_filled_default(); - *col = Some(ColumnValue::Null); - } - // In-xact chunks gapped or short: decode bug, surfaced loud - outcome => { - self.resolver.note_fetch_miss(); - return Err(match outcome { - Reassembled::SizeMismatch { got, want } => { - XactBufferError::Detoast(format!( - "toast value {}/{}: chunks sum to {got} bytes, pointer says {want}", - p.va_toastrelid, p.va_valueid - )) - } - _ => XactBufferError::MissingToastChunk { - toast_relid: p.va_toastrelid, - value_id: p.va_valueid, - missing: first_missing_seq_ref(v), - }, - }); - } - } - continue; + // Disabled mode: surface the unresolvable value as + // NULL/default downstream (`append_default`), counted, + // never an error. + Reassembled::Missing if self.resolver.fill_on_miss() => { + self.resolver.note_filled_default(); + Ok(ColumnValue::Null) + } + // In-xact chunks gapped or short: decode bug, surfaced loud + outcome => { + self.resolver.note_fetch_miss(); + Err(match outcome { + Reassembled::SizeMismatch { got, want } => XactBufferError::Detoast(format!( + "toast value {}/{}: chunks sum to {got} bytes, pointer says {want}", + p.va_toastrelid, p.va_valueid + )), + _ => XactBufferError::MissingToastChunk { + toast_relid: p.va_toastrelid, + value_id: p.va_valueid, + missing: first_missing_seq_ref(v), + }, + }) } - *col = Some(self.resolve_store(&p, type_oid)?); } - Ok(()) } /// Pre-window / bootstrap value whose chunks aren't in this xact, @@ -3710,15 +3768,15 @@ fn decode_image_insert( "stashed image insert: no LP_NORMAL tuple at ({blkno},{offnum})" ))); }; - let Some((_, _, _, mut columns)) = - crate::backfill::backup_page_walk::decode_on_page_tuple(tuple, rel) - else { + let Some((_, data)) = crate::backfill::backup_page_walk::on_page_tuple(tuple, rel) else { return Err(XactBufferError::Detoast(format!( "stashed image insert: malformed tuple at ({blkno},{offnum})" ))); }; + let view = crate::decode::heap_decoder::view_block_data(data, rel) + .map_err(|e| XactBufferError::Detoast(format!("stashed image insert: {e}")))?; let Some((value_id, chunk_seq, chunk_data)) = - crate::decode::heap_decoder::take_toast_chunk_columns(&mut columns) + crate::decode::heap_decoder::toast_chunk_datums(&view.datums) else { tracing::warn!( target: "walshadow::xact_buffer", @@ -3737,7 +3795,7 @@ fn decode_image_insert( source_lsn: raw.source_lsn, blkno, offnum, - chunk_data: bytes::Bytes::from(chunk_data), + chunk_data: bytes::Bytes::copy_from_slice(chunk_data), })]) } @@ -4709,7 +4767,6 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn read_ahead_declines_and_miss_leaves_pointer() { use std::sync::atomic::Ordering::Relaxed; - let rel = bytea_rel(); let p = toast_ptr(55); let stats = Arc::new(crate::emit::ch_emitter::EmitterStats::default()); let budget = crate::budget::MemoryBudget::with_leaf_reserve(1 << 20, 64 << 10); @@ -4747,9 +4804,9 @@ mod tests { while stats.toast_values_read_ahead.load(Relaxed) == 0 { tokio::task::yield_now().await; } - let mut t = toast_ptr_tuple(55); - r.fill_ahead(5, &mut t, &rel, 0x100, &mut None).await; - assert_eq!(t.columns[0], Some(ColumnValue::ExternalToast(p))); + let mut permit = None; + let miss = r.take_ahead(5, &p, crate::schema::BYTEAOID, 0x100, &mut permit); + assert_eq!(miss.await, None); } /// Ref map of memory bodies for one value's `(seq, body)` chunks @@ -4893,6 +4950,23 @@ mod tests { } /// One-key resolution scope with cache pre-seeded to a store outcome + impl ValueResolution<'_> { + /// Resolve every external cell of `t` in place + fn resolve_tuple( + &mut self, + t: &mut crate::decode::heap_decoder::DecodedTuple, + rel: &RelDescriptor, + ) -> std::result::Result<(), XactBufferError> { + for (idx, col) in t.columns.iter_mut().enumerate() { + if let Some(ColumnValue::ExternalToast(p)) = *col { + let type_oid = rel.attributes.get(idx).map_or(0, |a| a.type_oid); + *col = Some(self.resolve(&p, type_oid)?); + } + } + Ok(()) + } + } + fn seeded<'a>( resolver: &'a ToastResolver, spool: Option<&'a BodySpoolFile>, diff --git a/tests/bootstrap_direct_e2e.rs b/tests/bootstrap_direct_e2e.rs index 905c76be..48e51814 100644 --- a/tests/bootstrap_direct_e2e.rs +++ b/tests/bootstrap_direct_e2e.rs @@ -153,10 +153,10 @@ async fn direct_source_self_hosted_via_replication_protocol() { let shadow_data = tmp.path().join("shadow-data"); let cfg = BootstrapConfig::new(shadow_data.clone()); - let (rx, pump) = spawn_greenfield_bootstrap(cfg, Box::new(direct), catalog_map, false); + let (rx, pump) = spawn_greenfield_bootstrap(cfg, Box::new(direct), catalog_map.clone(), false); let mut observer = RecordingObserver::default(); - let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &mut observer), pump); + let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &catalog_map, &mut observer), pump); let shipped = drain_res.expect("drain task error"); let outcome = pump_res .expect("pump task panicked") diff --git a/tests/bootstrap_object_store_e2e.rs b/tests/bootstrap_object_store_e2e.rs index ea021949..5e8beea5 100644 --- a/tests/bootstrap_object_store_e2e.rs +++ b/tests/bootstrap_object_store_e2e.rs @@ -242,7 +242,8 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { tmp.path().join("part-spool"), ) .with_parallelism(2); - let (rx, pump) = spawn_greenfield_bootstrap(cfg, Box::new(object_source), catalog_map, false); + let (rx, pump) = + spawn_greenfield_bootstrap(cfg, Box::new(object_source), catalog_map.clone(), false); // RecordingObserver wraps a CollectingTupleObserver + counts // on_xact_end so the test asserts drain_backfill closes the @@ -251,7 +252,7 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { // transitional emitter's INSERT block before swapping to the // ShadowCatalog-backed emitter. let mut observer = RecordingObserver::default(); - let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &mut observer), pump); + let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &catalog_map, &mut observer), pump); let shipped = drain_res.expect("drain task error"); let outcome = pump_res .expect("pump task panicked") diff --git a/tests/bootstrap_pipeline_ch.rs b/tests/bootstrap_pipeline_ch.rs index 56a970cf..07acd619 100644 --- a/tests/bootstrap_pipeline_ch.rs +++ b/tests/bootstrap_pipeline_ch.rs @@ -24,7 +24,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; use walrus::pg::walparser::RelFileNode; -use walshadow::backup_page_walk::{BackfillTuple, CatalogMap}; +use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, TupleData}; use walshadow::ch::CompressionChoice; use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; use walshadow::emit::route::freeze_routes; @@ -100,7 +100,8 @@ fn tuple(rel_node: u32, id: i32) -> BackfillTuple { page_lsn: 0, blkno: 0, offnum: 0, - columns: vec![Some(ColumnValue::Int4(id))], + data: TupleData::from_cells(&[Some(ColumnValue::Int4(id))]), + pending: None, } } diff --git a/tests/copy_chunk_resume_e2e.rs b/tests/copy_chunk_resume_e2e.rs index b4869292..c794b6f1 100644 --- a/tests/copy_chunk_resume_e2e.rs +++ b/tests/copy_chunk_resume_e2e.rs @@ -54,12 +54,13 @@ async fn ids( ) -> Vec { let (tx, mut rx) = mpsc::channel::>(4); let stats = Arc::new(EmitterStats::default()); + let rel = desc.clone(); let collect = tokio::spawn(async move { let mut out = Vec::new(); while let Some(slab) = rx.recv().await { for t in slab { - match t.columns.first().and_then(|c| c.as_ref()) { - Some(ColumnValue::Int8(v)) => out.push(*v), + match t.cells(&rel).unwrap().first() { + Some(Some(ColumnValue::Int8(v))) => out.push(*v), other => panic!("unexpected id column {other:?}"), } } diff --git a/tests/emitter_budget_flush.rs b/tests/emitter_budget_flush.rs index 57d090f8..2a966091 100644 --- a/tests/emitter_budget_flush.rs +++ b/tests/emitter_budget_flush.rs @@ -27,7 +27,7 @@ use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; use walshadow::heap_decoder::{ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::ack::Publish; -use walshadow::pipeline::batcher::{BatcherMsg, RoutedRow, RowChunk, RowTable}; +use walshadow::pipeline::batcher::{BatcherMsg, PlacedSegment}; use walshadow::pipeline::{Fatal, tail}; use walshadow::pos::Pos; use walshadow::pos::{EmitterAck, Monotone}; @@ -183,23 +183,15 @@ async fn budget_trips_seal_complete_inserts() { let commit_lsn = 0xC0FFEE; let mut seqs = ack.seqs(0); let seq = seqs.open(commit_lsn, Publish::Commit); - let rows = (0..N) - .map(|i| RoutedRow { - seq: seq.seq(), - table: 0, - committed: tuple(i, 0x1000 + i as u64, commit_lsn), - value_permit: None, + let rel = rel_descriptor(); + let segments = (0..N) + .map(|i| { + let row = tuple(i, 0x1000 + i as u64, commit_lsn); + PlacedSegment::of_rows(seq.seq(), &rel, &route, &[row]).expect("encode row") }) .collect(); msg_tx - .send(BatcherMsg::Rows(RowChunk { - tables: vec![RowTable { - rel: rel_descriptor(), - route, - }], - rows, - permit: None, - })) + .send(BatcherMsg::Segments(segments)) .await .expect("route rows"); seqs.place(seq, N as u64); diff --git a/tests/emitter_native_types.rs b/tests/emitter_native_types.rs index 1d4b0a8f..13ccba98 100644 --- a/tests/emitter_native_types.rs +++ b/tests/emitter_native_types.rs @@ -24,7 +24,7 @@ use walshadow::codecs::NumericKind; use walshadow::heap_decoder::{ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::ack::Publish; -use walshadow::pipeline::batcher::{BatcherMsg, RoutedRow, RowChunk, RowTable}; +use walshadow::pipeline::batcher::{BatcherMsg, PlacedSegment}; use walshadow::pipeline::{Fatal, tail}; use walshadow::pos::{EmitterAck, Monotone}; use walshadow::schema::{RelDescriptor, RelName, ReplIdent}; @@ -176,23 +176,19 @@ async fn native_numeric_time_timetz_round_trip() { let mut seqs = ack.seqs(0); let seq = seqs.open(tuple.commit_lsn, Publish::Commit); msg_tx - .send(BatcherMsg::Rows(RowChunk { - tables: vec![RowTable { - rel, - route: walshadow::emit::route::RouteSnapshot::freeze( + .send(BatcherMsg::Segments(vec![ + PlacedSegment::of_rows( + seq.seq(), + &rel, + &walshadow::emit::route::RouteSnapshot::freeze( mapping, Arc::default(), Default::default(), ), - }], - rows: vec![RoutedRow { - seq: seq.seq(), - table: 0, - committed: tuple, - value_permit: None, - }], - permit: None, - })) + &[tuple], + ) + .expect("encode row"), + ])) .await .expect("route row"); seqs.place(seq, 1); diff --git a/tests/emitter_tls.rs b/tests/emitter_tls.rs index aecdb41c..2a6066f6 100644 --- a/tests/emitter_tls.rs +++ b/tests/emitter_tls.rs @@ -40,7 +40,7 @@ use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; use walshadow::heap_decoder::{ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::ack::Publish; -use walshadow::pipeline::batcher::{BatcherMsg, RoutedRow, RowChunk, RowTable}; +use walshadow::pipeline::batcher::{BatcherMsg, PlacedSegment}; use walshadow::pipeline::{Fatal, tail}; use walshadow::pos::{EmitterAck, Monotone}; use walshadow::schema::{RelDescriptor, RelName, ReplIdent}; @@ -377,23 +377,19 @@ async fn emitter_tls_round_trip() { let mut seqs = ack.seqs(0); let seq = seqs.open(tuple.commit_lsn, Publish::Commit); msg_tx - .send(BatcherMsg::Rows(RowChunk { - tables: vec![RowTable { - rel, - route: walshadow::emit::route::RouteSnapshot::freeze( + .send(BatcherMsg::Segments(vec![ + PlacedSegment::of_rows( + seq.seq(), + &rel, + &walshadow::emit::route::RouteSnapshot::freeze( mapping, Arc::default(), Default::default(), ), - }], - rows: vec![RoutedRow { - seq: seq.seq(), - table: 0, - committed: tuple, - value_permit: None, - }], - permit: None, - })) + &[tuple], + ) + .expect("encode row"), + ])) .await .expect("route row"); seqs.place(seq, 1); From 1d9cf3d3dd4520f720dd0b4b5179637598ad1d70 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:35:58 +0000 Subject: [PATCH 12/20] Plan stash-resolved records from WAL bytes, drop owned-cell path Records resolved from the stash at commit now drain as raw record bytes plus their resolved descriptor, the same shape live records already use Planner and WAL replay encode both straight from tuple views, so the owned-cell drain, spill, plan and replay branches go away --- architecture/README.md | 5 +- benches/spool_read.rs | 12 +- benches/xact_spill.rs | 93 +++- plans/INDEX.md | 2 +- plans/native_encoding.md | 12 +- src/backfill/backfill_bootstrap.rs | 250 +-------- src/backfill/backup_page_walk.rs | 31 +- src/backfill/copy_backfill.rs | 2 + src/backfill/wal_replay.rs | 59 +- src/bin/stream/bootstrap.rs | 21 +- src/bin/stream/metrics_publish.rs | 9 - src/decode/decoder_sink.rs | 141 +---- src/decode/heap_decoder.rs | 197 +------ src/emit/ch_emitter.rs | 12 +- src/emit/pipeline/bootstrap.rs | 2 + src/emit/pipeline/plan_spool.rs | 196 ++++--- src/emit/pipeline/planner.rs | 145 ++--- src/emit/pipeline/reorder.rs | 21 +- src/ops/metrics.rs | 6 - src/ops/oracle.rs | 7 +- src/xact/spill.rs | 819 ++++------------------------ src/xact/xact_buffer.rs | 610 ++++++++------------- tests/bootstrap_direct_e2e.rs | 74 +-- tests/bootstrap_object_store_e2e.rs | 103 +--- tests/xact_buffer.rs | 4 +- 25 files changed, 743 insertions(+), 2090 deletions(-) diff --git a/architecture/README.md b/architecture/README.md index 5edf6a4d..06189567 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -59,8 +59,9 @@ Queue worker runs on a dedicated thread. A dispatcher scans each batch ahead of it to start TOAST store reads for values updated rows keep. Ordinary heap records stay WAL bytes under their record-time descriptor until commit; planning reads tuples in place, resolved external values patching them. -Stash-resolved records use cell adapters; compressed inline values, numeric, -JSONB, and fast defaults can also need row scratch. Commits drain only their own +Records resolved from stash at commit carry the same bytes under their resolved +descriptor. Compressed inline values, numeric, JSONB, and fast defaults can +need row scratch. Commits drain only their own transaction trees. Consecutive small commits can share column blocks while retaining separate acknowledgements. COPY and backup loads carry tuple bytes or COPY fields and encode their own column chunks diff --git a/benches/spool_read.rs b/benches/spool_read.rs index e7fdd411..9b1af82a 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -2,7 +2,6 @@ //! `cargo bench --bench spool_read -- --rows 200000` //! Add `--plan-only --planner --mapped --relations 8 --control-every 10000` //! to include planning, route caching within benchmark, and column encoding. -//! `--cells` plans decoded rows instead of WAL record bytes. //! Excludes WAL drain, TOAST I/O, control execution, and destination insertion. use std::path::PathBuf; @@ -52,9 +51,6 @@ struct Args { /// Run Planner with cached routes. #[arg(long)] planner: bool, - /// Plan decoded rows rather than WAL record bytes. - #[arg(long)] - cells: bool, #[arg(long)] mapped: bool, #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u16).range(1..))] @@ -337,11 +333,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { event, }); } - heaps.push(if args.cells { - DrainHeap::Cells(make_heap(i)) - } else { - DrainHeap::Input(Box::new(make_input(i))) - }); + heaps.push(DrainHeap::Input(Box::new(make_input(i)))); } DrainedBatch { heaps, @@ -383,8 +375,6 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { let table = writer.table(rel, 1, || view.route_for(rel)); if !writer.routed(table) { writer.discard(table, 1); - } else if args.cells { - writer.push_cells(make_heap(i).decoded, table)?; } else { let input = make_input(i); let record = input.input.to_xlog_record(rel.rfn); diff --git a/benches/xact_spill.rs b/benches/xact_spill.rs index d29b6c68..32a0856c 100644 --- a/benches/xact_spill.rs +++ b/benches/xact_spill.rs @@ -1,5 +1,7 @@ //! Transaction buffering and verified commit drain, without PostgreSQL or ClickHouse. //! `cargo bench --bench xact_spill -- --buffer-bytes 1048576` +//! `--stashed` buffers records as stashed for commit resolution, `--plan` +//! drains into the planner with a mapped route instead of checking rows use std::path::PathBuf; use std::sync::Arc; @@ -11,10 +13,14 @@ use walrus::pg::walparser::{ XLogRecordHeader, }; use walshadow::decode::datum::DatumRef; +use walshadow::emit::route::RouteSnapshot; use walshadow::heap_decoder::view_heap_record; +use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; +use walshadow::pipeline::planner::{PlanRouteView, Planner}; use walshadow::schema::{RelAttr, RelDescriptor, RelName, ReplIdent, TEXTOID}; -use walshadow::spill::{DescribedInput, HeapInput}; -use walshadow::xact_buffer::{DrainHeap, StashResolved, XactBuffer, XactBufferConfig}; +use walshadow::spill::{DescribedInput, HeapInput, RawRecord}; +use walshadow::toast::ToastResolver; +use walshadow::xact_buffer::{DrainEntry, DrainHeap, StashResolved, XactBuffer, XactBufferConfig}; #[path = "common/alloc_meter.rs"] mod alloc_meter; @@ -37,6 +43,24 @@ struct Args { repeats: usize, #[arg(long, default_value = "target")] dir: PathBuf, + #[arg(long)] + stashed: bool, + #[arg(long)] + plan: bool, +} + +struct Mapped(Arc); + +impl PlanRouteView for Mapped { + fn route_for(&mut self, _: &Arc) -> Option> { + Some(self.0.clone()) + } + + fn unmapped(&mut self, _: &RelDescriptor, _: u64) {} + + async fn apply(&mut self, _: &DrainEntry) -> Result<(), String> { + Ok(()) + } } fn main() -> anyhow::Result<()> { @@ -81,6 +105,21 @@ async fn run(args: Args) -> anyhow::Result<()> { missing_default: None, }], }); + let mut routes = Mapped(RouteSnapshot::freeze( + Arc::new(TableMapping { + target: TableTarget::new("db", "spill_bench"), + columns: vec![ColumnMapping { + src_attnum: 1, + target_name: "v".into(), + target_type: "String".into(), + type_pinned: false, + }], + tees: Vec::new(), + }), + Arc::default(), + Default::default(), + )); + let resolver = ToastResolver::disabled(); let payload = "x".repeat(args.payload_bytes); // XLOG_HEAP_INSERT block data: xl_heap_header, then one 4-byte-header text let mut tuple = 1u16.to_le_bytes().to_vec(); @@ -126,13 +165,18 @@ async fn run(args: Args) -> anyhow::Result<()> { let record = record(xid); for row in 0..args.rows { let source_lsn = u64::from(xid) * (args.rows as u64 + 1) + row as u64; - buffer - .on_input(DescribedInput { - input: HeapInput::from_parsed(&record, source_lsn, 1), - descriptor: descriptor.clone(), - descriptor_valid_from: 1, - }) - .await?; + if args.stashed { + let raw = RawRecord::from_parsed(&record, source_lsn, 0); + buffer.stash_raw(xid, raw).await?; + } else { + buffer + .on_input(DescribedInput { + input: HeapInput::from_parsed(&record, source_lsn, 1), + descriptor: descriptor.clone(), + descriptor_valid_from: 1, + }) + .await?; + } peak_memory = peak_memory.max(buffer.stats().bytes_in_memory); } write_time += start.elapsed(); @@ -141,17 +185,34 @@ async fn run(args: Args) -> anyhow::Result<()> { let base = GLOBAL.reset_peak(); let start = Instant::now(); let commit_lsn = u64::from(xid + 1) * (args.rows as u64 + 1); + let resolved = if args.stashed { + StashResolved::ordinary(xid, descriptor.clone(), 1) + } else { + StashResolved::nothing_stashed(xid) + }; let mut drain = buffer - .drain_committed( - StashResolved::nothing_stashed(xid), - 0, - commit_lsn, - &[], - false, - ) + .drain_committed(resolved, 0, commit_lsn, &[], false) .await?; let mut rows = 0; + if args.plan { + let mut planner = Planner::create( + dir.path().join(format!("plan-{xid}.spool")), + u64::MAX, + (4096, 1 << 20), + 0, + &mut routes, + &resolver, + ); + while let Some(batch) = drain.next_batch(1024, 1 << 20, None).await? { + planner.plan_batch(batch).await?; + } + rows = planner.seal(commit_lsn, 0)?.routed_rows; + } while let Some(batch) = drain.next_batch(1024, 1 << 20, None).await? { + if args.stashed { + rows += batch.heaps.len() as u64; + continue; + } for heap in batch.heaps { let DrainHeap::Input(input) = heap else { panic!("ordinary rows drain as inputs"); diff --git a/plans/INDEX.md b/plans/INDEX.md index 7f21ea2c..118ec4a5 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -39,7 +39,7 @@ writes. A documented limitation does not imply code rejects it | [Multi-database loads and metrics](multi_database.md) | Extend heap-page bootstrap and backup loads beyond primary database, attribute metrics | | [Fuzzing](fuzzing.md) | Find parser and schema-transition interactions beyond fixed regressions | | [Performance](performance.md) | Establish healthy sustained WAL baseline with exact reconciliation before tuning | -| [Direct Native encoding](native_encoding.md) | Move stash-resolved rows off cells, bound memory across owners, prove resource gains end to end | +| [Direct Native encoding](native_encoding.md) | Bound memory across owners, prove resource gains end to end | | [Failover](failover.md) | Fence unplanned promotion and prove remaining archive/restart fault cases | | [Shadow TOAST reclamation](shadow_toast.md) | Keep historical values readable under lag and restart | | [Replay callback](custom_rmgr.md) | Reduce measured command-boundary capture stalls | diff --git a/plans/native_encoding.md b/plans/native_encoding.md index b23ca990..3be50d02 100644 --- a/plans/native_encoding.md +++ b/plans/native_encoding.md @@ -3,22 +3,12 @@ Commits, loads, and gap replay encode rows from tuple bytes or COPY fields into column chunks, and sealed Native blocks stay compressed until acknowledgement; see [architecture](../architecture/README.md#commit-pipeline). Remaining work -moves stash-resolved records off owned cells, bounds memory across owners, and -proves resource gains end to end. Follow [shared constraints](coordination.md), +bounds memory across owners and proves resource gains end to end. Follow [shared constraints](coordination.md), [value coercion](value_coercion.md), and [performance acceptance](performance.md) Share view encoders and column buffers with [container codecs](tier2.md) Container promotion can retain existing raw pending cells in owned adapters -## Stash-resolved records - -Records resolved at commit decode into owned cells in -[transaction buffer](../src/xact/xact_buffer.rs) `fold_raw_ordinary`, then plan -through the cell adapter. Carry them as raw inputs under their resolved -descriptor instead, keeping operation policy and ambiguity fences at fold time. -Resolved TOAST values stay owned cells patching tuple views; read-ahead hands -them over rendered - ## Bound memory across owners Track allocated capacity, not lengths, for raw inputs, open column builders, diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index b64d9032..c7555dca 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -39,7 +39,6 @@ use crate::backfill::backup_sink::{ CatalogFilenodes, DiskLanderSink, DiskLanderStats, MultiplexSink, }; use crate::backfill::backup_source::{BackupSink, BackupSource, EndInfo, PumpStats, StartInfo}; -use crate::decode::decoder_sink::TupleObserver; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; /// Live counter handles, readable while the pump runs. The pump publishes @@ -154,10 +153,9 @@ pub fn tap_filenode_set( /// `rx.recv() == None` signals channel close. /// /// ```ignore -/// let (rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); -/// let drained = tokio::spawn(drain_backfill(rx, &catalog, observer)); +/// let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog_map, store_toast); +/// while let Some(slab) = rx.recv().await { /* consume */ } /// let outcome = pump.await?.context("bootstrap pump")?; -/// let shipped = drained.await?; /// ``` /// /// Channel is bounded ([`BOOTSTRAP_TUPLE_CHANNEL_CAP`]): PageWalkSink::chunk @@ -223,8 +221,8 @@ pub fn spawn_greenfield_bootstrap( } /// Test/fixture wrapper: run pump to completion, collect every -/// `BackfillTuple` into a `Vec`. Production callers use -/// [`spawn_greenfield_bootstrap`] + [`drain_backfill`] to bound memory by +/// `BackfillTuple` into a `Vec`. Production callers drain +/// [`spawn_greenfield_bootstrap`] concurrently to bound memory by /// the emitter's drain rate, not the source's full tuple count pub async fn run_greenfield_bootstrap( cfg: BootstrapConfig, @@ -245,60 +243,6 @@ pub async fn run_greenfield_bootstrap( Ok((outcome, tuples)) } -/// Drain backfill tuples into a [`TupleObserver`]. Each `BackfillTuple` -/// becomes a synthetic -/// [`CommittedTuple`](crate::decode::heap_decoder::CommittedTuple) with `op = Insert`, -/// `commit_ts = 0`, `commit_lsn = source_lsn` (BASE_BACKUP `start_lsn`). -/// Synthesises committed rows from on-disk pages, bypassing the daemon's -/// xact-buffer + decoder chain. -/// -/// `PageWalkSink` emits all rows for one rfn contiguously, so drive -/// `on_xact_end` on every rfn flip: CH Native protocol forbids a new `Query` -/// while a prior INSERT's data stream is open; without per-table flushes the -/// second `send_query` races the first INSERT's body bytes and CH silently -/// drops everything on the connection. -/// -/// Final `on_xact_end` after channel-close lets the transitional emitter -/// release CH state before the daemon swaps to the shadow-catalog emitter. -pub async fn drain_backfill( - mut rx: mpsc::Receiver>, - catalog: &CatalogMap, - observer: &mut O, -) -> Result { - let mut shipped: u64 = 0; - let mut last_rfn: Option = None; - let mut last_lsn: u64 = 0; - while let Some(slab) = rx.recv().await { - for tuple in slab { - if let Some(prev) = last_rfn - && prev != tuple.rfn - { - observer.on_xact_end(last_lsn).await.map_err(|e| { - anyhow::anyhow!("bootstrap drain: emitter rejected mid-table xact end: {e}") - })?; - } - last_rfn = Some(tuple.rfn); - last_lsn = tuple.source_lsn; - let rel = catalog - .get(tuple.rfn.db_node, tuple.rfn.rel_node) - .with_context(|| format!("bootstrap drain: no descriptor for {:?}", tuple.rfn))?; - let committed = tuple - .into_committed_insert(&rel) - .map_err(|e| anyhow::anyhow!("bootstrap drain: {e}"))?; - observer - .on_tuple(&committed) - .await - .context("bootstrap drain: emitter rejected tuple")?; - shipped += 1; - } - } - observer - .on_xact_end(last_lsn) - .await - .context("bootstrap drain: emitter rejected xact end")?; - Ok(shipped) -} - /// Build a [`CatalogMap`] by enumerating every source PG user relation /// upfront. Mirrors `ShadowCatalog::{fetch_by_rfn,fetch_attributes, /// fetch_replident}` (shadow_catalog.rs) but eager rather than lazy by @@ -471,11 +415,8 @@ fn one_char(s: String, what: &str) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::backfill::backup_page_walk::{ - PAGE_BYTES, TupleData, make_rel, synth_single_tuple_page, - }; + use crate::backfill::backup_page_walk::{PAGE_BYTES, make_rel, synth_single_tuple_page}; use crate::backfill::backup_source::{BackupSink, BackupSource, FileMeta}; - use crate::decode::heap_decoder::HeapOp; use async_trait::async_trait; use walrus::pg::backup::tarball::{Entry, Kind, Reader, Writer}; @@ -649,185 +590,4 @@ mod tests { [Some(crate::decode::heap_decoder::ColumnValue::Int4(42))] ); } - - #[tokio::test] - async fn drain_backfill_synthesises_inserts_into_observer() { - use crate::decode::decoder_sink::CollectingTupleObserver; - - let (tx, rx) = mpsc::channel::>(64); - let rfn = RelFileNode { - spc_node: 1663, - db_node: 5, - rel_node: 16400, - }; - for v in 0..3 { - tx.send(vec![BackfillTuple { - rfn, - xid: 100 + v, - xmax: 0, - infomask: 0, - source_lsn: 0xCAFE, - page_lsn: 0, - blkno: 0, - offnum: 0, - data: TupleData::from_cells(&[Some( - crate::decode::heap_decoder::ColumnValue::Int4(v as i32), - )]), - pending: None, - }]) - .await - .unwrap(); - } - drop(tx); - - let mut observer = CollectingTupleObserver::default(); - let shipped = drain_backfill(rx, &int4_catalog(), &mut observer) - .await - .unwrap(); - assert_eq!(shipped, 3); - assert_eq!(observer.tuples.len(), 3); - for (i, c) in observer.tuples.iter().enumerate() { - assert_eq!(c.commit_ts, 0); - assert_eq!(c.commit_lsn, 0xCAFE); - assert_eq!(c.decoded.op, HeapOp::Insert); - assert!(c.decoded.old.is_none()); - let new = c.decoded.new.as_ref().unwrap(); - assert!(!new.partial); - assert!(matches!( - new.columns[0], - Some(crate::decode::heap_decoder::ColumnValue::Int4(v)) if v == i as i32 - )); - } - } - - fn int4_catalog() -> CatalogMap { - let mut catalog = CatalogMap::new(); - catalog.insert(Arc::new(make_rel())); - catalog - } - - #[derive(Default)] - struct CountingObserver { - on_tuple_calls: u32, - on_xact_end_calls: u32, - reject_xact_end: bool, - } - - impl crate::decode::decoder_sink::TupleObserver for CountingObserver { - fn on_tuple<'a>( - &'a mut self, - _committed: &'a crate::decode::heap_decoder::CommittedTuple, - ) -> std::pin::Pin< - Box< - dyn std::future::Future< - Output = Result<(), crate::decode::decoder_sink::DecoderSinkError>, - > + Send - + 'a, - >, - > { - self.on_tuple_calls += 1; - Box::pin(async { Ok(()) }) - } - fn on_xact_end<'a>( - &'a mut self, - commit_lsn: u64, - ) -> std::pin::Pin< - Box< - dyn std::future::Future< - Output = Result, - > + Send - + 'a, - >, - > { - self.on_xact_end_calls += 1; - let reject = self.reject_xact_end; - Box::pin(async move { - use crate::decode::decoder_sink::DecoderSinkError; - (!reject) - .then_some(commit_lsn) - .ok_or(DecoderSinkError::Observer("full".into())) - }) - } - } - - #[tokio::test] - async fn drain_backfill_calls_on_xact_end_after_channel_closes() { - let rfn = RelFileNode { - spc_node: 1663, - db_node: 5, - rel_node: 16400, - }; - let (tx, rx) = mpsc::channel::>(64); - for v in 0..4u32 { - tx.send(vec![BackfillTuple { - rfn, - xid: v, - xmax: 0, - infomask: 0, - source_lsn: 1, - page_lsn: 0, - blkno: 0, - offnum: 0, - data: TupleData::from_cells(&[Some( - crate::decode::heap_decoder::ColumnValue::Int4(v as i32), - )]), - pending: None, - }]) - .await - .unwrap(); - } - drop(tx); - - let mut obs = CountingObserver::default(); - let shipped = drain_backfill(rx, &int4_catalog(), &mut obs).await.unwrap(); - assert_eq!(shipped, 4); - assert_eq!(obs.on_tuple_calls, 4); - // One synthetic xact closes the backfill - assert_eq!(obs.on_xact_end_calls, 1); - } - - #[tokio::test] - async fn drain_backfill_surfaces_a_rejected_table_flip() { - let tuple = |rel_node| BackfillTuple { - rfn: RelFileNode { - spc_node: 1663, - db_node: 5, - rel_node, - }, - xid: 1, - xmax: 0, - infomask: 0, - source_lsn: 1, - page_lsn: 0, - blkno: 0, - offnum: 0, - data: TupleData::from_cells(&[Some(crate::decode::heap_decoder::ColumnValue::Int4(1))]), - pending: None, - }; - let (tx, rx) = mpsc::channel::>(1); - tx.send(vec![tuple(16400), tuple(16401)]).await.unwrap(); - drop(tx); - let mut obs = CountingObserver { - reject_xact_end: true, - ..Default::default() - }; - let err = drain_backfill(rx, &int4_catalog(), &mut obs) - .await - .unwrap_err(); - assert!(err.to_string().contains("mid-table xact end"), "{err}"); - assert_eq!(obs.on_tuple_calls, 1); - } - - #[tokio::test] - async fn drain_backfill_calls_on_xact_end_even_on_empty_channel() { - // Sender dropped without a tuple; `on_xact_end` still fires so the - // transitional emitter's INSERT cleanup runs unconditionally - let (tx, rx) = mpsc::channel::>(64); - drop(tx); - let mut obs = CountingObserver::default(); - let shipped = drain_backfill(rx, &int4_catalog(), &mut obs).await.unwrap(); - assert_eq!(shipped, 0); - assert_eq!(obs.on_tuple_calls, 0); - assert_eq!(obs.on_xact_end_calls, 1); - } } diff --git a/src/backfill/backup_page_walk.rs b/src/backfill/backup_page_walk.rs index e25f054d..364363d9 100644 --- a/src/backfill/backup_page_walk.rs +++ b/src/backfill/backup_page_walk.rs @@ -30,10 +30,9 @@ use crate::backfill::backup_source::{ use crate::backfill::copy_backfill::CopyRow; use crate::backfill::pg_path::{BaseRelFile, RelFork, parse_base_path}; use crate::backfill::walk_barrier::WalkBarrier; -use crate::decode::heap_decoder::{ - ColumnValue, CommittedTuple, DecodeError, DecodedHeap, DecodedTuple, HeapOp, TupleView, - decode_block_data, view_block_data, -}; +#[cfg(any(test, feature = "test-support"))] +use crate::decode::heap_decoder::{ColumnValue, decode_block_data}; +use crate::decode::heap_decoder::{DecodeError, TupleView, view_block_data}; use crate::decode::visibility::{ PendingXids, PgMultiXactAccum, PgXactAccum, PgXactView, SealedPatch, }; @@ -147,7 +146,8 @@ impl BackfillTuple { } } - /// Owned cells, attnum-1 indexed, for observers and tests + /// Owned cells, attnum-1 indexed + #[cfg(any(test, feature = "test-support"))] pub fn cells(&self, rel: &RelDescriptor) -> Result>, String> { match &self.data { TupleData::Heap(b) => decode_block_data(b, rel) @@ -156,27 +156,6 @@ impl BackfillTuple { TupleData::Copy(row) => crate::backfill::copy_backfill::copy_cells(row, rel), } } - - /// start_lsn rides as both source_lsn and commit_lsn so - /// ReplacingMergeTree(_lsn) collapses duplicates the WAL decoder - /// re-emits for records in [start_lsn, end_lsn] - pub fn into_committed_insert(self, rel: &RelDescriptor) -> Result { - Ok(CommittedTuple { - decoded: DecodedHeap { - rfn: self.rfn, - xid: self.xid, - source_lsn: self.source_lsn, - op: HeapOp::Insert, - new: Some(DecodedTuple { - columns: self.cells(rel)?, - partial: false, - }), - old: None, - }, - commit_ts: 0, - commit_lsn: self.source_lsn, - }) - } } /// Resolved `(db_node, rel_node) → RelDescriptor` map. Seeded before diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 1ceae5d6..9b12a8bd 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -766,6 +766,7 @@ pub(crate) fn copy_slot<'a>( } /// Owned cells of a COPY row, attnum-1 indexed +#[cfg(any(test, feature = "test-support"))] pub(crate) fn copy_cells( row: &CopyRow, rel: &RelDescriptor, @@ -775,6 +776,7 @@ pub(crate) fn copy_cells( .collect() } +#[cfg(any(test, feature = "test-support"))] fn owned(slot: Slot<'_>) -> ColumnValue { match slot { Slot::Cell(Cell::Value(v)) => v.to_owned_value(), diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index 0e8c2e02..1e06b36b 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -16,8 +16,8 @@ use crate::budget::{MemoryBudget, MemoryPermit, acquire_opt}; use crate::catalog::desc_log::{DescriptorLog, DescriptorLogs}; use crate::config::ResolvedConfig; use crate::decode::heap_decoder::{ - ColumnValue, DescribedHeap, HeapOp, XLH_DELETE_IS_SUPER, XLOG_HEAP_DELETE, - XLOG_HEAP_HOT_UPDATE, XLOG_HEAP_INSERT, XLOG_HEAP_OPMASK, XLOG_HEAP_UPDATE, view_heap_record, + ColumnValue, HeapOp, XLH_DELETE_IS_SUPER, XLOG_HEAP_DELETE, XLOG_HEAP_HOT_UPDATE, + XLOG_HEAP_INSERT, XLOG_HEAP_OPMASK, XLOG_HEAP_UPDATE, view_heap_record, }; use crate::decode::visibility::PgXactPatch; use crate::decode::wal_xact::{ @@ -39,8 +39,8 @@ use crate::source::wal_stream::WalStream; use crate::toast::{ChunkRefMap, ToastResolver, ToastRow, ToastRowRef}; use crate::xact::spill::{BodySpoolFile, DescribedInput}; use crate::xact::xact_buffer::{ - BufferingDecoderSink, DrainEntry, DrainHeap, DrainedBatch, SubxactTracker, WalkStep, - XactBuffer, detoast_heap, detoast_sites, external_sites, heap_reads_toast, resolve_stash, + BufferingDecoderSink, DrainEntry, DrainedBatch, SubxactTracker, WalkStep, XactBuffer, + detoast_sites, external_sites, resolve_stash, }; use foldhash::{HashMap, HashSet}; @@ -648,13 +648,9 @@ impl WalReplaySink { .await .map_err(SinkError::ctx("toast rewrite barrier"))?; } - WalkStep::Heap(drained) => { - let rel = drained.descriptor().clone(); - let rfn = match &drained { - DrainHeap::Cells(h) => h.decoded.rfn, - DrainHeap::Input(_) => rel.rfn, - }; - let rows = drained.rows() as u64; + WalkStep::Heap(input) => { + let rfn = input.descriptor.rfn; + let rows = u64::from(input.input.tuples); // Decode TOAST chunks, route through parent row let Some(target) = targets.get(&(rfn.db_node, rfn.rel_node)) else { continue; @@ -680,14 +676,7 @@ impl WalReplaySink { spool, ref_maps: &ref_maps, }; - match drained { - DrainHeap::Input(input) => { - Box::pin(self.replay_views(&input, &rel, route, &at)).await?; - } - DrainHeap::Cells(heap) => { - Box::pin(self.replay_cells(heap, route, &at)).await?; - } - } + Box::pin(self.replay_views(&input, route, &at)).await?; } } } @@ -710,10 +699,10 @@ impl WalReplaySink { async fn replay_views( &mut self, input: &DescribedInput, - rel: &Arc, route: &Arc, at: &Commit<'_>, ) -> std::result::Result<(), SinkError> { + let rel = &input.descriptor; let lsn = input.input.source_lsn; let images = view_heap_record(input.input.record_bytes(), rel) .map_err(|e| SinkError::Other(format!("wal_replay: decode at {lsn:#X}: {e}")))?; @@ -760,36 +749,6 @@ impl WalReplaySink { Ok(()) } - /// [`Self::replay_views`] for owned cells - async fn replay_cells( - &mut self, - mut heap: DescribedHeap, - route: &Arc, - at: &Commit<'_>, - ) -> std::result::Result<(), SinkError> { - if !self.lands(route, heap.decoded.op) { - return Ok(()); - } - // Gate stays as wide as detoast's leaf acquire: narrowing it to - // store-bound pointers would hold pending leaf permits across that - // acquire, which can wait on them - if heap_reads_toast(&heap) { - self.flush_rows().await?; - } - let permit = detoast_heap(&mut heap, at.spool, at.ref_maps, &self.resolver) - .await - .map_err(SinkError::from)?; - let rel = &heap.descriptor; - let key = Arc::as_ptr(rel) as usize; - let rows = self - .chunks - .push_cells(key, rel, route, heap.decoded, at.ts) - .map_err(encode_err)?; - let seq = self.place_rows(at.lsn, rows); - self.replay.rows_replayed += 1; - self.encoded(seq, permit).await - } - async fn handle(&mut self, record: &Record<'_>) -> std::result::Result<(), SinkError> { let rm = record.parsed.header.resource_manager_id; // Capture TOAST page images from Heap and XLOG resource managers diff --git a/src/bin/stream/bootstrap.rs b/src/bin/stream/bootstrap.rs index aa4dc985..83bbd9bf 100644 --- a/src/bin/stream/bootstrap.rs +++ b/src/bin/stream/bootstrap.rs @@ -18,7 +18,7 @@ use walshadow::backfill::visibility_gate::{ DeferredLane, GateStats, GreenfieldSink, PendingGate, resolve_greenfield, stream_phase, }; use walshadow::backfill_bootstrap::{ - BootstrapConfig, BootstrapOutcome, BootstrapProgress, drain_backfill, seed_in_snapshot, + BootstrapConfig, BootstrapOutcome, BootstrapProgress, seed_in_snapshot, spawn_greenfield_bootstrap, }; use walshadow::backup_source::BackupSource; @@ -27,7 +27,6 @@ use walshadow::backup_source_object_store::ObjectStoreSource; use walshadow::bootstrap_marker::{self, BootstrapMarker}; use walshadow::ch_emitter::{BootstrapMode, EmitterConfig, EmitterStats}; use walshadow::config::{CliOverrides, ConfigResolver, cli_over_toml}; -use walshadow::decoder_sink::MetricsTupleObserver; use walshadow::emit::route::routing; use walshadow::mapping::MappingHandle; use walshadow::metrics::{MetricsRegistry, MetricsSnapshot}; @@ -128,7 +127,7 @@ fn refuse_unowned_cluster(previous: Option<&BootstrapMarker>, dir: &Path) -> Res /// (synthetic INSERT `_lsn = start_lsn`, `_commit_ts = 0`, `_is_deleted = 0`). /// `wait_through(K)` proves every bootstrap seq durable on CH before /// teardown, so the WAL pump resumes against a fully-shipped baseline. -/// `None`: rows drain to a metrics-only observer via `drain_backfill`. +/// `None`: rows drain counted only. pub(crate) async fn run_bootstrap( src_cfg: &PgConfig, feed: &mut SourceFeed, @@ -981,11 +980,17 @@ pub(crate) async fn run_bootstrap( }); (rows_routed, outcome, window, landing_tracker) } else { - // Metrics-only skips destination convergence - let mut observer = MetricsTupleObserver::default(); - let (drain_res, pump_res) = - tokio::join!(drain_backfill(rx, &drain_catalog, &mut observer), pump); - let shipped = drain_res.context("bootstrap drain")?; + // Metrics-only skips destination convergence; the walk already + // validated every row against its descriptor + let count = async move { + let mut rx = rx; + let mut shipped = 0u64; + while let Some(slab) = rx.recv().await { + shipped += slab.len() as u64; + } + shipped + }; + let (shipped, pump_res) = tokio::join!(count, pump); let outcome: BootstrapOutcome = pump_res .context("bootstrap pump join")? .context("bootstrap pump")?; diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 5f64f426..154796ee 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -235,8 +235,6 @@ pub(crate) struct DrainResident { pub(crate) chunks: u64, pub(crate) rows: u64, pub(crate) spool: u64, - pub(crate) raw_pending_rows: u64, - pub(crate) raw_pending_bytes: u64, } impl DrainResident { @@ -246,8 +244,6 @@ impl DrainResident { chunks: b.drain_chunk_resident_bytes(), rows: b.drain_row_resident_bytes(), spool: b.toast_spool_bytes(), - raw_pending_rows: b.raw_pending_rows(), - raw_pending_bytes: b.raw_pending_bytes(), } } } @@ -431,8 +427,6 @@ pub(crate) async fn populate_pipeline_metrics( xact_stats.raw_stash_bytes_mem, xact_stats.raw_stash_bytes_spill, ], - raw_pending_rows: drain_resident.raw_pending_rows, - raw_pending_bytes: drain_resident.raw_pending_bytes, pump_queue_depth, queue_records_out_total, catalog_boundary_holds_total: boundary_hold.holds.load(Ordering::Relaxed), @@ -757,8 +751,6 @@ mod tests { chunks: 0, rows: n * 10, spool: 0, - raw_pending_rows: n, - raw_pending_bytes: n * 10, }, budget: None, decoder_stats: &decoder, @@ -782,7 +774,6 @@ mod tests { assert_eq!(snap.pump_queue_depth, n); assert_eq!(snap.queue_records_out_total, n * 3); assert_eq!(snap.drain_resident_bytes, n * 10); - assert_eq!(snap.raw_pending_rows, n); assert_eq!(snap.uptime_seconds, n); assert_eq!(snap.archive_restore_active, 1); assert_eq!(snap.archive_wal_segments_total, 42); diff --git a/src/decode/decoder_sink.rs b/src/decode/decoder_sink.rs index 7708a066..3d70518a 100644 --- a/src/decode/decoder_sink.rs +++ b/src/decode/decoder_sink.rs @@ -1,20 +1,15 @@ -//! Shared types for the heap-tuple decoder fan-out: [`TupleObserver`], -//! [`DecoderStats`], [`DecoderSinkError`]. -//! -//! WAL commits drain through the parallel [`pipeline`](crate::emit::pipeline); -//! [`TupleObserver`] is the greenfield-bootstrap page-walk drain destination -//! (`drain_backfill`). Catalog gate timeouts poison the stream, so no +//! Shared types for the heap-tuple decoder fan-out: [`DecoderStats`], +//! [`DecoderSinkError`]. WAL commits drain through the parallel +//! [`pipeline`](crate::emit::pipeline). Catalog gate timeouts poison the stream, so no //! `replay_timeout` counter: silent loss is impossible by construction. -use std::future::Future; -use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use thiserror::Error; use crate::catalog::shadow_catalog::CatalogError; use crate::ch::EmitterError; -use crate::decode::heap_decoder::{CommittedTuple, DecodeError}; +use crate::decode::heap_decoder::DecodeError; use crate::record::SinkError; use walrus::pg::walparser::{RmId, XLogRecordHeader}; @@ -40,28 +35,6 @@ impl From for DecoderSinkError { } } -/// Destination of decoded + committed heap events. Wire type is -/// [`CommittedTuple`] not [`DecodedHeap`](crate::decode::heap_decoder::DecodedHeap): -/// CH emitter needs `commit_ts` for its `_commit_ts` synthetic column. -/// -/// `on_xact_end` fires after every tuple in a committed xact is delivered. -/// Returns highest commit_lsn now durable on the observer. Callers advance -/// their ack ceiling from the return value, NOT `commit_lsn`. Default -/// returns `commit_lsn` (instant ack). -pub trait TupleObserver: Send { - fn on_tuple<'a>( - &'a mut self, - committed: &'a CommittedTuple, - ) -> Pin> + Send + 'a>>; - - fn on_xact_end<'a>( - &'a mut self, - commit_lsn: u64, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { Ok(commit_lsn) }) - } -} - crate::atomic_stats! { pub struct DecoderStats { pub decoded, @@ -168,50 +141,7 @@ pub struct SkipKey { pub type SkipCounts = crate::emit::pipeline::row_ledger::KeyedCounts; -#[derive(Debug, Default)] -pub struct MetricsTupleObserver { - pub stats: DecoderStats, -} - -impl TupleObserver for MetricsTupleObserver { - fn on_tuple<'a>( - &'a mut self, - committed: &'a CommittedTuple, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - self.stats.record(&committed.decoded); - Ok(()) - }) - } -} - -/// In-memory test observer; retains full committed tuples for assertions. -#[derive(Debug, Default)] -pub struct CollectingTupleObserver { - pub tuples: Vec, -} - -impl TupleObserver for CollectingTupleObserver { - fn on_tuple<'a>( - &'a mut self, - committed: &'a CommittedTuple, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - self.tuples.push(committed.clone()); - Ok(()) - }) - } -} - impl DecoderStats { - /// Single source of truth for the `HeapOp → counter` mapping; all decoder - /// dispatch sites route through here. - pub fn record(&self, decoded: &crate::decode::heap_decoder::DecodedHeap) { - let partial = decoded.new.as_ref().is_some_and(|t| t.partial) - || decoded.old.as_ref().is_some_and(|t| t.partial); - self.record_op(decoded.op, partial); - } - pub fn record_op(&self, op: crate::decode::heap_decoder::HeapOp, partial: bool) { use crate::decode::heap_decoder::HeapOp; self.decoded.fetch_add(1, Ordering::Relaxed); @@ -338,31 +268,11 @@ impl DecoderStats { #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, HeapOp}; - use walrus::pg::walparser::RelFileNode; + use crate::decode::heap_decoder::HeapOp; - fn wrap(decoded: DecodedHeap) -> CommittedTuple { - CommittedTuple { - decoded, - commit_ts: 0, - commit_lsn: 0, - } - } - - #[tokio::test(flavor = "current_thread")] - async fn metrics_observer_buckets_by_op() { - let mut obs = MetricsTupleObserver::default(); - let mk = |op: HeapOp, partial: bool| DecodedHeap { - rfn: RelFileNode::default(), - xid: 0, - source_lsn: 0, - op, - new: Some(crate::decode::heap_decoder::DecodedTuple { - columns: vec![Some(ColumnValue::Null)], - partial, - }), - old: None, - }; + #[test] + fn record_op_buckets_by_op() { + let s = DecoderStats::default(); for op in [ HeapOp::Insert, HeapOp::Insert, @@ -370,10 +280,9 @@ mod tests { HeapOp::HotUpdate, HeapOp::Delete, ] { - obs.on_tuple(&wrap(mk(op, false))).await.unwrap(); + s.record_op(op, false); } - obs.on_tuple(&wrap(mk(HeapOp::Update, true))).await.unwrap(); - let s = &obs.stats; + s.record_op(HeapOp::Update, true); let ld = |a: &AtomicU64| a.load(Ordering::Relaxed); assert_eq!(ld(&s.decoded), 6); assert_eq!(ld(&s.inserts), 2); @@ -383,36 +292,6 @@ mod tests { assert_eq!(ld(&s.partial), 1); } - #[tokio::test(flavor = "current_thread")] - async fn collecting_observer_keeps_full_clone() { - let mut obs = CollectingTupleObserver::default(); - let d = DecodedHeap { - rfn: RelFileNode { - spc_node: 1663, - db_node: 5, - rel_node: 16385, - }, - xid: 42, - source_lsn: 0x1234, - op: HeapOp::Insert, - new: Some(crate::decode::heap_decoder::DecodedTuple { - columns: vec![Some(ColumnValue::Int4(7))], - partial: false, - }), - old: None, - }; - let c = CommittedTuple { - decoded: d, - commit_ts: 9_876, - commit_lsn: 0, - }; - obs.on_tuple(&c).await.unwrap(); - assert_eq!(obs.tuples.len(), 1); - assert_eq!(obs.tuples[0].decoded.xid, 42); - assert_eq!(obs.tuples[0].decoded.source_lsn, 0x1234); - assert_eq!(obs.tuples[0].commit_ts, 9_876); - } - #[test] fn stats_summary_skips_zero_buckets() { let s = DecoderStats::default(); diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index a67d6def..ccb31fb3 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -345,6 +345,7 @@ pub struct DecodedHeap { pub old: Option, } +#[cfg(any(test, feature = "test-support"))] impl DecodedHeap { /// Sum across both images (REPLICA IDENTITY FULL keeps a fat `old` too). pub fn approx_bytes(&self) -> usize { @@ -367,6 +368,7 @@ impl DecodedHeap { /// between decode and drain can't reinterpret the row. Spill serializes /// `descriptor_valid_from` instead of the `Arc`; rehydrate re-obtains the /// descriptor by key and fails closed on span mismatch +#[cfg(any(test, feature = "test-support"))] #[derive(Debug, Clone, PartialEq)] pub struct DescribedHeap { pub decoded: DecodedHeap, @@ -375,6 +377,7 @@ pub struct DescribedHeap { pub descriptor_valid_from: u64, } +#[cfg(any(test, feature = "test-support"))] impl DescribedHeap { /// Resident accounting counts the retained `Arc` reference (contents /// shared, not per-row) plus the span key @@ -399,6 +402,7 @@ pub enum HeapOp { Truncate, } +#[cfg(any(test, feature = "test-support"))] /// One drained tuple, fully reassembled. `commit_ts` is the commit-record /// `xact_time` carried through /// [`crate::xact::xact_buffer::XactBuffer::drain_committed`]. @@ -478,6 +482,7 @@ impl<'a> TupleBuild<'a> for TupleView<'a> { } } +#[cfg(test)] impl TupleView<'_> { /// Owned cells, as [`decode_heap_record`] builds them pub fn to_tuple(&self, rel: &RelDescriptor) -> DecodedTuple { @@ -812,10 +817,8 @@ fn decode_old_tuple_from_main_data<'a, T: TupleBuild<'a>>( Ok(Some(decode_tuple_payload(main_data, header_off, rel, 0)?)) } -/// Decode a `block.data`-shaped tuple for the bootstrap page-walk sink, -/// which reshapes on-disk `HeapTupleHeaderData`-prefixed tuples into this -/// form. `prefixlen=0`: WAL prefix/suffix-from-old compression -/// doesn't apply at backup time. +/// Owned cells of a `block.data`-shaped tuple +#[cfg(any(test, feature = "test-support"))] pub(crate) fn decode_block_data( block_data: &[u8], rel: &RelDescriptor, @@ -849,7 +852,7 @@ pub fn scan_block_data(block_data: &[u8], rel: &RelDescriptor) -> Result]) -> Vec { let natts = values.len(); @@ -890,7 +893,11 @@ pub fn form_block_data(values: &[Option]) -> Vec { Some(ColumnValue::Float8(f)) => (f.to_le_bytes().to_vec(), 8), Some(ColumnValue::Uuid(u)) => (u.to_vec(), 1), Some(ColumnValue::Text(t) | ColumnValue::Json(t)) => varlena(t.as_bytes()), - Some(ColumnValue::Bytea(b)) => varlena(b), + Some( + ColumnValue::Bytea(b) + | ColumnValue::PgPending { raw: b, .. } + | ColumnValue::Unsupported { raw: b, .. }, + ) => varlena(b), Some(ColumnValue::ExternalToast(p)) => { let mut v = vec![0x01, 18]; for word in [ @@ -1710,27 +1717,6 @@ pub(crate) fn toast_chunk_datums<'a>(datums: &[DatumRef<'a>]) -> Option<(u32, u3 )) } -/// Take `(chunk_id, chunk_seq, chunk_data)` out of a `pg_toast_*` tuple's -/// 3 columns (`chunk_id oid`, `chunk_seq int4`, `chunk_data bytea`); -/// `None` for shapes that don't fit -pub(crate) fn take_toast_chunk_columns( - cols: &mut [Option], -) -> Option<(u32, u32, Vec)> { - if cols.len() < 3 { - return None; - } - let &ColumnValue::Oid(chunk_id) = cols[0].as_ref()? else { - return None; - }; - let &ColumnValue::Int4(chunk_seq) = cols[1].as_ref()? else { - return None; - }; - let ColumnValue::Bytea(chunk_data) = cols[2].take()? else { - return None; - }; - Some((chunk_id, chunk_seq as u32, chunk_data)) -} - /// True iff `attnum` is part of the relation's replica identity. Emitter gates /// delete/update old-key propagation on this, staying agnostic to [`ReplIdent`]. pub fn is_replica_identity_attr(replident: &ReplIdent, attnum: i16) -> bool { @@ -1745,39 +1731,6 @@ pub fn is_replica_identity_attr(replident: &ReplIdent, attnum: i16) -> bool { } } -/// Build a delete marker when UPDATE changes primary/index key, or any column -/// under `REPLICA IDENTITY FULL` without a primary key. Use `lsn - 1` so new -/// row wins if both share a destination key. PostgreSQL default/index identity -/// logs old values only when key changes or contains TOASTed values -pub fn take_key_change_tombstone( - heap: &mut DecodedHeap, - rel: &RelDescriptor, -) -> Option { - let hot = match heap.op { - HeapOp::Update => false, - HeapOp::HotUpdate => true, - _ => return None, - }; - let (old, new) = (heap.old.as_ref()?, heap.new.as_ref()?); - let differs = |a: usize| matches!((old.columns.get(a), new.columns.get(a)), (Some(Some(o)), Some(Some(n))) if o != n); - let keys = crate::schema::replident_key_attnums(rel); - let moved = if keys.is_empty() { - matches!(rel.replident, ReplIdent::Full { pk_attnums: None }) - && (0..new.columns.len()).any(differs) - } else { - // PostgreSQL HOT updates leave indexed columns unchanged - !hot && keys.iter().any(|&a| differs(a as usize - 1)) - }; - moved.then(|| DecodedHeap { - rfn: heap.rfn, - xid: heap.xid, - source_lsn: heap.source_lsn - 1, - op: HeapOp::Delete, - new: None, - old: heap.old.take(), - }) -} - /// PG `attmissingval`: one-element, no-null `ArrayType`, payload at `MAXALIGN(24)` #[cfg(test)] pub(crate) fn raw_missing_array(elemtype: u32, elem: &[u8]) -> Vec { @@ -2942,112 +2895,6 @@ mod tests { assert!(!is_replica_identity_attr(&idx, 1)); } - #[test] - fn key_change_tombstone_matrix() { - let tuple = |id: i32, v: &str| DecodedTuple { - columns: vec![ - Some(ColumnValue::Int4(id)), - Some(ColumnValue::Text(v.into())), - ], - partial: false, - }; - let heap = |op, old: Option| DecodedHeap { - rfn: RelFileNode::default(), - xid: 7, - source_lsn: 0x100, - op, - new: Some(tuple(2, "b")), - old, - }; - let mut rel = descriptor(1, vec![]); - let pk = Some(vec![1]); - let cases = [ - ( - ReplIdent::Default { - pk_attnums: pk.clone(), - }, - HeapOp::Update, - Some(tuple(1, "")), - true, - ), - ( - ReplIdent::Default { - pk_attnums: pk.clone(), - }, - HeapOp::Update, - None, - false, - ), - // PostgreSQL logs TOASTed keys even when unchanged - ( - ReplIdent::Default { - pk_attnums: pk.clone(), - }, - HeapOp::Update, - Some(tuple(2, "")), - false, - ), - ( - ReplIdent::Full { - pk_attnums: pk.clone(), - }, - HeapOp::Update, - Some(tuple(2, "a")), - false, - ), - ( - ReplIdent::Full { - pk_attnums: pk.clone(), - }, - HeapOp::HotUpdate, - Some(tuple(1, "a")), - false, - ), - ( - ReplIdent::Full { pk_attnums: None }, - HeapOp::HotUpdate, - Some(tuple(2, "a")), - true, - ), - ( - ReplIdent::Full { pk_attnums: None }, - HeapOp::Update, - Some(tuple(2, "b")), - false, - ), - ( - ReplIdent::Full { pk_attnums: None }, - HeapOp::Delete, - Some(tuple(2, "a")), - false, - ), - ( - ReplIdent::UsingIndex { - index_oid: 9, - key_attnums: vec![2], - }, - HeapOp::Update, - Some(tuple(2, "a")), - true, - ), - ]; - for (replident, op, old, split) in cases { - rel.replident = replident; - let mut h = heap(op, old.clone()); - let tomb = take_key_change_tombstone(&mut h, &rel); - assert_eq!(tomb.is_some(), split, "{:?} {op:?} {old:?}", rel.replident); - let Some(tomb) = tomb else { - assert_eq!(h.old, old); - continue; - }; - assert_eq!( - (tomb.op, tomb.source_lsn, tomb.xid), - (HeapOp::Delete, 0xff, 7) - ); - assert_eq!((tomb.old, tomb.new, h.old), (old, None, None)); - } - } - #[test] fn heap_op_index_maps_every_op() { let (heap, heap2) = (RmId::Heap as u8, RmId::Heap2 as u8); @@ -3422,24 +3269,6 @@ mod tests { assert!(local_matrix_covers(2275, -2)); } - #[test] - fn take_toast_chunk_columns_rejects_misshapen_rows() { - let good = || { - vec![ - Some(ColumnValue::Oid(7)), - Some(ColumnValue::Int4(2)), - Some(ColumnValue::Bytea(vec![1])), - ] - }; - assert_eq!(take_toast_chunk_columns(&mut good()), Some((7, 2, vec![1]))); - assert_eq!(take_toast_chunk_columns(&mut good()[..2].to_vec()), None); - for slot in 0..3 { - let mut cols = good(); - cols[slot] = Some(ColumnValue::Text("x".into())); - assert_eq!(take_toast_chunk_columns(&mut cols), None, "slot {slot}"); - } - } - #[test] fn fast_default_raw_honours_dataoffset_and_rejects_multidim() { let mut att = rel_attr(1, "x", INT4OID, 4, 'i'); diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 5a4f9b69..2ccc6101 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -37,7 +37,9 @@ use crate::column_rules::{ColumnEntry, ColumnRule, ColumnRules, Substitute}; use crate::decode::datum::{DatumRef, ValueRef}; #[cfg(test)] use crate::decode::decoder_sink::DecoderSinkError; -use crate::decode::heap_decoder::{ColumnValue, CommittedTuple, HeapOp}; +use crate::decode::heap_decoder::ColumnValue; +#[cfg(any(test, feature = "test-support"))] +use crate::decode::heap_decoder::{CommittedTuple, HeapOp}; use crate::decode::heap_decoder::{datum_value, datum_view}; use crate::mapping::{ ColumnMapping, DropTableStrategy, NamespaceMapping, SystemColumnNames, SystemColumns, @@ -2379,8 +2381,9 @@ impl TableEncoder { self.append_routed(committed, mapping, op_code, None) } - /// Append one row, stamping `leaf` into the partition column. Return - /// [`Append::Full`] without mutation when oracle threshold is hit + #[cfg(any(test, feature = "test-support"))] + /// Append one owned row, stamping `leaf` into the partition column. + /// Return [`Append::Full`] without mutation when oracle threshold is hit pub fn append_routed( &mut self, committed: &CommittedTuple, @@ -2412,7 +2415,8 @@ impl TableEncoder { self.append_cells(|i| cells[i], &stamp) } - /// [`Self::append_routed`] reading one image of `rel` in place. `patch` + /// Append one image of `rel` read in place. Return [`Append::Full`] + /// without mutation when oracle threshold is hit. `patch` /// replaces attributes by index: resolved TOAST values, and values past /// `rel`'s attributes. Values whose decode allocates, and PostGIS /// renders, build in row scratch diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index db5a528c..8b67ea23 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -449,6 +449,7 @@ async fn resolve_spooled( total_placed += placed; placed = 0; if ticker.fire() { + out.flush().await?; c.tail.checkpoint(out.seqs.next()).await?; c.state.offset = total_bytes - remaining; c.state.rows += total_placed; @@ -460,6 +461,7 @@ async fn resolve_spooled( } let rows_routed = match checkpoint.as_mut() { Some(c) => { + out.flush().await?; c.tail.checkpoint(out.seqs.next()).await?; c.state.offset = total_bytes; c.state.rows += total_placed; diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index a95a5656..270153d0 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -27,15 +27,13 @@ use thiserror::Error; use crate::ch::EmitterError; use crate::decode::datum::DatumRef; -use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, HeapOp, TupleView, datum_value, take_key_change_tombstone, -}; +use crate::decode::heap_decoder::{ColumnValue, HeapOp, TupleView, datum_value}; use crate::emit::ch_emitter::{ Append, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, RowStamp, TableEncoder, TablePlan, }; use crate::emit::route::RouteSnapshot; use crate::schema::RelDescriptor; -use crate::xact::spill::{self, Cursor, IO_CHUNK, SpillError}; +use crate::xact::spill::{self, Cursor, IO_CHUNK, SpillError, Truncate}; use crate::xact::xact_buffer::{DrainEntry, ToastRowBatch}; use foldhash::HashMap; @@ -68,8 +66,9 @@ impl PlanTable { /// Truncate under its plan table #[derive(Debug)] -pub struct PlannedHeap { - pub decoded: DecodedHeap, +pub struct PlannedTruncate { + pub xid: u32, + pub source_lsn: u64, /// Index into plan's `tables` pub table: u32, } @@ -158,7 +157,7 @@ enum Step { row_idx: usize, }, Truncate { - heap: PlannedHeap, + truncate: PlannedTruncate, upto: usize, }, } @@ -295,47 +294,9 @@ impl ChunkBuilder { Ok(()) } - /// Encode one owned row change: drop deletes an append-only route has - /// nowhere to land, render PostGIS cells, split key changes into a - /// delete marker plus the new row. Returns rows appended - pub fn push_cells( - &mut self, - key: K, - rel: &Arc, - route: &Arc, - mut decoded: DecodedHeap, - commit_ts: i64, - ) -> std::result::Result { - if route.drops_deletes() && decoded.op == HeapOp::Delete { - return Ok(0); - } - for t in [decoded.new.as_mut(), decoded.old.as_mut()] - .into_iter() - .flatten() - { - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); - } - let tomb = (!route.drops_deletes()) - .then(|| take_key_change_tombstone(&mut decoded, rel)) - .flatten(); - let leaf = route.partition.as_ref().map(|p| p.leaf); - let mut rows = 0; - for decoded in tomb.into_iter().chain([decoded]) { - let op = op_code(decoded.op)?; - let committed = crate::decode::heap_decoder::CommittedTuple { - decoded, - commit_ts, - commit_lsn: 0, - }; - self.append(key, rel, route, |enc| { - enc.append_routed(&committed, &route.mapping, op, leaf) - })?; - rows += 1; - } - Ok(rows) - } - - /// [`Self::push_cells`] reading tuple images in place + /// Encode one row change from tuple images in place: drop deletes an + /// append-only route has nowhere to land, split key changes into a delete + /// marker plus the new row. Returns rows appended pub fn push_views( &mut self, key: K, @@ -545,18 +506,40 @@ impl PlanWriter { self.tables[table as usize].heaps += rows; } - /// Encode one owned row change, see [`ChunkBuilder::push_cells`] - pub fn push_cells(&mut self, decoded: DecodedHeap, table: u32) -> Result<()> { - let t = &mut self.tables[table as usize]; - t.heaps += 1; - let route = t.route.as_ref().expect("routed table"); - let rows = self - .chunks - .push_cells(table, &t.descriptor, route, decoded, self.commit_ts)?; - self.pushed(rows) + #[cfg(any(test, feature = "test-support"))] + /// [`Self::push_views`] over an owned change's record bytes + pub fn push_cells( + &mut self, + decoded: crate::decode::heap_decoder::DecodedHeap, + table: u32, + ) -> Result<()> { + let input = spill::DescribedInput::from(crate::decode::heap_decoder::DescribedHeap { + decoded, + descriptor: self.tables[table as usize].descriptor.clone(), + descriptor_valid_from: 0, + }); + let images = crate::decode::heap_decoder::view_heap_record( + input.input.record_bytes(), + &input.descriptor, + ) + .expect("formed record views"); + for h in &images { + let change = ViewChange { + op: h.op, + xid: input.input.xid, + source_lsn: input.input.source_lsn, + new: h.new.as_ref(), + old: h.old.as_ref(), + new_patch: &[], + old_patch: &[], + }; + self.push_views(change, table)?; + } + Ok(()) } - /// [`Self::push_cells`] reading tuple images in place + /// Encode one change from tuple images in place, see + /// [`ChunkBuilder::push_views`] pub fn push_views(&mut self, change: ViewChange<'_, '_>, table: u32) -> Result<()> { let t = &mut self.tables[table as usize]; t.heaps += 1; @@ -661,13 +644,15 @@ impl PlanWriter { /// Append `HeapOp::Truncate`; executor stores mirror rows with indices /// below `upto`, then applies truncate - pub fn push_truncate(&mut self, decoded: DecodedHeap, table: u32, upto: usize) -> Result<()> { + pub fn push_truncate(&mut self, t: Truncate, table: u32, upto: usize) -> Result<()> { self.seal_segment()?; self.tables[table as usize].heaps += 1; - self.steps.push(Step::Truncate { - heap: PlannedHeap { decoded, table }, - upto, - }); + let truncate = PlannedTruncate { + xid: t.xid, + source_lsn: t.source_lsn, + table, + }; + self.steps.push(Step::Truncate { truncate, upto }); Ok(()) } @@ -742,7 +727,9 @@ pub struct ViewChange<'v, 'a> { pub old_patch: &'v [(usize, ColumnValue)], } -/// [`take_key_change_tombstone`]'s test over tuple images in place +/// Whether an UPDATE changes primary/index key, or any column under +/// `REPLICA IDENTITY FULL` without a primary key. PostgreSQL default/index +/// identity logs old values only when key changes or contains TOASTed values fn key_moved( rel: &RelDescriptor, change: &ViewChange<'_, '_>, @@ -910,7 +897,7 @@ pub enum PlanItem { Segment(Segment), /// Mirror rows below `upto` put before the truncate applies Truncate { - heap: PlannedHeap, + truncate: PlannedTruncate, upto: usize, }, } @@ -942,7 +929,7 @@ impl PlanReader { }; Ok(Some(match step { Step::Control { event, row_idx } => PlanItem::Control { event, row_idx }, - Step::Truncate { heap, upto } => PlanItem::Truncate { heap, upto }, + Step::Truncate { truncate, upto } => PlanItem::Truncate { truncate, upto }, Step::Segment { count, rows, @@ -1012,7 +999,7 @@ fn read_frame(f: &mut FileInput, tables: &[PlanTable]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{ColumnValue, DecodedTuple}; + use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple}; use crate::emit::ch_emitter::Wire; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{INT4OID, RelAttr, RelName, SchemaEvent}; @@ -1050,6 +1037,67 @@ mod tests { }) } + /// Delete marker splits off when UPDATE moves a key, or any column under + /// `REPLICA IDENTITY FULL` without a primary key + #[test] + fn key_moved_matrix() { + use crate::decode::heap_decoder::{form_block_data, view_block_data}; + use crate::schema::ReplIdent; + let mut rel = (*descriptor(1)).clone(); + let mut v = rel.attributes[0].clone(); + (v.attnum, v.name, v.type_oid) = (2, "v".into(), crate::schema::TEXTOID); + (v.type_name, v.type_byval, v.type_len) = ("text".into(), false, -1); + rel.attributes.push(v); + let row = |id: i32, v: &str| { + form_block_data(&[ + Some(ColumnValue::Int4(id)), + Some(ColumnValue::Text(v.into())), + ]) + }; + let pk = || ReplIdent::Default { + pk_attnums: Some(vec![1]), + }; + let full_pk = || ReplIdent::Full { + pk_attnums: Some(vec![1]), + }; + let full = || ReplIdent::Full { pk_attnums: None }; + let index = ReplIdent::UsingIndex { + index_oid: 9, + key_attnums: vec![2], + }; + let cases = [ + (pk(), HeapOp::Update, (1, ""), true), + // PostgreSQL logs TOASTed keys even when unchanged + (pk(), HeapOp::Update, (2, ""), false), + (full_pk(), HeapOp::Update, (2, "a"), false), + (full_pk(), HeapOp::HotUpdate, (1, "a"), false), + (full(), HeapOp::HotUpdate, (2, "a"), true), + (full(), HeapOp::Update, (2, "b"), false), + (full(), HeapOp::Delete, (2, "a"), false), + (index, HeapOp::Update, (2, "a"), true), + ]; + let new = row(2, "b"); + for (replident, op, (id, v), split) in cases { + rel.replident = replident; + let old = row(id, v); + let (old, new) = ( + view_block_data(&old, &rel).unwrap(), + view_block_data(&new, &rel).unwrap(), + ); + let change = ViewChange { + op, + xid: 7, + source_lsn: 0x100, + new: Some(&new), + old: Some(&old), + new_patch: &[], + old_patch: &[], + }; + let moved = key_moved(&rel, &change, &old, &new); + assert_eq!(moved, split, "{:?} {op:?} ({id}, {v:?})", rel.replident); + } + } + fn heap(rel: &Arc, lsn: u64, v: i32) -> DecodedHeap { DecodedHeap { rfn: rel.rfn, @@ -1161,8 +1209,12 @@ mod tests { w.push_control(dropped(1), 0).unwrap(); push(&mut w, &d1, heap(&d1, 100, 10), Some(r.clone())).unwrap(); push(&mut w, &d2, heap(&d2, 110, 20), None).unwrap(); - let mut truncate = heap(&d2, 115, 25); - truncate.op = HeapOp::Truncate; + let truncate = Truncate { + xid: 7, + source_lsn: 115, + descriptor: d2.clone(), + descriptor_valid_from: 0x40, + }; let table = w.table(&d2, 0x40, || unreachable!("table live")); w.push_truncate(truncate, table, 0).unwrap(); w.push_control(dropped(2), 0).unwrap(); @@ -1197,9 +1249,9 @@ mod tests { } format!("{:?}", rows(&s.chunks)) } - PlanItem::Truncate { heap: h, .. } => { - assert!(rd.tables[h.table as usize].route.is_none()); - format!("t{}", h.decoded.source_lsn) + PlanItem::Truncate { truncate: t, .. } => { + assert!(rd.tables[t.table as usize].route.is_none()); + format!("t{}", t.source_lsn) } }); } diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 75d9cf1b..6f9de741 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -39,7 +39,7 @@ use std::sync::Arc; -use crate::decode::heap_decoder::{ColumnValue, view_heap_record}; +use crate::decode::heap_decoder::{ColumnValue, HeapImages, TupleView, view_heap_record}; use crate::emit::pipeline::plan_spool::{ DEFAULT_PLAN_MEM_MAX, EncoderCache, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, @@ -49,8 +49,8 @@ use crate::schema::RelDescriptor; use crate::toast::{ChunkRefMap, ToastResolver}; use crate::xact::spill::{BodySpoolFile, DescribedInput}; use crate::xact::xact_buffer::{ - DrainEntry, DrainHeap, DrainWalk, DrainedBatch, FailClosedReason, WalkStep, XactBufferError, - detoast_heap, detoast_sites, external_sites, heap_reads_toast, + DrainEntry, DrainWalk, DrainedBatch, FailClosedReason, WalkStep, XactBufferError, + detoast_sites, external_sites, }; /// Planner's window onto route state. Implementations resolve from frozen @@ -188,47 +188,31 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { .map_err(PlanError::View)?; self.writer.push_control(e, self.rows_base + cursor)?; } - WalkStep::Truncate { heap, upto } => { + WalkStep::Truncate { truncate, upto } => { self.resolver.clear_reads(); - let table = - self.writer - .table(&heap.descriptor, heap.descriptor_valid_from, || { - self.view.route_for(&heap.descriptor) - }); + let rel = &truncate.descriptor; + let table = self.writer.table(rel, truncate.descriptor_valid_from, || { + self.view.route_for(rel) + }); self.writer - .push_truncate(heap.decoded, table, self.rows_base + upto)?; + .push_truncate(*truncate, table, self.rows_base + upto)?; } - WalkStep::Heap(drained) => { + WalkStep::Heap(input) => { // Route before validation and detoast: unmapped rows // discard without touching the resolver or codec checks - let rel = drained.descriptor().clone(); - let table = self - .writer - .table(&rel, drained.descriptor_valid_from(), || { - self.view.route_for(&rel) - }); + let rel = &input.descriptor; + let table = self.writer.table(rel, input.descriptor_valid_from, || { + self.view.route_for(rel) + }); if !self.writer.routed(table) { - self.writer.discard(table, drained.rows() as u64); + self.writer.discard(table, u64::from(input.input.tuples)); continue; } - let mut heap = match drained { - DrainHeap::Input(input) => { - Box::pin(self.plan_views(&input, &rel, table, spool, &ref_maps)) - .await?; - continue; - } - DrainHeap::Cells(heap) => heap, - }; - if heap.decoded.new.as_ref().is_some_and(|t| t.partial) { - return Err(PlanError::PartialUpdate { - lsn: heap.decoded.source_lsn, - }); - } - if heap_reads_toast(&heap) { - // Value permit drops once bytes land in the plan - Box::pin(detoast_heap(&mut heap, spool, &ref_maps, self.resolver)).await?; + if !self.plan_inline(&input, table)? { + // Boxed: external values are rare, and awaiting them + // holds tuple views deep enough to bloat every plan future + Box::pin(self.plan_views(&input, table, spool, &ref_maps)).await?; } - self.writer.push_cells(heap.decoded, table)?; } } } @@ -237,16 +221,11 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Ok(()) } - /// Encode an input from tuple images in place, external values resolved - /// into per-image patches - async fn plan_views( - &mut self, - input: &DescribedInput, - rel: &Arc, - table: u32, - spool: Option<&BodySpoolFile>, - ref_maps: &[&ChunkRefMap], - ) -> Result<(), PlanError> { + /// Tuple images of `input`, rejecting partial new images + fn views<'i>( + input: &'i DescribedInput, + ) -> Result>; 1]>, PlanError> { + let rel = &input.descriptor; let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { XactBufferError::OrdinaryFailClosed { relid: rel.oid, @@ -256,12 +235,63 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { reason: FailClosedReason::Malformed(e.to_string()), } })?; + if images + .iter() + .any(|h| h.new.as_ref().is_some_and(|t| t.partial)) + { + return Err(PlanError::PartialUpdate { + lsn: input.input.source_lsn, + }); + } + Ok(images) + } + + fn push_images( + &mut self, + input: &DescribedInput, + h: &HeapImages>, + patches: &[Vec<(usize, ColumnValue)>; 2], + table: u32, + ) -> Result<(), PlanError> { + let change = ViewChange { + op: h.op, + xid: input.input.xid, + source_lsn: input.input.source_lsn, + new: h.new.as_ref(), + old: h.old.as_ref(), + new_patch: &patches[0], + old_patch: &patches[1], + }; + Ok(self.writer.push_views(change, table)?) + } + + /// Encode an input without external values; false leaves it to + /// [`Self::plan_views`] + fn plan_inline(&mut self, input: &DescribedInput, table: u32) -> Result { + let images = Self::views(input)?; + if images + .iter() + .any(|h| !external_sites([h.new.as_ref(), h.old.as_ref()]).is_empty()) + { + return Ok(false); + } for h in &images { - if h.new.as_ref().is_some_and(|t| t.partial) { - return Err(PlanError::PartialUpdate { - lsn: input.input.source_lsn, - }); - } + self.push_images(input, h, &Default::default(), table)?; + } + Ok(true) + } + + /// Encode an input from tuple images in place, external values resolved + /// into per-image patches + async fn plan_views( + &mut self, + input: &DescribedInput, + table: u32, + spool: Option<&BodySpoolFile>, + ref_maps: &[&ChunkRefMap], + ) -> Result<(), PlanError> { + let rel = &input.descriptor; + for h in &Self::views(input)? { let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); if !sites.is_empty() { @@ -274,16 +304,7 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { patches[site.image].push((site.idx, v)); } } - let change = ViewChange { - op: h.op, - xid: input.input.xid, - source_lsn: input.input.source_lsn, - new: h.new.as_ref(), - old: h.old.as_ref(), - new_patch: &patches[0], - old_patch: &patches[1], - }; - self.writer.push_views(change, table)?; + self.push_images(input, h, &patches, table)?; } Ok(()) } @@ -587,9 +608,9 @@ mod tests { order.extend(lsns(c, 0).iter().map(|lsn| format!("h{lsn}r"))); } } - PlanItem::Truncate { heap: h, .. } => order.push(format!( + PlanItem::Truncate { truncate: h, .. } => order.push(format!( "h{}{}", - h.decoded.source_lsn, + h.source_lsn, if rd.tables[h.table as usize].route.is_some() { "r" } else { diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index a2659840..37e929ca 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -27,7 +27,6 @@ use crate::backfill::visibility_pending::{self, SharedPendingLedger}; use crate::catalog::pending::PendingCatalog; #[cfg(feature = "test-support")] use crate::crash_test; -use crate::decode::heap_decoder::DecodedHeap; use crate::decode::visibility::{PgXactView, SealedPatch, read_pg_xact}; use crate::emit::ch_ddl::{DdlApplicator, Stamp}; use crate::emit::ch_emitter::EmitterStats; @@ -827,8 +826,7 @@ impl ReorderSink { &mut self, _: Fenced, rel: &RelDescriptor, - heap: &DecodedHeap, - commit_ts: i64, + at: Stamp, ) -> Result<(), SinkError> { let Some(scope) = self.scopes.get_mut(&rel.rfn.db_node) else { return Ok(()); @@ -839,11 +837,6 @@ impl ReorderSink { // Attached at truncate fan-out (record time = pre-capture Present), // so the rotation's drain-time Retired answer no longer needs a // predecessor walk here - let at = Stamp { - lsn: heap.source_lsn, - xid: heap.xid, - commit_ts, - }; applicator .truncate(rel, at) .await @@ -961,7 +954,7 @@ impl ReorderSink { }) .await?; } - PlanItem::Truncate { heap: h, upto } => { + PlanItem::Truncate { truncate: t, upto } => { Box::pin(async { self.put_plan_rows(&rd.row_batches, &mut rows_cursor, upto) .await?; @@ -970,9 +963,13 @@ impl ReorderSink { .await?; } let fenced = self.barrier_fence().await?; - let rel = &rd.tables[h.table as usize].descriptor; - self.apply_truncate(fenced, rel, &h.decoded, commit_ts) - .await + let rel = &rd.tables[t.table as usize].descriptor; + let at = Stamp { + lsn: t.source_lsn, + xid: t.xid, + commit_ts, + }; + self.apply_truncate(fenced, rel, at).await }) .await?; } diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index f8161585..d976ff76 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -383,8 +383,6 @@ snapshot! { "Cumulative raw-stash payload bytes by first landing.", counter raw_decode_rows_by_op: [u64; 7] ["op" = HEAP_OP_LABELS] = "Rows fanned out of decoded raw records.", - gauge raw_pending_rows: u64 = "Raw-decoded heaps queued for pending-first yield.", - gauge raw_pending_bytes: u64 = "Bytes held by the pending raw fanout.", counter backfill_backup_rows_total: u64 = "Tuples decoded by table backfill backup walks, including TOAST chunks and rows awaiting visibility checks. Cumulative across passes.", counter backfill_backup_bytes_total: u64 = "Backup body bytes handed to table backfill page walks, excluding skipped files. Cumulative across passes.", counter backfill_copy_rows_total: u64 = "Rows decoded from source COPY for table backfills.", @@ -1338,8 +1336,6 @@ mod tests { raw_stash_bytes_by_storage: [11, 12], raw_decode_rows_by_op: [1, 2, 3, 4, 5, 6, 7], decoder_records_by_op: [41, 42, 43, 44, 45], - raw_pending_rows: 13, - raw_pending_bytes: 14, by_database: vec![DbSeries { database: "app".into(), descriptor_ambiguous_total: 21, @@ -1371,8 +1367,6 @@ mod tests { ); assert!(body.contains("walshadow_raw_decode_rows_total{op=\"multi_insert\"} 6")); assert!(body.contains("walshadow_decoder_records_total{op=\"hot_update\"} 43")); - assert!(body.contains("walshadow_raw_pending_rows 13")); - assert!(body.contains("walshadow_raw_pending_bytes 14")); // Published intervals and lookups landing in one stay distinct // families assert!(body.contains("walshadow_descriptor_ambiguous_total{database=\"app\"} 21")); diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 495c45e2..e814c2b9 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -8,7 +8,6 @@ use clickhouse_c::{Allocator, Block, BlockOpts, BlockReader, Column, Kind, Slice use crate::ch::types; use crate::decode::heap_decoder::ColumnValue; use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES, request_frame}; -use crate::schema::RelAttr; use tokio_postgres::types::Oid; /// Cell tags, matching `WS_CELL_*` in `pgext/walshadow.h` @@ -564,7 +563,8 @@ fn decode_block( } /// Render PostGIS 2-D points as WKT, typoutput yields HEXEWKB -pub fn render_ext_columns(attrs: &[RelAttr], columns: &mut [Option]) { +#[cfg(test)] +pub fn render_ext_columns(attrs: &[crate::schema::RelAttr], columns: &mut [Option]) { // Scan cells first: attribute type names sit behind a pointer per column for (idx, cell) in columns.iter_mut().enumerate() { let Some(ColumnValue::PgPending { .. }) = cell.as_ref() else { @@ -581,7 +581,7 @@ pub fn render_ext_columns(attrs: &[RelAttr], columns: &mut [Option] } } -/// Render one PostGIS cell, see [`render_ext_columns`] +/// Render one PostGIS cell as WKT, typoutput yields HEXEWKB pub(crate) fn render_ext_value(cell: &mut ColumnValue) { if let ColumnValue::PgPending { raw, .. } = cell && let Some(text) = gserialized_point_to_wkt(raw) @@ -633,6 +633,7 @@ impl OracleStats { mod tests { use super::*; use crate::ops::bridge::FRAME_PREFIX_BYTES; + use crate::schema::RelAttr; use clickhouse_c::TypeAst; fn col<'a>( diff --git a/src/xact/spill.rs b/src/xact/spill.rs index 8c187e69..8d41a01e 100644 --- a/src/xact/spill.rs +++ b/src/xact/spill.rs @@ -15,7 +15,7 @@ //! ```text //! [2 bytes "WS" magic] [u16 LE version] then repeating: //! [u8 tag] [u32 len LE] [body of `len` bytes] -//! TAG_HEAP (body = u32 dict id + encoded DecodedHeap, rfn from dict) +//! TAG_TRUNCATE (body = u32 dict id + u32 xid + u64 source_lsn) //! TAG_CHUNK (body = encoded ToastChunk) //! TAG_TOAST_DELETE (body = encoded ToastDelete) //! TAG_RAW (body = encoded RawRecord) @@ -24,8 +24,8 @@ //! ``` //! //! `TAG_DESCRIPTOR` never surfaces as an entry: each distinct `(rfn, valid_from)` -//! descriptor writes once before its first referencing heap, so spilled -//! [`DescribedHeap`]s rehydrate the exact attached descriptor without a +//! descriptor writes once before its first referencing entry, so spilled +//! inputs and truncates rehydrate the exact attached descriptor without a //! live log lookup (file stays self-contained; capture landing mid-spill //! can't reinterpret rows) //! @@ -60,9 +60,6 @@ use tokio::task::JoinHandle; use walrus::pg::walparser::RelFileNode; use crate::catalog::desc_log::{decode_descriptor_bytes, encode_descriptor_bytes}; -use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, ToastPointer, -}; use crate::schema::RelDescriptor; use foldhash::{HashMap, HashMapExt}; @@ -70,7 +67,7 @@ use foldhash::{HashMap, HashMapExt}; /// WAL-aligned ordering keeps the drain a single linear read #[derive(Debug, Clone, PartialEq)] pub enum SpillEntry { - Heap(Box), + Truncate(Box), Chunk(Box), ToastDelete(Box), /// Undecoded record on a filenode invisible at record time (same-xact @@ -82,6 +79,15 @@ pub enum SpillEntry { Input(Box), } +/// TRUNCATE of one relation under its descriptor at the record +#[derive(Debug, Clone, PartialEq)] +pub struct Truncate { + pub xid: u32, + pub source_lsn: u64, + pub descriptor: Arc, + pub descriptor_valid_from: u64, +} + /// Heap record bytes decode reads: `main_data` and block 0 registered data. /// Block images and other blocks drop; `wal_level=logical` keeps tuple /// data registered (`REGBUF_KEEP_DATA`) @@ -174,8 +180,7 @@ impl HeapInput { } } -/// [`HeapInput`] pinned to its record-time descriptor version, as -/// [`DescribedHeap`] pins decoded cells +/// [`HeapInput`] pinned to its record-time descriptor version #[derive(Debug, Clone, PartialEq)] pub struct DescribedInput { pub input: HeapInput, @@ -209,6 +214,68 @@ impl DescribedInput { const _: () = assert!(std::mem::size_of::() <= 16); +#[cfg(any(test, feature = "test-support"))] +impl From for DescribedInput { + /// WAL record bytes of an owned change, the inverse of [`Self::decode`]. + /// Partial new images carry an empty suffix-from-old + fn from(h: crate::decode::heap_decoder::DescribedHeap) -> Self { + use crate::decode::heap_decoder::*; + let d = &h.decoded; + let image = |t: &Option| t.as_ref().map(|t| form_block_data(&t.columns)); + let (old, new) = (image(&d.old), image(&d.new)); + let (info, mut main, flag_at, old_flag) = match d.op { + HeapOp::Insert => (XLOG_HEAP_INSERT, vec![0; SIZE_OF_HEAP_INSERT], None, 0), + HeapOp::Update | HeapOp::HotUpdate => ( + if d.op == HeapOp::Update { + XLOG_HEAP_UPDATE + } else { + XLOG_HEAP_HOT_UPDATE + }, + vec![0; SIZE_OF_HEAP_UPDATE], + Some(7), + XLH_UPDATE_CONTAINS_OLD_TUPLE, + ), + HeapOp::Delete => ( + XLOG_HEAP_DELETE, + vec![0; SIZE_OF_HEAP_DELETE], + Some(7), + XLH_DELETE_CONTAINS_OLD_TUPLE, + ), + HeapOp::Truncate => unreachable!("truncates buffer as Truncate"), + }; + let partial = d.new.as_ref().is_some_and(|t| t.partial); + if let (Some(at), Some(old)) = (flag_at, &old) { + main[at] |= old_flag; + main.extend_from_slice(old); + } + if partial { + main[7] |= XLH_UPDATE_SUFFIX_FROM_OLD; + } + let block = new.map(|t| { + let mut b = if partial { vec![0, 0] } else { Vec::new() }; + b.extend_from_slice(&t); + b + }); + let main_len = main.len() as u32; + let has_block = block.is_some(); + main.extend(block.unwrap_or_default()); + Self { + input: HeapInput { + xid: d.xid, + rm: walrus::pg::walparser::RmId::Heap as u8, + info, + source_lsn: d.source_lsn, + tuples: 1, + bytes: main.into_boxed_slice(), + main_len, + has_block, + }, + descriptor: h.descriptor, + descriptor_valid_from: h.descriptor_valid_from, + } + } +} + /// Raw decode inputs for one WAL record, enough to re-run /// [`crate::decode::heap_decoder::decode_heap_record`] after commit-time /// resolution. Block images survive so an insert whose tuple lives only @@ -418,68 +485,23 @@ pub const SPILL_MAGIC: [u8; 2] = *b"WS"; /// id). v6 added writer xid to heap + raw bodies. v7 added database OID to /// chunk and `ToastDelete` bodies. v8 dropped heap rfn, restored from the /// dict descriptor. v9 stored inet addr as fixed 16 bytes. v10 added -/// `TAG_INPUT` undecoded ordinary heap records. `DrainEntry` +/// `TAG_INPUT` undecoded ordinary heap records. v11 replaced owned heap +/// bodies with `TAG_TRUNCATE`. `DrainEntry` /// events (Catalog, ToastBarrier, Config/Signal) are drain-time, never /// spilled, so they don't touch this format -pub const SPILL_VERSION: u16 = 10; +pub const SPILL_VERSION: u16 = 11; /// Wire tags, one block per tag space so encode and decode read off the same /// table. Renumbering is a body-encoding change: bump [`SPILL_VERSION`] /// /// Entry frames: [`SpillEntry`] variants plus the descriptor dictionary -const TAG_HEAP: u8 = 0; +const TAG_TRUNCATE: u8 = 0; const TAG_CHUNK: u8 = 1; const TAG_TOAST_DELETE: u8 = 2; const TAG_RAW: u8 = 3; const TAG_DESCRIPTOR: u8 = 4; const TAG_INPUT: u8 = 5; -/// [`HeapOp`], one byte in every heap body -const OP_INSERT: u8 = 0; -const OP_UPDATE: u8 = 1; -const OP_HOT_UPDATE: u8 = 2; -const OP_DELETE: u8 = 3; -const OP_TRUNCATE: u8 = 4; - -/// [`ColumnValue`], leading byte of every encoded column. Shared with the -/// backfill spool, which reuses [`encode_value`] / [`decode_value`] -const VAL_NULL: u8 = 0; -const VAL_BOOL: u8 = 1; -const VAL_CHAR: u8 = 2; -const VAL_INT2: u8 = 3; -const VAL_INT4: u8 = 4; -const VAL_INT8: u8 = 5; -const VAL_FLOAT4: u8 = 6; -const VAL_FLOAT8: u8 = 7; -const VAL_OID: u8 = 8; -const VAL_DATE: u8 = 9; -const VAL_TIME: u8 = 10; -const VAL_TIMESTAMP: u8 = 11; -const VAL_TIMESTAMPTZ: u8 = 12; -const VAL_TIMETZ: u8 = 13; -const VAL_UUID: u8 = 14; -const VAL_NAME: u8 = 15; -const VAL_BYTEA: u8 = 16; -const VAL_TEXT: u8 = 17; -const VAL_EXTERNAL_TOAST: u8 = 18; -const VAL_UNSUPPORTED: u8 = 19; -const VAL_NUMERIC: u8 = 20; -const VAL_INET: u8 = 21; -const VAL_INTERVAL: u8 = 22; -const VAL_JSON: u8 = 23; -const VAL_PG_PENDING: u8 = 24; -const VAL_PG_PENDING_TEXT: u8 = 25; -const VAL_OID8: u8 = 26; -const VAL_MACADDR: u8 = 27; -const VAL_MACADDR8: u8 = 28; -const VAL_TID: u8 = 29; - -/// [`crate::decode::codecs::NumericKind`], inside a [`VAL_NUMERIC`] body -const NUM_FINITE: u8 = 0; -const NUM_NAN: u8 = 1; -const NUM_PINF: u8 = 2; -const NUM_NINF: u8 = 3; - pub struct SpillStore { dir: PathBuf, } @@ -701,11 +723,12 @@ impl SpillWriter { fn encode(&mut self, entry: &SpillEntry) { match entry { - SpillEntry::Heap(h) => { - let id = self.dict_id(&h.descriptor, h.descriptor_valid_from); - frame_into(&mut self.out.buf, TAG_HEAP, |out| { + SpillEntry::Truncate(t) => { + let id = self.dict_id(&t.descriptor, t.descriptor_valid_from); + frame_into(&mut self.out.buf, TAG_TRUNCATE, |out| { push_u32(out, id); - encode_heap_into(out, &h.decoded); + push_u32(out, t.xid); + push_u64(out, t.source_lsn); }); } SpillEntry::Input(i) => { @@ -722,7 +745,7 @@ impl SpillWriter { fn encode_payload(&mut self, entry: &SpillEntry) { let body = &mut self.out.buf; match entry { - SpillEntry::Heap(_) | SpillEntry::Input(_) => unreachable!("dictionary entries"), + SpillEntry::Truncate(_) | SpillEntry::Input(_) => unreachable!("dictionary entries"), SpillEntry::Chunk(c) => frame_into(body, TAG_CHUNK, |out| encode_chunk_into(out, c)), SpillEntry::ToastDelete(d) => frame_into(body, TAG_TOAST_DELETE, |out| { encode_toast_delete_into(out, d) @@ -898,16 +921,17 @@ impl SpillReader { .ok_or_else(|| SpillError::Format { offset: cur.pos, detail: format!( - "heap references dict id {dict_id}, only {} defined", + "entry references dict id {dict_id}, only {} defined", self.dict.len(), ), }) }; let entry = match tag { - TAG_HEAP => { + TAG_TRUNCATE => { let (descriptor, valid_from) = described(&mut cur)?; - SpillEntry::Heap(Box::new(DescribedHeap { - decoded: decode_heap(&mut cur, descriptor.rfn)?, + SpillEntry::Truncate(Box::new(Truncate { + xid: cur.u32()?, + source_lsn: cur.u64()?, descriptor, descriptor_valid_from: valid_from, })) @@ -1071,197 +1095,10 @@ pub(crate) fn push_i32(out: &mut Vec, v: i32) { pub(crate) fn push_u64(out: &mut Vec, v: u64) { out.extend_from_slice(&v.to_le_bytes()); } -fn push_i64(out: &mut Vec, v: i64) { - out.extend_from_slice(&v.to_le_bytes()); -} pub(crate) fn push_bytes(out: &mut Vec, b: &[u8]) { push_u32(out, b.len() as u32); out.extend_from_slice(b); } -fn push_str(out: &mut Vec, s: &str) { - push_bytes(out, s.as_bytes()); -} - -/// Shared with the plan spool: heap body without rfn, which readers take -/// from the heap's descriptor -pub(crate) fn encode_heap_into(out: &mut Vec, h: &DecodedHeap) { - push_u32(out, h.xid); - push_u64(out, h.source_lsn); - let op_byte: u8 = match h.op { - HeapOp::Insert => OP_INSERT, - HeapOp::Update => OP_UPDATE, - HeapOp::HotUpdate => OP_HOT_UPDATE, - HeapOp::Delete => OP_DELETE, - HeapOp::Truncate => OP_TRUNCATE, - }; - push_u8(out, op_byte); - encode_opt_tuple(out, h.new.as_ref()); - encode_opt_tuple(out, h.old.as_ref()); -} - -fn encode_opt_tuple(out: &mut Vec, t: Option<&DecodedTuple>) { - match t { - None => push_u8(out, 0), - Some(t) => { - push_u8(out, 1); - push_u8(out, t.partial as u8); - push_u32(out, t.columns.len() as u32); - for col in &t.columns { - match col { - None => push_u8(out, 0), - Some(v) => { - push_u8(out, 1); - encode_value(out, v); - } - } - } - } - } -} - -pub(crate) fn encode_value(out: &mut Vec, v: &ColumnValue) { - match v { - ColumnValue::Null => push_u8(out, VAL_NULL), - ColumnValue::Bool(b) => { - push_u8(out, VAL_BOOL); - push_u8(out, *b as u8); - } - ColumnValue::Char(i) => { - push_u8(out, VAL_CHAR); - push_u8(out, *i as u8); - } - ColumnValue::Int2(i) => { - push_u8(out, VAL_INT2); - push_u16(out, *i as u16); - } - ColumnValue::Int4(i) => { - push_u8(out, VAL_INT4); - push_i32(out, *i); - } - ColumnValue::Int8(i) => { - push_u8(out, VAL_INT8); - push_i64(out, *i); - } - ColumnValue::Float4(f) => { - push_u8(out, VAL_FLOAT4); - out.extend_from_slice(&f.to_le_bytes()); - } - ColumnValue::Float8(f) => { - push_u8(out, VAL_FLOAT8); - out.extend_from_slice(&f.to_le_bytes()); - } - ColumnValue::Oid(i) => { - push_u8(out, VAL_OID); - push_u32(out, *i); - } - ColumnValue::Oid8(i) => { - push_u8(out, VAL_OID8); - push_u64(out, *i); - } - ColumnValue::MacAddr(b) => { - push_u8(out, VAL_MACADDR); - out.extend_from_slice(b); - } - ColumnValue::MacAddr8(b) => { - push_u8(out, VAL_MACADDR8); - out.extend_from_slice(b); - } - ColumnValue::Tid { block, offset } => { - push_u8(out, VAL_TID); - push_u32(out, *block); - push_u16(out, *offset); - } - ColumnValue::Date(i) => { - push_u8(out, VAL_DATE); - push_i32(out, *i); - } - ColumnValue::Time(i) => { - push_u8(out, VAL_TIME); - push_i64(out, *i); - } - ColumnValue::Timestamp(i) => { - push_u8(out, VAL_TIMESTAMP); - push_i64(out, *i); - } - ColumnValue::TimestampTz(i) => { - push_u8(out, VAL_TIMESTAMPTZ); - push_i64(out, *i); - } - ColumnValue::TimeTz { micros, tz_seconds } => { - push_u8(out, VAL_TIMETZ); - push_i64(out, *micros); - push_i32(out, *tz_seconds); - } - ColumnValue::Uuid(b) => { - push_u8(out, VAL_UUID); - out.extend_from_slice(b); - } - ColumnValue::Name(s) => { - push_u8(out, VAL_NAME); - push_str(out, s); - } - ColumnValue::Bytea(b) => { - push_u8(out, VAL_BYTEA); - push_bytes(out, b); - } - ColumnValue::Text(s) => { - push_u8(out, VAL_TEXT); - push_str(out, s); - } - ColumnValue::ExternalToast(p) => { - push_u8(out, VAL_EXTERNAL_TOAST); - push_i32(out, p.va_rawsize); - push_u32(out, p.va_extinfo); - push_u32(out, p.va_valueid); - push_u32(out, p.va_toastrelid); - } - ColumnValue::Unsupported { type_oid, raw } => { - push_u8(out, VAL_UNSUPPORTED); - push_u32(out, *type_oid); - push_bytes(out, raw); - } - ColumnValue::Numeric(n) => { - use crate::decode::codecs::NumericKind; - push_u8(out, VAL_NUMERIC); - match n { - NumericKind::Finite(s) => { - push_u8(out, NUM_FINITE); - push_str(out, s); - } - NumericKind::NaN => push_u8(out, NUM_NAN), - NumericKind::PInf => push_u8(out, NUM_PINF), - NumericKind::NInf => push_u8(out, NUM_NINF), - } - } - ColumnValue::Inet(v) => { - push_u8(out, VAL_INET); - push_u8(out, v.family); - push_u8(out, v.bits); - push_u8(out, v.is_cidr as u8); - out.extend_from_slice(&v.addr); - } - ColumnValue::Interval(v) => { - push_u8(out, VAL_INTERVAL); - push_i32(out, v.months); - push_i32(out, v.days); - push_i64(out, v.micros); - } - ColumnValue::Json(s) => { - push_u8(out, VAL_JSON); - push_str(out, s); - } - ColumnValue::PgPending { type_oid, raw } => { - push_u8(out, VAL_PG_PENDING); - push_u32(out, *type_oid); - push_bytes(out, raw); - } - ColumnValue::PgPendingText { type_oid, text } => { - push_u8(out, VAL_PG_PENDING_TEXT); - push_u32(out, *type_oid); - push_str(out, text); - } - } -} fn encode_chunk_into(out: &mut Vec, c: &ToastChunk) { out.reserve(44 + c.chunk_data.len()); @@ -1364,183 +1201,10 @@ impl<'a> Cursor<'a> { pub(crate) fn u64(&mut self) -> Result { self.take().map(u64::from_le_bytes) } - fn i64(&mut self) -> Result { - self.take().map(i64::from_le_bytes) - } - fn f32(&mut self) -> Result { - self.take().map(f32::from_le_bytes) - } - fn f64(&mut self) -> Result { - self.take().map(f64::from_le_bytes) - } pub(crate) fn bytes(&mut self) -> Result> { let n = self.u32()? as usize; Ok(self.need(n)?.to_vec()) } - fn string(&mut self) -> Result { - let n = self.u32()? as usize; - let bs = self.need(n)?; - simdutf8::compat::from_utf8(bs) - .map(str::to_owned) - .map_err(|e| SpillError::Format { - offset: self.pos, - detail: format!("utf8: {e}"), - }) - } -} - -pub(crate) fn decode_heap(c: &mut Cursor, rfn: RelFileNode) -> Result { - let xid = c.u32()?; - let source_lsn = c.u64()?; - let op = match c.u8()? { - OP_INSERT => HeapOp::Insert, - OP_UPDATE => HeapOp::Update, - OP_HOT_UPDATE => HeapOp::HotUpdate, - OP_DELETE => HeapOp::Delete, - OP_TRUNCATE => HeapOp::Truncate, - other => { - return Err(SpillError::Format { - offset: c.pos, - detail: format!("unknown HeapOp tag {other}"), - }); - } - }; - let new = decode_opt_tuple(c)?; - let old = decode_opt_tuple(c)?; - Ok(DecodedHeap { - rfn, - xid, - source_lsn, - op, - new, - old, - }) -} - -fn decode_opt_tuple(c: &mut Cursor) -> Result> { - if c.u8()? == 0 { - return Ok(None); - } - let partial = c.u8()? != 0; - let n = c.u32()? as usize; - let mut columns = Vec::with_capacity(n); - for _ in 0..n { - if c.u8()? == 0 { - columns.push(None); - } else { - columns.push(Some(decode_value(c)?)); - } - } - Ok(Some(DecodedTuple { columns, partial })) -} - -pub(crate) fn decode_value(c: &mut Cursor) -> Result { - let tag = c.u8()?; - let v = match tag { - VAL_NULL => ColumnValue::Null, - VAL_BOOL => ColumnValue::Bool(c.u8()? != 0), - VAL_CHAR => ColumnValue::Char(c.u8()? as i8), - VAL_INT2 => ColumnValue::Int2(c.u16()? as i16), - VAL_INT4 => ColumnValue::Int4(c.i32()?), - VAL_INT8 => ColumnValue::Int8(c.i64()?), - VAL_FLOAT4 => ColumnValue::Float4(c.f32()?), - VAL_FLOAT8 => ColumnValue::Float8(c.f64()?), - VAL_OID => ColumnValue::Oid(c.u32()?), - VAL_OID8 => ColumnValue::Oid8(c.u64()?), - VAL_MACADDR => ColumnValue::MacAddr(c.need(6)?.try_into().unwrap()), - VAL_MACADDR8 => ColumnValue::MacAddr8(c.need(8)?.try_into().unwrap()), - VAL_TID => { - let block = c.u32()?; - let offset = c.u16()?; - ColumnValue::Tid { block, offset } - } - VAL_DATE => ColumnValue::Date(c.i32()?), - VAL_TIME => ColumnValue::Time(c.i64()?), - VAL_TIMESTAMP => ColumnValue::Timestamp(c.i64()?), - VAL_TIMESTAMPTZ => ColumnValue::TimestampTz(c.i64()?), - VAL_TIMETZ => { - let micros = c.i64()?; - let tz_seconds = c.i32()?; - ColumnValue::TimeTz { micros, tz_seconds } - } - VAL_UUID => ColumnValue::Uuid(c.need(16)?.try_into().unwrap()), - VAL_NAME => ColumnValue::Name(c.string()?), - VAL_BYTEA => ColumnValue::Bytea(c.bytes()?), - VAL_TEXT => ColumnValue::Text(c.string()?), - VAL_EXTERNAL_TOAST => { - let va_rawsize = c.i32()?; - let va_extinfo = c.u32()?; - let va_valueid = c.u32()?; - let va_toastrelid = c.u32()?; - ColumnValue::ExternalToast(ToastPointer { - va_rawsize, - va_extinfo, - va_valueid, - va_toastrelid, - }) - } - VAL_UNSUPPORTED => { - let type_oid = c.u32()?; - let raw = c.bytes()?; - ColumnValue::Unsupported { type_oid, raw } - } - VAL_NUMERIC => { - use crate::decode::codecs::NumericKind; - let kind = c.u8()?; - ColumnValue::Numeric(match kind { - NUM_FINITE => NumericKind::Finite(c.string()?), - NUM_NAN => NumericKind::NaN, - NUM_PINF => NumericKind::PInf, - NUM_NINF => NumericKind::NInf, - other => { - return Err(SpillError::Format { - offset: c.pos, - detail: format!("unknown NumericKind tag {other}"), - }); - } - }) - } - VAL_INET => { - let family = c.u8()?; - let bits = c.u8()?; - let is_cidr = c.u8()? != 0; - let addr = c.take()?; - ColumnValue::Inet(crate::decode::codecs::InetValue { - family, - bits, - is_cidr, - addr, - }) - } - VAL_INTERVAL => { - let months = c.i32()?; - let days = c.i32()?; - let micros = c.i64()?; - ColumnValue::Interval(crate::decode::codecs::IntervalValue { - months, - days, - micros, - }) - } - VAL_JSON => ColumnValue::Json(c.string()?), - VAL_PG_PENDING => { - let type_oid = c.u32()?; - let raw = c.bytes()?; - ColumnValue::PgPending { type_oid, raw } - } - VAL_PG_PENDING_TEXT => { - let type_oid = c.u32()?; - let text = c.string()?; - ColumnValue::PgPendingText { type_oid, text } - } - other => { - return Err(SpillError::Format { - offset: c.pos, - detail: format!("unknown ColumnValue tag {other}"), - }); - } - }; - Ok(v) } fn decode_chunk(c: &mut Cursor) -> Result { @@ -1677,30 +1341,17 @@ mod tests { }) } - fn sample_heap(xid: u32, lsn: u64) -> DescribedHeap { + /// Insert of one bytea cell, `body` bytes long + fn sample_input(xid: u32, lsn: u64, body: usize) -> DescribedInput { + use crate::decode::heap_decoder::*; DescribedHeap { decoded: DecodedHeap { - rfn: RelFileNode { - spc_node: 1663, - db_node: 5, - rel_node: 16385, - }, + rfn: RelFileNode::default(), xid, source_lsn: lsn, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![ - Some(ColumnValue::Int4(7)), - Some(ColumnValue::Text("hello".into())), - None, - Some(ColumnValue::Null), - Some(ColumnValue::ExternalToast(ToastPointer { - va_rawsize: 1024, - va_extinfo: 0x80000200, - va_valueid: 99, - va_toastrelid: 16400, - })), - ], + columns: vec![Some(ColumnValue::Bytea(vec![7; body]))], partial: false, }), old: None, @@ -1708,6 +1359,7 @@ mod tests { descriptor: sample_descriptor(16385), descriptor_valid_from: 0x1000, } + .into() } fn sample_chunk(value_id: u32, seq: u32, lsn: u64, body: &[u8]) -> ToastChunk { @@ -1730,17 +1382,10 @@ mod tests { let mut writer = store.writer(42, 0x1000).await.unwrap(); let mut expected = Vec::new(); for i in 0..600 { - let mut heap = sample_heap(42, 0x2000 + i); - heap.decoded.new.as_mut().unwrap().columns = vec![Some(ColumnValue::Bytea(vec![ - i as u8; - if i == 300 { - IO_CHUNK + 17 - } else { - 1021 - } - ]))]; - heap.descriptor_valid_from += i % 2; - expected.push(SpillEntry::Heap(Box::new(heap))); + let body = if i == 300 { IO_CHUNK + 17 } else { 1021 }; + let mut input = sample_input(42, 0x2000 + i, body); + input.descriptor_valid_from += i % 2; + expected.push(SpillEntry::Input(Box::new(input))); } writer.write_batch(Vec::new()).await.unwrap(); for batch in expected.chunks(137) { @@ -1789,13 +1434,18 @@ mod tests { } #[tokio::test(flavor = "current_thread")] - async fn round_trip_heap_and_chunk() { + async fn round_trip_truncate_and_chunk() { let tmp = tempdir().unwrap(); let store = SpillStore::new(tmp.path().to_path_buf()).unwrap(); let mut w = store.writer(42, 0x1000).await.unwrap(); - let h = sample_heap(42, 0x2000); + let h = Truncate { + xid: 42, + source_lsn: 0x2000, + descriptor: sample_descriptor(16385), + descriptor_valid_from: 0x1000, + }; let c = sample_chunk(99, 0, 0x2100, &[0xDE, 0xAD, 0xBE, 0xEF]); - w.write_batch(vec![SpillEntry::Heap(Box::new(h.clone()))]) + w.write_batch(vec![SpillEntry::Truncate(Box::new(h.clone()))]) .await .unwrap(); w.write_batch(vec![SpillEntry::Chunk(Box::new(c.clone()))]) @@ -1806,7 +1456,7 @@ mod tests { let mut r = w.finish().await.unwrap(); assert_eq!( r.next().await.unwrap().unwrap(), - SpillEntry::Heap(Box::new(h)) + SpillEntry::Truncate(Box::new(h)) ); assert_eq!( r.next().await.unwrap().unwrap(), @@ -1841,7 +1491,7 @@ mod tests { } /// Inputs keep only main data and block 0 data, share the descriptor - /// dictionary with heaps, and decode identically after spill + /// dictionary, and decode identically after spill #[tokio::test(flavor = "current_thread")] async fn round_trip_heap_input() { let tmp = tempdir().unwrap(); @@ -1855,14 +1505,15 @@ mod tests { descriptor: descriptor.clone(), descriptor_valid_from: 0x100, }; - let heap = DescribedHeap { - decoded: input.decode().unwrap().remove(0), + let truncate = Truncate { + xid: 9, + source_lsn: 0x4243, descriptor, descriptor_valid_from: 0x100, }; let entries = vec![ SpillEntry::Input(Box::new(input.clone())), - SpillEntry::Heap(Box::new(heap.clone())), + SpillEntry::Truncate(Box::new(truncate)), SpillEntry::Input(Box::new(input.clone())), ]; w.write_batch(entries.clone()).await.unwrap(); @@ -1988,7 +1639,7 @@ mod tests { let tmp = tempdir().unwrap(); let store = SpillStore::new(crate::fs::scratch_dir(tmp.path())).unwrap(); let mut w1 = store.writer(1, 0).await.unwrap(); - w1.write_batch(vec![SpillEntry::Heap(Box::new(sample_heap(1, 0)))]) + w1.write_batch(vec![SpillEntry::Input(Box::new(sample_input(1, 0, 8)))]) .await .unwrap(); // Drop without finish() so file stays on disk @@ -2005,7 +1656,7 @@ mod tests { let tmp = tempdir().unwrap(); let store = SpillStore::new(tmp.path().to_path_buf()).unwrap(); let mut w = store.writer(7, 0).await.unwrap(); - w.write_batch(vec![SpillEntry::Heap(Box::new(sample_heap(7, 0)))]) + w.write_batch(vec![SpillEntry::Input(Box::new(sample_input(7, 0, 8)))]) .await .unwrap(); let path = w.path().to_path_buf(); @@ -2147,7 +1798,7 @@ mod tests { garbled.push(0xFF); let cases = [ ( - frame(TAG_HEAP, &0u32.to_le_bytes()), + frame(TAG_TRUNCATE, &0u32.to_le_bytes()), "dict id 0, only 0 defined", ), (frame(TAG_DESCRIPTOR, &garbled), "short read"), @@ -2180,191 +1831,6 @@ mod tests { assert_eq!(r.dict.len(), 1); } - /// Exhaustive: a new [`ColumnValue`] variant fails to compile here until - /// named, and the tag cross-checks `encode_value`'s leading byte - fn expected_tag(v: &ColumnValue) -> u8 { - match v { - ColumnValue::Null => VAL_NULL, - ColumnValue::Bool(_) => VAL_BOOL, - ColumnValue::Char(_) => VAL_CHAR, - ColumnValue::Int2(_) => VAL_INT2, - ColumnValue::Int4(_) => VAL_INT4, - ColumnValue::Int8(_) => VAL_INT8, - ColumnValue::Float4(_) => VAL_FLOAT4, - ColumnValue::Float8(_) => VAL_FLOAT8, - ColumnValue::Oid(_) => VAL_OID, - ColumnValue::Oid8(_) => VAL_OID8, - ColumnValue::MacAddr(_) => VAL_MACADDR, - ColumnValue::MacAddr8(_) => VAL_MACADDR8, - ColumnValue::Tid { .. } => VAL_TID, - ColumnValue::Date(_) => VAL_DATE, - ColumnValue::Time(_) => VAL_TIME, - ColumnValue::Timestamp(_) => VAL_TIMESTAMP, - ColumnValue::TimestampTz(_) => VAL_TIMESTAMPTZ, - ColumnValue::TimeTz { .. } => VAL_TIMETZ, - ColumnValue::Uuid(_) => VAL_UUID, - ColumnValue::Name(_) => VAL_NAME, - ColumnValue::Bytea(_) => VAL_BYTEA, - ColumnValue::Text(_) => VAL_TEXT, - ColumnValue::ExternalToast(_) => VAL_EXTERNAL_TOAST, - ColumnValue::Unsupported { .. } => VAL_UNSUPPORTED, - ColumnValue::Numeric(_) => VAL_NUMERIC, - ColumnValue::Inet(_) => VAL_INET, - ColumnValue::Interval(_) => VAL_INTERVAL, - ColumnValue::Json(_) => VAL_JSON, - ColumnValue::PgPending { .. } => VAL_PG_PENDING, - ColumnValue::PgPendingText { .. } => VAL_PG_PENDING_TEXT, - } - } - - #[test] - fn encode_decode_value_round_trip_for_every_variant() { - use crate::decode::codecs::{InetValue, IntervalValue, NumericKind, PGSQL_AF_INET6}; - let cases = [ - ColumnValue::Null, - ColumnValue::Bool(true), - ColumnValue::Bool(false), - ColumnValue::Char(-5), - ColumnValue::Int2(-32000), - ColumnValue::Int4(-1_000_000), - ColumnValue::Int8(1 << 40), - ColumnValue::Float4(std::f32::consts::PI), - ColumnValue::Float8(std::f64::consts::E), - ColumnValue::Oid(0xDEADBEEF), - ColumnValue::Oid8(u64::MAX), - ColumnValue::MacAddr([1, 2, 3, 4, 5, 6]), - ColumnValue::MacAddr8([1, 2, 3, 4, 5, 6, 7, 8]), - ColumnValue::Tid { - block: u32::MAX, - offset: 7, - }, - ColumnValue::Date(-1), - ColumnValue::Time(86_400_000_000), - ColumnValue::Timestamp(-1), - ColumnValue::TimestampTz(0x1234_5678), - ColumnValue::TimeTz { - micros: 3600 * 1_000_000, - tz_seconds: -28800, - }, - ColumnValue::Uuid([7u8; 16]), - ColumnValue::Name("nspname".into()), - ColumnValue::Bytea(vec![0, 1, 2, 3, 4, 5]), - ColumnValue::Text("héllo, мир ✓".into()), - ColumnValue::ExternalToast(ToastPointer { - va_rawsize: 2 * 1024 * 1024, - va_extinfo: 0x40000300, - va_valueid: 12345, - va_toastrelid: 56789, - }), - ColumnValue::Unsupported { - type_oid: 1700, - raw: vec![0xAB; 32], - }, - ColumnValue::Numeric(NumericKind::Finite("3.14".into())), - ColumnValue::Numeric(NumericKind::NaN), - ColumnValue::Numeric(NumericKind::PInf), - ColumnValue::Numeric(NumericKind::NInf), - ColumnValue::Inet(InetValue { - family: PGSQL_AF_INET6, - bits: 128, - is_cidr: false, - addr: [0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01], - }), - ColumnValue::Interval(IntervalValue { - months: -3, - days: 14, - micros: 123_456, - }), - ColumnValue::Json("{\"a\":1}".into()), - ColumnValue::PgPending { - type_oid: 3802, - raw: vec![0xAA, 0xBB, 0xCC], - }, - ColumnValue::PgPendingText { - type_oid: 1007, - text: "{1,2}".into(), - }, - ]; - let mut seen = std::collections::BTreeSet::new(); - for v in cases { - let mut out = Vec::new(); - encode_value(&mut out, &v); - assert_eq!(out[0], expected_tag(&v), "tag mismatch for {v:?}"); - seen.insert(out[0]); - let mut cur = Cursor::new(&out); - let decoded = decode_value(&mut cur).unwrap(); - assert_eq!(decoded, v, "round-trip mismatch for {v:?}"); - assert_eq!(cur.pos, out.len(), "trailing bytes for {v:?}"); - } - // Samples span a gapless 0..next, and next decodes as unknown: together - // that is the whole tag space, so a variant added without a sample here - // trips the probe below - let next = seen.len() as u8; - assert_eq!( - seen, - (0..next).collect(), - "sampled tags are not gapless from {VAL_NULL}" - ); - let probe = decode_value(&mut Cursor::new(&[next])); - let detail = probe - .expect_err("decodable tag needs a round-trip sample here") - .to_string(); - assert!( - detail.contains("unknown ColumnValue tag"), - "tag {next} is live, so it needs a round-trip sample here: {detail}" - ); - } - - #[tokio::test(flavor = "current_thread")] - async fn round_trip_all_heap_ops() { - let tmp = tempdir().unwrap(); - let store = SpillStore::new(tmp.path().to_path_buf()).unwrap(); - assert_eq!(store.dir(), tmp.path()); - let mut w = store.writer(11, 0x500).await.unwrap(); - assert!(w.path().exists()); - for op in [ - HeapOp::Insert, - HeapOp::Update, - HeapOp::HotUpdate, - HeapOp::Delete, - HeapOp::Truncate, - ] { - let mut h = sample_heap(11, 0x600); - h.decoded.op = op; - if matches!(op, HeapOp::Truncate) { - // Truncate carries no tuple payload - h.decoded.new = None; - h.decoded.old = None; - } else { - h.decoded.old = Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(42))], - partial: false, - }); - } - w.write_batch(vec![SpillEntry::Heap(Box::new(h))]) - .await - .unwrap(); - } - let mut r = w.finish().await.unwrap(); - assert!(r.path().to_str().unwrap().contains("xid-")); - let mut ops = Vec::new(); - while let Some(e) = r.next().await.unwrap() { - let SpillEntry::Heap(b) = e else { panic!() }; - ops.push(b.decoded.op); - } - assert_eq!( - ops, - vec![ - HeapOp::Insert, - HeapOp::Update, - HeapOp::HotUpdate, - HeapOp::Delete, - HeapOp::Truncate, - ], - ); - r.unlink().await.unwrap(); - } - #[test] fn cursor_short_read_surfaces_format_error() { let mut cur = Cursor::new(&[1u8, 2]); @@ -2372,53 +1838,6 @@ mod tests { assert!(matches!(err, SpillError::Format { offset: 0, .. })); } - #[test] - fn cursor_string_rejects_invalid_utf8() { - // len 2, body 0xFF 0xFE invalid UTF-8 - let buf = [2u8, 0, 0, 0, 0xFF, 0xFE]; - let mut cur = Cursor::new(&buf); - let err = cur.string().unwrap_err(); - assert_eq!( - err.to_string(), - "spill format at offset 6: utf8: invalid utf-8 sequence of 1 bytes from index 0" - ); - } - - #[test] - fn decode_value_rejects_unknown_tag() { - let mut cur = Cursor::new(&[99u8]); - let err = decode_value(&mut cur).unwrap_err(); - assert_eq!( - err.to_string(), - "spill format at offset 1: unknown ColumnValue tag 99" - ); - } - - #[test] - fn decode_value_rejects_unknown_numeric_kind() { - // tag=20 Numeric, bad NumericKind sub-tag - let buf = [20u8, 99u8]; - let mut cur = Cursor::new(&buf); - let err = decode_value(&mut cur).unwrap_err(); - assert_eq!( - err.to_string(), - "spill format at offset 2: unknown NumericKind tag 99" - ); - } - - #[test] - fn decode_heap_rejects_unknown_op_tag() { - // u32 xid + u64 source_lsn = 12 bytes, then bad op 99 - let mut buf = vec![0u8; 12]; - buf.push(99u8); - let mut cur = Cursor::new(&buf); - let err = decode_heap(&mut cur, RelFileNode::default()).unwrap_err(); - assert_eq!( - err.to_string(), - "spill format at offset 13: unknown HeapOp tag 99" - ); - } - #[tokio::test(flavor = "current_thread")] async fn writer_unlink_tolerates_already_gone() { let tmp = tempdir().unwrap(); @@ -2435,7 +1854,7 @@ mod tests { let tmp = tempdir().unwrap(); let store = SpillStore::new(tmp.path().to_path_buf()).unwrap(); let mut w = store.writer(14, 0).await.unwrap(); - w.write_batch(vec![SpillEntry::Heap(Box::new(sample_heap(14, 0)))]) + w.write_batch(vec![SpillEntry::Input(Box::new(sample_input(14, 0, 8)))]) .await .unwrap(); let r = w.finish().await.unwrap(); diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index 9a3ffd72..c39fa247 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -1,9 +1,9 @@ //! Per-xid xact buffer + TOAST reassembly. //! -//! Holds every [`DecodedHeap`] + TOAST chunk for an xid until the -//! matching `XLOG_XACT_COMMIT` / `XLOG_XACT_ABORT` lands. Commit drains -//! in WAL order substituting each `ColumnValue::ExternalToast` with its -//! reassembled `Bytea` / `Text`; abort drops buffer + spill file. +//! Holds every heap record + TOAST chunk for an xid until the matching +//! `XLOG_XACT_COMMIT` / `XLOG_XACT_ABORT` lands. Commit drains in WAL +//! order, planning resolves external values from buffered chunks; abort +//! drops buffer + spill file. //! //! ## Why bundle TOAST chunks with heap tuples //! @@ -54,9 +54,11 @@ use crate::catalog::pending::{PendingCatalog, PendingSlot}; use crate::decode::datum::DatumRef; use crate::decode::decoder_sink::{DecoderSinkError, DecoderStats, SkipReason}; use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, DescribedHeap, HeapImages, HeapOp, ToastPointer, TupleView, - decode_heap_record, scan_heap_record, view_heap_record, + ColumnValue, HeapImages, HeapOp, ToastPointer, TupleView, decode_heap_record, scan_heap_record, + view_heap_record, }; +#[cfg(any(test, feature = "test-support"))] +use crate::decode::heap_decoder::{DecodedHeap, DescribedHeap}; #[cfg(test)] use crate::decode::wal_xact::{ XACT_XINFO_HAS_DBINFO, XACT_XINFO_HAS_GID, XACT_XINFO_HAS_INVALS, XACT_XINFO_HAS_SUBXACTS, @@ -74,7 +76,7 @@ use crate::toast::{ }; use crate::xact::spill::{ BodySpoolFile, BodySpoolWriter, DescribedInput, HeapInput, RawRecord, SpillEntry, SpillError, - SpillReader, SpillStore, SpillWriter, ToastChunk, ToastDelete, + SpillReader, SpillStore, SpillWriter, ToastChunk, ToastDelete, Truncate, }; use foldhash::{HashMap, HashMapExt}; #[cfg(any(test, feature = "test-support"))] @@ -151,7 +153,7 @@ impl SubxactTracker { /// `source_lsn` is the WAL LSN stamped at decode; merge-drain orders by it fn entry_lsn(e: &SpillEntry) -> u64 { match e { - SpillEntry::Heap(h) => h.decoded.source_lsn, + SpillEntry::Truncate(t) => t.source_lsn, SpillEntry::Chunk(c) => c.source_lsn, SpillEntry::ToastDelete(d) => d.source_lsn, SpillEntry::Raw(r) => r.source_lsn, @@ -159,61 +161,30 @@ fn entry_lsn(e: &SpillEntry) -> u64 { } } -/// One drained row change: cells for truncates, stash-resolved records and -/// directly buffered heaps; record bytes for covered inputs, which planning -/// encodes without building cells +/// One drained change: a truncate, or record bytes planning encodes in place #[derive(Debug, Clone, PartialEq)] pub enum DrainHeap { - Cells(DescribedHeap), + Truncate(Box), Input(Box), } impl DrainHeap { - pub fn descriptor(&self) -> &Arc { - match self { - Self::Cells(h) => &h.descriptor, - Self::Input(i) => &i.descriptor, - } - } - - pub fn descriptor_valid_from(&self) -> u64 { - match self { - Self::Cells(h) => h.descriptor_valid_from, - Self::Input(i) => i.descriptor_valid_from, - } - } - - pub fn source_lsn(&self) -> u64 { - match self { - Self::Cells(h) => h.decoded.source_lsn, - Self::Input(i) => i.input.source_lsn, - } - } - - pub fn is_truncate(&self) -> bool { - matches!(self, Self::Cells(h) if h.decoded.op == HeapOp::Truncate) - } - - pub fn approx_bytes(&self) -> usize { - match self { - Self::Cells(h) => h.approx_bytes(), - Self::Input(i) => i.approx_bytes(), - } - } - - /// Row changes this entry yields - pub fn rows(&self) -> usize { - match self { - Self::Cells(_) => 1, - Self::Input(i) => usize::from(i.input.tuples), - } - } - #[cfg(any(test, feature = "test-support"))] - /// Owned cells per tuple, through the cell adapter + /// Owned cells per tuple, through the owned decoder pub fn into_cells(self) -> std::result::Result, XactBufferError> { match self { - Self::Cells(h) => Ok(smallvec::smallvec![h]), + Self::Truncate(t) => Ok(smallvec::smallvec![DescribedHeap { + decoded: DecodedHeap { + rfn: t.descriptor.rfn, + xid: t.xid, + source_lsn: t.source_lsn, + op: HeapOp::Truncate, + new: None, + old: None, + }, + descriptor: t.descriptor, + descriptor_valid_from: t.descriptor_valid_from, + }]), Self::Input(i) => input_heaps(&i), } } @@ -581,18 +552,7 @@ impl StashMark { /// heap allocation. Good enough for the eviction threshold fn approximate_size(entry: &SpillEntry) -> usize { match entry { - SpillEntry::Heap(h) => { - // size_of:: counts the retained descriptor Arc + - // span key (contents shared, not per-row) - let mut sz = std::mem::size_of::(); - if let Some(t) = &h.decoded.new { - sz += tuple_size(t); - } - if let Some(t) = &h.decoded.old { - sz += tuple_size(t); - } - sz - } + SpillEntry::Truncate(_) => std::mem::size_of::(), SpillEntry::Chunk(c) => std::mem::size_of::() + c.chunk_data.len(), SpillEntry::ToastDelete(_) => std::mem::size_of::(), SpillEntry::Raw(r) => r.approx_bytes(), @@ -600,24 +560,6 @@ fn approximate_size(entry: &SpillEntry) -> usize { } } -fn tuple_size(t: &crate::decode::heap_decoder::DecodedTuple) -> usize { - let mut sz = std::mem::size_of::() - + t.columns.capacity() * std::mem::size_of::>(); - for v in t.columns.iter().flatten() { - sz += value_size(v); - } - sz -} - -fn value_size(v: &ColumnValue) -> usize { - match v { - ColumnValue::Bytea(b) => b.len(), - ColumnValue::Text(s) | ColumnValue::Name(s) => s.len(), - ColumnValue::Unsupported { raw, .. } => raw.len(), - _ => 0, - } -} - #[derive(Debug, Eq, Ord, PartialEq, PartialOrd)] struct PendingDurableXact { first_lsn: Pos, @@ -663,9 +605,8 @@ const MARKER_CAP: usize = 65536; pub enum StashOutcome { /// Resolved toast heap: decode stashed records at drain Toast(Arc), - /// Resolved ordinary heap: decode stashed records to rows at drain via - /// the merge's pending queue. Carries commit-resolution descriptor + - /// its interval's valid_from + /// Resolved ordinary heap: stashed records drain as inputs under the + /// commit-resolution descriptor + its interval's valid_from Ordinary { rel: Arc, valid_from: u64, @@ -702,6 +643,27 @@ impl StashResolved { pub const fn nothing_stashed(top_xid: u32) -> Self { Self { top_xid, res: None } } + + /// `Ordinary` verdict for `rel`'s filenode, unfenced, no same-xact shapes + #[cfg(any(test, feature = "test-support"))] + pub fn ordinary(top_xid: u32, rel: Arc, valid_from: u64) -> Self { + let mut outcomes = HashMap::new(); + let rfn = rel.rfn; + let outcome = StashOutcome::Ordinary { + rel, + valid_from, + fence: Vec::new(), + pending: Vec::new(), + }; + outcomes.insert(rfn, outcome); + Self { + top_xid, + res: Some(StashResolution { + outcomes, + stats: None, + }), + } + } } /// Resolve the finishing tree's stashed filenodes against the descriptor @@ -890,10 +852,6 @@ pub struct XactBuffer { drain_row_resident: Arc, /// Bytes in transaction body spool files (disk, not resident) toast_spool_bytes: Arc, - /// Raw-decoded heaps queued for pending-first yield - /// (`raw_pending_rows` / `raw_pending_bytes` gauges) - raw_pending_rows: Arc, - raw_pending_bytes: Arc, } impl XactBuffer { @@ -916,8 +874,6 @@ impl XactBuffer { drain_chunk_resident: Arc::new(AtomicU64::new(0)), drain_row_resident: Arc::new(AtomicU64::new(0)), toast_spool_bytes: Arc::new(AtomicU64::new(0)), - raw_pending_rows: Arc::new(AtomicU64::new(0)), - raw_pending_bytes: Arc::new(AtomicU64::new(0)), }) } @@ -945,15 +901,6 @@ impl XactBuffer { self.toast_spool_bytes.load(Ordering::Relaxed) } - /// Raw-decoded heaps awaiting pending-first yield; `0` outside a drain - pub fn raw_pending_rows(&self) -> u64 { - self.raw_pending_rows.load(Ordering::Relaxed) - } - - pub fn raw_pending_bytes(&self) -> u64 { - self.raw_pending_bytes.load(Ordering::Relaxed) - } - /// Monotonic high-water mark of [`Self::drain_resident_bytes`]. For a /// spilled xact this stays near merge-heads + chunk bytes, far below the /// xact's decoded size — the drain-streaming bound. @@ -996,10 +943,10 @@ impl XactBuffer { let mut chunk_count = 0u64; for e in &st.in_mem { match e { - SpillEntry::Heap(h) => { + SpillEntry::Truncate(t) => { heap_count += 1; - last_lsn = last_lsn.max(h.decoded.source_lsn); - rels.insert((h.decoded.rfn.db_node, h.decoded.rfn.rel_node)); + last_lsn = last_lsn.max(t.source_lsn); + rels.insert((t.descriptor.rfn.db_node, t.descriptor.rfn.rel_node)); } SpillEntry::Chunk(c) => { chunk_count += 1; @@ -1049,15 +996,26 @@ impl XactBuffer { } /// Envelope retains the decode-time descriptor through buffer/spill; - /// detoast and routing read it attached, never a live lookup - pub async fn on_heap( - &mut self, - described: DescribedHeap, - ) -> std::result::Result<(), XactBufferError> { - let xid = described.decoded.xid; - let first_lsn = described.decoded.source_lsn; - let entry = SpillEntry::Heap(Box::new(described)); - self.absorb(xid, first_lsn, entry).await + /// routing reads it attached, never a live lookup + pub async fn on_truncate(&mut self, t: Truncate) -> std::result::Result<(), XactBufferError> { + let (xid, first_lsn) = (t.xid, t.source_lsn); + self.absorb(xid, first_lsn, SpillEntry::Truncate(Box::new(t))) + .await + } + + #[cfg(any(test, feature = "test-support"))] + /// Buffer an owned change as its record bytes + pub async fn on_heap(&mut self, h: DescribedHeap) -> std::result::Result<(), XactBufferError> { + if h.decoded.op != HeapOp::Truncate { + return self.on_input(h.into()).await; + } + self.on_truncate(Truncate { + xid: h.decoded.xid, + source_lsn: h.decoded.source_lsn, + descriptor: h.descriptor, + descriptor_valid_from: h.descriptor_valid_from, + }) + .await } /// Ordinary heap record kept undecoded until commit @@ -1280,11 +1238,7 @@ impl XactBuffer { ) -> std::result::Result<(), XactBufferError> { let sz = approximate_size(&entry); let raw_sz = matches!(&entry, SpillEntry::Raw(_)).then_some(sz as u64); - let plain = match &entry { - SpillEntry::Heap(h) => h.decoded.op != HeapOp::Truncate, - SpillEntry::Input(_) => true, - _ => false, - }; + let plain = matches!(&entry, SpillEntry::Input(_)); let st = self.state_for(xid, first_lsn); st.plain &= plain; st.in_mem.push(entry); @@ -1386,7 +1340,6 @@ impl XactBuffer { sources, events, allowed_xids, - pending_heaps: VecDeque::new(), chunks: ChunkRefMap::new(), chunk_bytes: 0, collect_rows, @@ -1396,12 +1349,6 @@ impl XactBuffer { gauge, chunk_gauge: self.drain_gauge(&self.drain_chunk_resident), row_gauge: self.drain_gauge(&self.drain_row_resident), - pending_gauge: PendingGauge { - rows: self.raw_pending_rows.clone(), - bytes: self.raw_pending_bytes.clone(), - held_rows: 0, - held_bytes: 0, - }, spool: None, spool_dir: self.store.dir().to_path_buf(), spool_xid: top_xid, @@ -1488,20 +1435,16 @@ impl XactBuffer { { let mut heaps = Vec::with_capacity(st.in_mem.len()); for entry in std::mem::take(&mut st.in_mem) { - heaps.push(match entry { - SpillEntry::Heap(h) => DrainHeap::Cells(*h), - SpillEntry::Input(i) => { - if !xids.contains(&i.input.xid) { - return Err(XactBufferError::ForeignXid { - xid: i.input.xid, - top: top_xid, - }); - } - DrainHeap::Input(i) - } - // Plain state holds heaps and inputs only - _ => unreachable!(), - }); + let SpillEntry::Input(i) = entry else { + unreachable!("plain state holds inputs only"); + }; + if !xids.contains(&i.input.xid) { + return Err(XactBufferError::ForeignXid { + xid: i.input.xid, + top: top_xid, + }); + } + heaps.push(DrainHeap::Input(i)); } self.stats.committed_xacts_total += 1; return Ok(CommittedDrain { @@ -1671,39 +1614,6 @@ impl Drop for ResidentGauge { } } -/// Pending-queue share of the raw fanout gauges; drop releases whatever -/// an abandoned drain still held -struct PendingGauge { - rows: Arc, - bytes: Arc, - held_rows: u64, - held_bytes: u64, -} - -impl PendingGauge { - fn add(&mut self, bytes: usize) { - self.held_rows += 1; - self.held_bytes += bytes as u64; - self.rows.fetch_add(1, Ordering::Relaxed); - self.bytes.fetch_add(bytes as u64, Ordering::Relaxed); - } - - fn sub(&mut self, bytes: usize) { - let bytes = (bytes as u64).min(self.held_bytes); - self.held_rows = self.held_rows.saturating_sub(1); - self.held_bytes -= bytes; - self.rows.fetch_sub(1, Ordering::Relaxed); - self.bytes.fetch_sub(bytes, Ordering::Relaxed); - } -} - -impl Drop for PendingGauge { - fn drop(&mut self) { - self.rows.fetch_sub(self.held_rows, Ordering::Relaxed); - self.bytes.fetch_sub(self.held_bytes, Ordering::Relaxed); - } -} - /// Lazy merge source for one xid: spill-reader head (older in WAL order) /// chained with the in-mem tail, one decoded entry resident. `pop` refills /// from the reader until EOF, then drains `in_mem`. The EOF reader parks as @@ -1790,7 +1700,7 @@ impl MergeSource { } enum MergeItem { - Heap(Box), + Truncate(Box), Input(Box), /// Boxed: a config row carries every per-relation setting, so the variant /// dwarfs the heap pointer the merge yields per row @@ -1822,12 +1732,6 @@ enum MergeItem { struct MergedDrain { sources: Vec, events: Vec>, - /// Heaps decoded from one raw record (MULTI_INSERT fans one record out - /// to N tuples), yielded in tuple order before the merge advances past - /// the record's LSN. Event-first tie break is preserved: every event at - /// `lsn <= L` drained before the raw at `L` was popped. Queued bytes - /// charge `gauge`, released as each heap yields - pending_heaps: VecDeque>, /// Live (unsealed) generation chunks: ChunkRefMap, chunk_bytes: usize, @@ -1848,9 +1752,6 @@ struct MergedDrain { chunk_gauge: ResidentGauge, /// Collected mirror rows; shares split off with each taken batch row_gauge: ResidentGauge, - /// Pending-queue view for the `raw_pending_*` gauges; queued bytes - /// still charge `gauge` for resident accounting - pending_gauge: PendingGauge, /// Lazily created at threshold crossing; None while memory-resident spool: Option, spool_dir: PathBuf, @@ -1870,15 +1771,6 @@ struct MergedDrain { impl MergedDrain { async fn next(&mut self) -> std::result::Result, XactBufferError> { loop { - // Pending head first: a raw record's fanned-out tuples all yield - // before any source or event past its LSN is considered - if let Some(h) = self.pending_heaps.pop_front() { - let sz = h.approx_bytes(); - self.gauge.sub(sz); - self.pending_gauge.sub(sz); - self.check_owned(h.decoded.xid)?; - return Ok(Some(MergeItem::Heap(h))); - } enum Pick { Data(usize), Event(usize), @@ -1909,13 +1801,18 @@ impl MergedDrain { Pick::Data(i) => { let entry = self.sources[i].pop(&mut self.gauge).await?; match entry { - SpillEntry::Heap(h) => { - self.check_owned(h.decoded.xid)?; - return Ok(Some(MergeItem::Heap(h))); + SpillEntry::Truncate(t) => { + self.check_owned(t.xid)?; + return Ok(Some(MergeItem::Truncate(t))); } SpillEntry::Chunk(c) => self.fold_chunk(*c)?, SpillEntry::ToastDelete(d) => self.fold_delete(*d)?, - SpillEntry::Raw(raw) => self.fold_raw(&raw)?, + SpillEntry::Raw(raw) => { + if let Some(i) = self.fold_raw(&raw)? { + self.check_owned(i.input.xid)?; + return Ok(Some(MergeItem::Input(i))); + } + } SpillEntry::Input(i) => { self.check_owned(i.input.xid)?; return Ok(Some(MergeItem::Input(i))); @@ -2014,9 +1911,12 @@ impl MergedDrain { /// Decode a stashed record against its commit-time verdict: toast heaps /// fold chunks/tombstones into the same maps as live-path entries (so /// same-xact referrers detoast and mirror rows flow), ordinary heaps - /// fan out rows through the pending queue, unresolvable filenodes - /// (end-state-neutral verdicts only) count and drop - fn fold_raw(&mut self, raw: &RawRecord) -> std::result::Result<(), XactBufferError> { + /// yield one input, unresolvable filenodes (end-state-neutral verdicts + /// only) count and drop + fn fold_raw( + &mut self, + raw: &RawRecord, + ) -> std::result::Result>, XactBufferError> { let bump = |c: &dyn Fn(&EmitterStats) -> &AtomicU64| { if let Some(s) = &self.stash.stats { c(s).fetch_add(1, Ordering::Relaxed); @@ -2054,7 +1954,7 @@ impl MergedDrain { // makes this discard sound None => { bump(&|s| &s.toast_stash_discarded); - return Ok(()); + return Ok(None); } }; bump(&|s| &s.toast_stash_decoded); @@ -2070,13 +1970,12 @@ impl MergedDrain { StashedToastOp::Delete(d) => self.fold_delete(d)?, } } - Ok(()) + Ok(None) } - /// Decode one ordinary raw record to zero or more heaps in tuple order, - /// queued for pending-first yield so a MULTI_INSERT's whole fanout - /// precedes anything past the record's LSN. Rebuilt header carries the - /// writer xid, so decoded heaps keep the source `_xid`. + /// Validate one ordinary raw record into an input under its resolved + /// descriptor, `None` when it yields no rows. Rebuilt header carries the + /// writer xid, so rows keep the source `_xid`. /// /// Operation policy: admit only shapes whose logical content is provably /// whole in the record, skip page maintenance, fail closed on anything @@ -2089,7 +1988,7 @@ impl MergedDrain { rel: Arc, valid_from: u64, fence: &[Arc], - ) -> std::result::Result<(), XactBufferError> { + ) -> std::result::Result>, XactBufferError> { use crate::decode::heap_decoder::{ XLH_INSERT_CONTAINS_NEW_TUPLE, XLOG_HEAP_CONFIRM, XLOG_HEAP_DELETE, XLOG_HEAP_HOT_UPDATE, XLOG_HEAP_INSERT, XLOG_HEAP_LOCK, XLOG_HEAP_OPMASK, @@ -2130,7 +2029,7 @@ impl MergedDrain { XLOG_HEAP_INSERT | XLOG_HEAP_UPDATE | XLOG_HEAP_HOT_UPDATE | XLOG_HEAP_DELETE => {} // LOCK changes no row; CONFIRM finalizes a speculative // insert whose tuple decoded from its own record - XLOG_HEAP_LOCK | XLOG_HEAP_CONFIRM => return Ok(()), + XLOG_HEAP_LOCK | XLOG_HEAP_CONFIRM => return Ok(None), // TRUNCATE intercepts at the sink, INPLACE never targets // ordinary heaps: reaching either breaks the verdict _ => return Err(fail(FailClosedReason::UnsupportedOperation)), @@ -2139,7 +2038,7 @@ impl MergedDrain { if op != XLOG_HEAP2_MULTI_INSERT { // PRUNE/VACUUM/FREEZE/VISIBLE/LOCK_UPDATED/NEW_CID/REWRITE: // page maintenance, no logical row change - return Ok(()); + return Ok(None); } // Absent flag means a sub-logical wal_level wrote the record and // an FPI may have consumed its tuple data @@ -2155,31 +2054,30 @@ impl MergedDrain { // Sink stashes heap/heap2 only; anything else broke that invariant return Err(fail(FailClosedReason::UnsupportedOperation)); } - let decoded_set = decode_heap_record(&rec, raw.source_lsn, &rel) + let shapes = scan_heap_record(&rec, &rel) .map_err(|e| fail(FailClosedReason::Malformed(e.to_string())))?; if let Some(s) = &self.stash.stats { s.raw_decode_ordinary_ops.bump(raw.rm, raw.info); s.raw_decode_rows_ops - .add(raw.rm, raw.info, decoded_set.len() as u64); - } - for decoded in decoded_set { - // New-tuple prefix/suffix elision references a same-page - // predecessor image this path cannot reconstruct. Replident-shaped - // old keys keep live-path semantics - if decoded.new.as_ref().is_some_and(|t| t.partial) { - return Err(fail(FailClosedReason::PartialUpdate)); - } - let heap = DescribedHeap { - decoded, - descriptor: rel.clone(), - descriptor_valid_from: valid_from, - }; - let sz = heap.approx_bytes(); - self.gauge.add(sz); - self.pending_gauge.add(sz); - self.pending_heaps.push_back(Box::new(heap)); + .add(raw.rm, raw.info, shapes.len() as u64); } - Ok(()) + // New-tuple prefix/suffix elision references a same-page predecessor + // image this path cannot reconstruct. Replident-shaped old keys keep + // live-path semantics + if shapes + .iter() + .any(|t| t.new.as_ref().is_some_and(|n| n.partial)) + { + return Err(fail(FailClosedReason::PartialUpdate)); + } + if shapes.is_empty() { + return Ok(None); + } + Ok(Some(Box::new(DescribedInput { + input: HeapInput::from_parsed(&rec, raw.source_lsn, shapes.len() as u16), + descriptor: rel, + descriptor_valid_from: valid_from, + }))) } /// Make appended bodies readable through the spool handle; no-op @@ -2359,10 +2257,10 @@ pub enum WalkStep { Event(DrainEntry), /// Follows the `Rows` step sealing `new_rows[..upto]` Truncate { - heap: DescribedHeap, + truncate: Box, upto: usize, }, - Heap(DrainHeap), + Heap(Box), } /// [`DrainedBatch`] decomposed into its apply plan plus the payload fields @@ -2399,14 +2297,14 @@ impl DrainedBatch { steps.push(WalkStep::Event(e.event)); } match heap { - DrainHeap::Cells(heap) if heap.decoded.op == HeapOp::Truncate => { + DrainHeap::Truncate(truncate) => { let upto = trunc .next() .expect("truncate_rows cursor per Truncate heap"); steps.push(WalkStep::Rows { upto }); - steps.push(WalkStep::Truncate { heap, upto }); + steps.push(WalkStep::Truncate { truncate, upto }); } - heap => steps.push(WalkStep::Heap(heap)), + DrainHeap::Input(input) => steps.push(WalkStep::Heap(input)), } } for e in events { @@ -2500,12 +2398,9 @@ impl CommittedDrain { row_idx: m.rows.len(), event: *event, }), - Some(MergeItem::Heap(h)) => { - if h.decoded.op == HeapOp::Truncate { - truncate_rows.push(m.rows.len()); - } - bytes += h.approx_bytes(); - heaps.push(DrainHeap::Cells(*h)); + Some(MergeItem::Truncate(t)) => { + truncate_rows.push(m.rows.len()); + heaps.push(DrainHeap::Truncate(t)); } Some(MergeItem::Input(i)) => { bytes += i.approx_bytes(); @@ -2567,8 +2462,8 @@ pub struct ToastSite { pub p: ToastPointer, } -/// Resolve owned cells' external values in place, see [`detoast_sites`]. -/// Pub for the planner, gap replay, and tests +#[cfg(any(test, feature = "test-support"))] +/// Resolve owned cells' external values in place, see [`detoast_sites`] pub async fn detoast_heap( heap: &mut DescribedHeap, spool: Option<&BodySpoolFile>, @@ -2576,25 +2471,15 @@ pub async fn detoast_heap( resolver: &ToastResolver, ) -> std::result::Result, XactBufferError> { let mut images = [heap.decoded.new.as_mut(), heap.decoded.old.as_mut()]; - let sites: Vec = images - .iter() - .enumerate() - .flat_map(|(image, t)| { - t.iter().flat_map(move |t| { - t.columns - .iter() - .enumerate() - .filter_map(move |(idx, c)| match c { - Some(ColumnValue::ExternalToast(p)) => { - Some(ToastSite { image, idx, p: *p }) - } - _ => None, - }) - }) - }) - .collect(); - let lsn = heap.decoded.source_lsn; - let db = heap.decoded.rfn.db_node; + let mut sites = Vec::new(); + for (image, t) in images.iter().enumerate() { + for (idx, c) in t.iter().flat_map(|t| t.columns.iter().enumerate()) { + if let Some(ColumnValue::ExternalToast(p)) = *c { + sites.push(ToastSite { image, idx, p }); + } + } + } + let (lsn, db) = (heap.decoded.source_lsn, heap.decoded.rfn.db_node); let (values, permit) = detoast_sites( &sites, &heap.descriptor, @@ -2709,17 +2594,6 @@ pub fn external_sites<'v, 'a: 'v>( sites } -/// Whether [`detoast_heap`] resolves anything, ie whether it reads the -/// chunk store. Lets callers owing the read a flush skip it for -/// inline-only heaps -pub fn heap_reads_toast(heap: &DescribedHeap) -> bool { - [heap.decoded.new.as_ref(), heap.decoded.old.as_ref()] - .into_iter() - .flatten() - .flat_map(|t| t.columns.iter()) - .any(|c| matches!(c, Some(ColumnValue::ExternalToast(_)))) -} - /// Fetch every pointer the xact's chunk maps can't cover in one round /// trip per toast rel, so resolution never waits a value at a time. Ids /// dedup: a key used by both tuples fetches once. @@ -2939,7 +2813,7 @@ impl ValueResolution<'_> { let cached = if self.resolver.fill_on_miss() { Err(StoreMiss::NoStore) } else { - let uses = self.uses.get_mut(&key).expect("counted in detoast_heap"); + let uses = self.uses.get_mut(&key).expect("counted in detoast_sites"); *uses -= 1; if *uses > 0 { self.cache.get(&key).cloned() @@ -3321,18 +3195,11 @@ impl BufferingDecoderSink { if rel.kind != 'r' && rel.kind != 'p' { continue; } - let decoded = DecodedHeap { - rfn: rel.rfn, + self.stats.record_op(HeapOp::Truncate, false); + let mut buf = self.lock_buffer().await; + buf.on_truncate(Truncate { xid, source_lsn, - op: HeapOp::Truncate, - new: None, - old: None, - }; - self.stats.record(&decoded); - let mut buf = self.lock_buffer().await; - buf.on_heap(DescribedHeap { - decoded, descriptor: rel, descriptor_valid_from: valid_from, }) @@ -3514,18 +3381,16 @@ impl BufferingDecoderSink { .await .map_err(SinkError::from); } - let decoded_set = { - let _decode = decode_span(); - match decode_heap_record(&record.parsed, record.source_lsn, &rel) { - Ok(set) => set, - Err(e) => return Err(DecoderSinkError::from(e).into()), - } - }; - if decoded_set.is_empty() { - self.skip(SkipReason::NoTuple, record, rfn); - return Ok(()); - } if let Some(kind) = config_kind { + let decoded_set = { + let _decode = decode_span(); + decode_heap_record(&record.parsed, record.source_lsn, &rel) + .map_err(DecoderSinkError::from)? + }; + if decoded_set.is_empty() { + self.skip(SkipReason::NoTuple, record, rfn); + return Ok(()); + } let mut buf = self.lock_buffer().await; for decoded in &decoded_set { if let Some(ev) = crate::runtime_config::interpret(kind, decoded, &rel) { @@ -3534,7 +3399,15 @@ impl BufferingDecoderSink { } return Ok(()); } - let n_decoded = decoded_set.len(); + let images = { + let _decode = decode_span(); + view_heap_record(&record.parsed, &rel).map_err(DecoderSinkError::from)? + }; + if images.is_empty() { + self.skip(SkipReason::NoTuple, record, rfn); + return Ok(()); + } + let n_decoded = images.len(); let buffer_span = trace_span!(sampled, "buffer", rows = n_decoded); // TID for the toast branch: single-tuple INSERT/DELETE only // (toast_save_datum never multi-inserts). blkno rides block ref @@ -3546,10 +3419,13 @@ impl BufferingDecoderSink { // Lock once per record (not per tuple); on_toast_chunk never // touches the catalog, so no buffer→catalog inversion. let mut buf = self.lock_buffer().await; - for decoded in decoded_set { - self.stats.record(&decoded); - let xid = decoded.xid; - if decoded.op == HeapOp::Delete + for img in &images { + let partial = [&img.new, &img.old] + .into_iter() + .flatten() + .any(|t| t.partial); + self.stats.record_op(img.op, partial); + if img.op == HeapOp::Delete && let Some((blkno, offnum)) = tid { // heap_toast_delete's DELETE: TID-keyed. Buffered @@ -3563,20 +3439,24 @@ impl BufferingDecoderSink { toast_relid: rel.oid, blkno, offnum, - source_lsn: decoded.source_lsn, + source_lsn: record.source_lsn, }, - xid, + txn_xid, ) .await .map_err(SinkError::from)?; - } else if decoded.op != HeapOp::Insert { + } else if img.op != HeapOp::Insert { // Non-delete non-insert (TRUNCATE fan-out never // reaches here — kind 't' is filtered there): // nothing to apply against the store self.stats .toast_chunk_deletes .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } else if let Some(chunk) = toast_chunk_from_decoded(decoded, &rel, tid) { + } else if let Some(chunk) = img + .new + .as_ref() + .and_then(|t| toast_chunk(t, &rel, record.source_lsn, tid)) + { if tid.is_none() { // No TID → no store row key: the chunk still // serves same-xact resolution, but its birth @@ -3597,7 +3477,7 @@ impl BufferingDecoderSink { self.stats .toast_chunks_buffered .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - buf.on_toast_chunk(chunk, xid) + buf.on_toast_chunk(chunk, txn_xid) .await .map_err(SinkError::from)?; } else { @@ -3710,16 +3590,20 @@ fn decode_stashed_toast( { return decode_image_insert(raw, rel); } - let decoded_set = decode_heap_record(&rec, raw.source_lsn, rel) + let images = view_heap_record(&rec, rel) .map_err(|e| XactBufferError::Detoast(format!("stashed record decode: {e}")))?; - let tid = (decoded_set.len() == 1) + let tid = (images.len() == 1) .then(|| toast_record_tid(&rec)) .flatten(); - let mut out = Vec::with_capacity(decoded_set.len()); - for decoded in decoded_set { - match decoded.op { + let mut out = Vec::with_capacity(images.len()); + for img in images { + match img.op { HeapOp::Insert => { - if let Some(chunk) = toast_chunk_from_decoded(decoded, rel, tid) { + if let Some(chunk) = img + .new + .as_ref() + .and_then(|t| toast_chunk(t, rel, raw.source_lsn, tid)) + { if tid.is_none() { tracing::warn!( target: "walshadow::xact_buffer", @@ -3775,9 +3659,7 @@ fn decode_image_insert( }; let view = crate::decode::heap_decoder::view_block_data(data, rel) .map_err(|e| XactBufferError::Detoast(format!("stashed image insert: {e}")))?; - let Some((value_id, chunk_seq, chunk_data)) = - crate::decode::heap_decoder::toast_chunk_datums(&view.datums) - else { + let Some(chunk) = toast_chunk(&view, rel, raw.source_lsn, Some((blkno, offnum))) else { tracing::warn!( target: "walshadow::xact_buffer", toast_relid = rel.oid, @@ -3787,44 +3669,32 @@ fn decode_image_insert( ); return Ok(Vec::new()); }; - Ok(vec![StashedToastOp::Chunk(ToastChunk { - db_oid: rel.rfn.db_node, - toast_relid: rel.oid, - value_id, - chunk_seq, - source_lsn: raw.source_lsn, - blkno, - offnum, - chunk_data: bytes::Bytes::copy_from_slice(chunk_data), - })]) + Ok(vec![StashedToastOp::Chunk(chunk)]) } -/// Repack a TOAST table INSERT into a [`ToastChunk`]; `None` for shapes -/// that don't fit. +/// [`ToastChunk`] of a TOAST table tuple; `None` for shapes that don't fit /// /// Keyed on the toast rel's pg_class OID ([`RelDescriptor::oid`]), not /// `rel_node`: the referring tuple's `va_toastrelid` is the OID. They /// diverge after `VACUUM FULL` / `CLUSTER` on the toast rel. -fn toast_chunk_from_decoded( - mut d: DecodedHeap, +fn toast_chunk( + view: &TupleView<'_>, rel: &RelDescriptor, + source_lsn: u64, tid: Option<(u32, u16)>, ) -> Option { - if d.op != HeapOp::Insert { - return None; - } let (value_id, chunk_seq, chunk_data) = - crate::decode::heap_decoder::take_toast_chunk_columns(&mut d.new.as_mut()?.columns)?; + crate::decode::heap_decoder::toast_chunk_datums(&view.datums)?; let (blkno, offnum) = tid.unwrap_or((0, 0)); Some(ToastChunk { db_oid: rel.rfn.db_node, toast_relid: rel.oid, value_id, chunk_seq, - source_lsn: d.source_lsn, + source_lsn, blkno, offnum, - chunk_data: bytes::Bytes::from(chunk_data), + chunk_data: bytes::Bytes::copy_from_slice(chunk_data), }) } @@ -4281,7 +4151,9 @@ mod tests { async fn repeated_eviction_preserves_spilled_prefix_and_memory_tail() { let tmp = tempdir().unwrap(); let mut config = cfg(tmp.path().to_path_buf()); - let row_bytes = approximate_size(&SpillEntry::Heap(Box::new(heap_with_value(7, 1, 128)))); + let row_bytes = approximate_size(&SpillEntry::Input(Box::new( + heap_with_value(7, 1, 128).into(), + ))); config.xact_buffer_max = 4 * row_bytes; let mut buffer = XactBuffer::new(config).unwrap(); for lsn in 1..=12 { @@ -4481,8 +4353,7 @@ mod tests { } #[test] - fn toast_chunk_from_decoded_recognises_three_col_shape() { - use crate::decode::heap_decoder::{DecodedTuple, HeapOp}; + fn toast_chunk_recognises_three_col_shape() { use crate::schema::{RelAttr, RelName, ReplIdent}; let rel = RelDescriptor { rfn: RelFileNode { @@ -4543,34 +4414,22 @@ mod tests { }, ], }; - let d = DecodedHeap { - rfn: rel.rfn, - xid: 5, - source_lsn: 0x1234, - op: HeapOp::Insert, - new: Some(DecodedTuple { - columns: vec![ - Some(ColumnValue::Oid(55)), - Some(ColumnValue::Int4(2)), - Some(ColumnValue::Bytea(b"hello".to_vec())), - ], - partial: false, - }), - old: None, - }; - let chunk = toast_chunk_from_decoded(d.clone(), &rel, Some((7, 3))) - .expect("recognised toast shape"); + use crate::decode::heap_decoder::{form_block_data, view_block_data}; + let full = form_block_data(&[ + Some(ColumnValue::Oid(55)), + Some(ColumnValue::Int4(2)), + Some(ColumnValue::Bytea(b"hello".to_vec())), + ]); + let view = view_block_data(&full, &rel).unwrap(); + let chunk = toast_chunk(&view, &rel, 0x1234, Some((7, 3))).expect("recognised toast shape"); assert_eq!(chunk.toast_relid, 99); // pg_class.oid, not rel_node assert_eq!(chunk.value_id, 55); assert_eq!(chunk.chunk_seq, 2); assert_eq!(&chunk.chunk_data[..], b"hello"); assert_eq!((chunk.blkno, chunk.offnum), (7, 3)); - let mut d2 = d.clone(); - d2.op = HeapOp::Update; - assert!(toast_chunk_from_decoded(d2, &rel, None).is_none()); - let mut d3 = d.clone(); - d3.new.as_mut().unwrap().columns.pop(); - assert!(toast_chunk_from_decoded(d3, &rel, None).is_none()); + let short = form_block_data(&[Some(ColumnValue::Oid(55)), Some(ColumnValue::Int4(2))]); + let view = view_block_data(&short, &rel).unwrap(); + assert!(toast_chunk(&view, &rel, 0, None).is_none()); } #[test] @@ -4826,19 +4685,6 @@ mod tests { map } - /// Callers owing a store read a flush gate on this, so an inline-only - /// heap must answer false whichever tuple carries the pointer - #[test] - fn heap_reads_toast_tracks_external_pointers() { - let mut heap = heap_with_value(1, 0x100, 8); - assert!(!heap_reads_toast(&heap)); - heap.decoded.old = Some(toast_ptr_tuple(9)); - assert!(heap_reads_toast(&heap)); - heap.decoded.new = Some(toast_ptr_tuple(9)); - heap.decoded.old = None; - assert!(heap_reads_toast(&heap)); - } - /// V3 cap fires before allocation with a typed non-retryable error; /// leaf need sizes for the worst per-value transient #[test] @@ -5589,10 +5435,9 @@ mod tests { drain.finish().await.unwrap(); } - /// Decode counters label the record's op; pending gauges track the - /// fanout queue and return to zero once every tuple yields + /// Decode counters label the record's op; one record drains as one input #[tokio::test(flavor = "current_thread")] - async fn raw_fanout_counts_decode_and_pending_gauges() { + async fn raw_fanout_counts_decode_ops() { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16415); @@ -5610,22 +5455,13 @@ mod tests { .next_batch(2, usize::MAX, None) .await .unwrap() - .expect("first slice"); - assert_eq!(batch.cells().unwrap().len(), 2); - assert_eq!(b.raw_pending_rows(), 1, "third fanned tuple still queued"); - assert!(b.raw_pending_bytes() > 0); + .expect("one slice"); + assert!(batch.is_final); + assert_eq!(batch.heaps.len(), 1); + assert_eq!(batch.cells().unwrap().len(), 3); let ops = stats.raw_decode_ordinary_ops.load(); assert_eq!(ops[5], 1, "one multi_insert record decoded"); assert_eq!(stats.raw_decode_rows_ops.load()[5], 3, "three rows fanned"); - let batch = drain - .next_batch(8, usize::MAX, None) - .await - .unwrap() - .expect("final slice"); - assert!(batch.is_final); - assert_eq!(batch.cells().unwrap().len(), 1); - assert_eq!(b.raw_pending_rows(), 0); - assert_eq!(b.raw_pending_bytes(), 0); drain.finish().await.unwrap(); } diff --git a/tests/bootstrap_direct_e2e.rs b/tests/bootstrap_direct_e2e.rs index 48e51814..dc3e5f8f 100644 --- a/tests/bootstrap_direct_e2e.rs +++ b/tests/bootstrap_direct_e2e.rs @@ -15,7 +15,7 @@ //! → DirectSource(PgConfig→source, BaseBackupOpts{fast_checkpoint}) //! → MultiplexSink(DiskLanderSink + PageWalkSink) //! → mpsc -//! → drain_backfill → RecordingObserver +//! → collected tuples //! ``` //! //! Skipped silently when `initdb` is not on `$PATH`. @@ -35,11 +35,9 @@ use foldhash::{HashSet, HashSetExt}; use walrus::pg::replication::base_backup::BaseBackupOpts; use walrus::pg::replication::conn::PgConfig; use walrus::pg::replication::tls::{SslMode, TlsParams}; -use walshadow::backfill_bootstrap::{ - BootstrapConfig, drain_backfill, seed_in_snapshot, spawn_greenfield_bootstrap, -}; +use walshadow::backfill_bootstrap::{BootstrapConfig, run_greenfield_bootstrap, seed_in_snapshot}; use walshadow::backup_source_direct::DirectSource; -use walshadow::heap_decoder::{ColumnValue, HeapOp}; +use walshadow::heap_decoder::ColumnValue; use walshadow::shadow::{Shadow, ShadowConfig}; /// Same row budget as the object-store sibling: one heap page worth so @@ -153,15 +151,10 @@ async fn direct_source_self_hosted_via_replication_protocol() { let shadow_data = tmp.path().join("shadow-data"); let cfg = BootstrapConfig::new(shadow_data.clone()); - let (rx, pump) = spawn_greenfield_bootstrap(cfg, Box::new(direct), catalog_map.clone(), false); - - let mut observer = RecordingObserver::default(); - let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &catalog_map, &mut observer), pump); - let shipped = drain_res.expect("drain task error"); - let outcome = pump_res - .expect("pump task panicked") - .expect("pump task returned error"); - assert_eq!(observer.xact_end_calls, 1); + let (outcome, tuples) = + run_greenfield_bootstrap(cfg, Box::new(direct), catalog_map.clone(), false) + .await + .expect("bootstrap"); // LSN handoff assert!(outcome.start.start_lsn > 0, "start_lsn must be non-zero"); @@ -253,24 +246,20 @@ async fn direct_source_self_hosted_via_replication_protocol() { .load(std::sync::atomic::Ordering::Relaxed), ); assert_eq!( - shipped, + tuples.len() as u64, outcome .page_walk .tuples_emitted .load(std::sync::atomic::Ordering::Relaxed) ); - // CommittedTuple shape let mut int_ids: HashSet = HashSet::new(); - for c in &observer.tuples { - assert_eq!(c.commit_ts, 0); - assert_eq!(c.commit_lsn, outcome.start.start_lsn); - assert_eq!(c.decoded.op, HeapOp::Insert); - assert_eq!(c.decoded.source_lsn, outcome.start.start_lsn); - assert!(c.decoded.old.is_none()); - let new = c.decoded.new.as_ref().expect("Insert must carry `new`"); - assert!(!new.partial); - if let Some(Some(ColumnValue::Int4(v))) = new.columns.first() { + for t in &tuples { + assert_eq!(t.source_lsn, outcome.start.start_lsn); + let rel = catalog_map + .get(t.rfn.db_node, t.rfn.rel_node) + .expect("descriptor"); + if let Some(Some(ColumnValue::Int4(v))) = t.cells(&rel).expect("cells").first() { int_ids.insert(*v); } } @@ -283,38 +272,3 @@ async fn direct_source_self_hosted_via_replication_protocol() { int_ids.len(), ); } - -#[derive(Default)] -struct RecordingObserver { - tuples: Vec, - xact_end_calls: u32, -} - -impl walshadow::decoder_sink::TupleObserver for RecordingObserver { - fn on_tuple<'a>( - &'a mut self, - committed: &'a walshadow::heap_decoder::CommittedTuple, - ) -> std::pin::Pin< - Box< - dyn std::future::Future> - + Send - + 'a, - >, - > { - self.tuples.push(committed.clone()); - Box::pin(async { Ok(()) }) - } - fn on_xact_end<'a>( - &'a mut self, - commit_lsn: u64, - ) -> std::pin::Pin< - Box< - dyn std::future::Future> - + Send - + 'a, - >, - > { - self.xact_end_calls += 1; - Box::pin(async move { Ok(commit_lsn) }) - } -} diff --git a/tests/bootstrap_object_store_e2e.rs b/tests/bootstrap_object_store_e2e.rs index 5e8beea5..ed1394cf 100644 --- a/tests/bootstrap_object_store_e2e.rs +++ b/tests/bootstrap_object_store_e2e.rs @@ -17,7 +17,7 @@ //! → ObjectStoreSource(settings, FsStorage(temp/storage), name) //! → MultiplexSink(DiskLanderSink + PageWalkSink) //! → mpsc -//! → drain_backfill → RecordingObserver (records tuples + on_xact_end) +//! → collected tuples //! ``` //! //! What this exercises end-to-end (not covered by lib unit tests): @@ -64,11 +64,9 @@ use walrus::pg::backup::push::{self, PushArgs}; use walrus::pg::replication::conn::PgConfig; use walrus::storage::DynStorage; use walrus::storage::fs::FsStorage; -use walshadow::backfill_bootstrap::{ - BootstrapConfig, drain_backfill, seed_in_snapshot, spawn_greenfield_bootstrap, -}; +use walshadow::backfill_bootstrap::{BootstrapConfig, run_greenfield_bootstrap, seed_in_snapshot}; use walshadow::backup_source_object_store::ObjectStoreSource; -use walshadow::heap_decoder::{ColumnValue, HeapOp}; +use walshadow::heap_decoder::ColumnValue; use walshadow::shadow::{Shadow, ShadowConfig}; /// Row count loaded into the user table before push runs. Must be >0 @@ -229,10 +227,6 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { .unwrap() .get(0); - // Run the orchestrator. spawn_greenfield_bootstrap yields the - // receiver up front so drain_backfill runs concurrently with the - // source pump — bounded by source decode bandwidth, not by total - // tuple count. let shadow_data = tmp.path().join("shadow-data"); let cfg = BootstrapConfig::new(shadow_data.clone()); let object_source = ObjectStoreSource::new( @@ -242,27 +236,10 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { tmp.path().join("part-spool"), ) .with_parallelism(2); - let (rx, pump) = - spawn_greenfield_bootstrap(cfg, Box::new(object_source), catalog_map.clone(), false); - - // RecordingObserver wraps a CollectingTupleObserver + counts - // on_xact_end so the test asserts drain_backfill closes the - // synthetic backfill xact exactly once after channel drain. The - // production daemon path uses this signal to flush the - // transitional emitter's INSERT block before swapping to the - // ShadowCatalog-backed emitter. - let mut observer = RecordingObserver::default(); - let (drain_res, pump_res) = tokio::join!(drain_backfill(rx, &catalog_map, &mut observer), pump); - let shipped = drain_res.expect("drain task error"); - let outcome = pump_res - .expect("pump task panicked") - .expect("pump task returned error"); - assert_eq!( - observer.xact_end_calls, 1, - "drain_backfill must call on_xact_end exactly once after channel drain — \ - got {} calls", - observer.xact_end_calls, - ); + let (outcome, tuples) = + run_greenfield_bootstrap(cfg, Box::new(object_source), catalog_map.clone(), false) + .await + .expect("bootstrap"); // --- LSN handoff --- assert!(outcome.start.start_lsn > 0, "start_lsn must be non-zero"); @@ -369,32 +346,21 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { // --- Drain delivered every tuple --- assert_eq!( - shipped, + tuples.len() as u64, outcome .page_walk .tuples_emitted .load(std::sync::atomic::Ordering::Relaxed), - "drain count != page walk emit count — channel dropped tuples" + "collected count != page walk emit count, channel dropped tuples" ); - // --- CommittedTuple shape (every captured tuple) --- let mut int_ids: HashSet = HashSet::new(); - for c in &observer.tuples { - assert_eq!(c.commit_ts, 0, "commit_ts must be 0 for backfill rows"); - assert_eq!( - c.commit_lsn, outcome.start.start_lsn, - "commit_lsn must equal start_lsn", - ); - assert_eq!(c.decoded.op, HeapOp::Insert); - assert_eq!(c.decoded.source_lsn, outcome.start.start_lsn); - assert!(c.decoded.old.is_none()); - let new = c.decoded.new.as_ref().expect("Insert must carry `new`"); - assert!(!new.partial); - // The s12.t.id column is int4 at attnum 1. PG may have given - // us catalog rows too if their filenode rotated above 16384, - // but ordinary fresh-cluster catalogs stay < 16384 — the - // captured stream should be pure s12.t rows here. - if let Some(Some(ColumnValue::Int4(v))) = new.columns.first() { + for t in &tuples { + assert_eq!(t.source_lsn, outcome.start.start_lsn); + let rel = catalog_map + .get(t.rfn.db_node, t.rfn.rel_node) + .expect("descriptor"); + if let Some(Some(ColumnValue::Int4(v))) = t.cells(&rel).expect("cells").first() { int_ids.insert(*v); } } @@ -407,42 +373,3 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { int_ids.len(), ); } - -/// Recording observer = `CollectingTupleObserver` + `on_xact_end` -/// counter. Lets the e2e drill assert both per-tuple shape and the -/// synthetic xact-close signal that the daemon's transitional emitter -/// path (Solution 2) relies on to flush its INSERT block. -#[derive(Default)] -struct RecordingObserver { - tuples: Vec, - xact_end_calls: u32, -} - -impl walshadow::decoder_sink::TupleObserver for RecordingObserver { - fn on_tuple<'a>( - &'a mut self, - committed: &'a walshadow::heap_decoder::CommittedTuple, - ) -> std::pin::Pin< - Box< - dyn std::future::Future> - + Send - + 'a, - >, - > { - self.tuples.push(committed.clone()); - Box::pin(async { Ok(()) }) - } - fn on_xact_end<'a>( - &'a mut self, - commit_lsn: u64, - ) -> std::pin::Pin< - Box< - dyn std::future::Future> - + Send - + 'a, - >, - > { - self.xact_end_calls += 1; - Box::pin(async move { Ok(commit_lsn) }) - } -} diff --git a/tests/xact_buffer.rs b/tests/xact_buffer.rs index e1cfb5d2..76e662f8 100644 --- a/tests/xact_buffer.rs +++ b/tests/xact_buffer.rs @@ -39,7 +39,7 @@ use walshadow::shadow_catalog::{ShadowCatalog, ShadowCatalogConfig}; use walshadow::spill::ToastChunk; use walshadow::toast::{ChunkRefMap, MemChunkStore, ToastResolver}; use walshadow::xact_buffer::{ - StashResolved, WalkStep, XactBuffer, XactBufferConfig, XactBufferError, detoast_heap, + DrainHeap, StashResolved, WalkStep, XactBuffer, XactBufferConfig, XactBufferError, detoast_heap, }; fn make_shadow(tmp: &tempfile::TempDir, port: u16) -> Shadow { @@ -252,7 +252,7 @@ async fn drain_all( let WalkStep::Heap(drained) = step else { continue; }; - for mut heap in drained.into_cells()? { + for mut heap in DrainHeap::Input(drained).into_cells()? { detoast_heap(&mut heap, spool, &ref_maps, resolver).await?; out.push(CommittedTuple { decoded: heap.decoded, From 2f16cd5328a6ea0d1da82cc9af28a70a05ce385c Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:53:28 +0000 Subject: [PATCH 13/20] Count heap records from headers, plan views without boxing Sink and stash gather more from record headers instead of walking every tuple. WAL replay now rejects partial images itself Tuple views keep datums in Vec rather than 32-slot inline SmallVec, so per-row copies shrink and plan futures stay small enough to await without boxing. The inline-only planner fast path goes away --- src/backfill/wal_replay.rs | 7 ++- src/decode/decoder_sink.rs | 19 +++---- src/decode/heap_decoder.rs | 96 ++++++++++++++++++++++++++++++++---- src/emit/pipeline/planner.rs | 95 ++++++++++------------------------- src/xact/xact_buffer.rs | 65 +++++++++++------------- 5 files changed, 158 insertions(+), 124 deletions(-) diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index 1e06b36b..8ad559e8 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -676,7 +676,7 @@ impl WalReplaySink { spool, ref_maps: &ref_maps, }; - Box::pin(self.replay_views(&input, route, &at)).await?; + self.replay_views(&input, route, &at).await?; } } } @@ -707,6 +707,11 @@ impl WalReplaySink { let images = view_heap_record(input.input.record_bytes(), rel) .map_err(|e| SinkError::Other(format!("wal_replay: decode at {lsn:#X}: {e}")))?; for h in &images { + if h.new.as_ref().is_some_and(|t| t.partial) { + return Err(SinkError::Other(format!( + "wal_replay: partial update at {lsn:#X}" + ))); + } if !self.lands(route, h.op) { continue; } diff --git a/src/decode/decoder_sink.rs b/src/decode/decoder_sink.rs index 3d70518a..9d494c89 100644 --- a/src/decode/decoder_sink.rs +++ b/src/decode/decoder_sink.rs @@ -142,15 +142,16 @@ pub struct SkipKey { pub type SkipCounts = crate::emit::pipeline::row_ledger::KeyedCounts; impl DecoderStats { - pub fn record_op(&self, op: crate::decode::heap_decoder::HeapOp, partial: bool) { + pub fn record_op(&self, op: crate::decode::heap_decoder::HeapOp, rows: u16, partial: bool) { use crate::decode::heap_decoder::HeapOp; - self.decoded.fetch_add(1, Ordering::Relaxed); + let rows = u64::from(rows); + self.decoded.fetch_add(rows, Ordering::Relaxed); match op { - HeapOp::Insert => self.inserts.fetch_add(1, Ordering::Relaxed), - HeapOp::Update => self.updates.fetch_add(1, Ordering::Relaxed), - HeapOp::HotUpdate => self.hot_updates.fetch_add(1, Ordering::Relaxed), - HeapOp::Delete => self.deletes.fetch_add(1, Ordering::Relaxed), - HeapOp::Truncate => self.truncates.fetch_add(1, Ordering::Relaxed), + HeapOp::Insert => self.inserts.fetch_add(rows, Ordering::Relaxed), + HeapOp::Update => self.updates.fetch_add(rows, Ordering::Relaxed), + HeapOp::HotUpdate => self.hot_updates.fetch_add(rows, Ordering::Relaxed), + HeapOp::Delete => self.deletes.fetch_add(rows, Ordering::Relaxed), + HeapOp::Truncate => self.truncates.fetch_add(rows, Ordering::Relaxed), }; if partial { self.partial.fetch_add(1, Ordering::Relaxed); @@ -280,9 +281,9 @@ mod tests { HeapOp::HotUpdate, HeapOp::Delete, ] { - s.record_op(op, false); + s.record_op(op, 1, false); } - s.record_op(HeapOp::Update, true); + s.record_op(HeapOp::Update, 1, true); let ld = |a: &AtomicU64| a.load(Ordering::Relaxed); assert_eq!(ld(&s.decoded), 6); assert_eq!(ld(&s.inserts), 2); diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index ccb31fb3..7001ac4e 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -458,17 +458,17 @@ impl<'a> TupleBuild<'a> for DecodedTuple { } /// Tuple image borrowing its record. Indexed by attnum-1 like -/// [`DecodedTuple::columns`]; inline up to 32 attributes +/// [`DecodedTuple::columns`] #[derive(Clone, Debug, Default, PartialEq)] pub struct TupleView<'a> { - pub datums: SmallVec<[DatumRef<'a>; 32]>, + pub datums: Vec>, pub partial: bool, } impl<'a> TupleBuild<'a> for TupleView<'a> { fn with_capacity(n: usize) -> Self { Self { - datums: SmallVec::with_capacity(n), + datums: Vec::with_capacity(n), partial: false, } } @@ -585,12 +585,60 @@ impl<'a> From<&'a XLogRecord<'_>> for HeapRecordBytes<'a> { } } -/// Validate `record` as [`decode_heap_record`] would, without building cells -pub fn scan_heap_record( - record: &XLogRecord<'_>, - rel: &RelDescriptor, -) -> Result; 1]>, DecodeError> { - parse_heap_record(record.into(), rel) +/// Heap record summary read from its headers, tuple bytes unchecked +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordShape { + pub op: HeapOp, + pub tuples: u16, + /// PG elided an update's prefix or suffix + pub partial: bool, +} + +/// Shape of a record [`view_heap_record`] yields images for, `None` when it +/// yields none +pub fn record_shape<'a>( + record: impl Into>, +) -> Result, DecodeError> { + let record = record.into(); + let md = record.main_data; + let heap2 = record.rm == RmId::Heap2 as u8; + let (op, need) = match record.info & XLOG_HEAP_OPMASK { + XLOG_HEAP2_MULTI_INSERT if heap2 => (HeapOp::Insert, SIZE_OF_HEAP_MULTI_INSERT), + _ if heap2 || record.rm != RmId::Heap as u8 => return Ok(None), + XLOG_HEAP_INSERT => (HeapOp::Insert, 0), + XLOG_HEAP_UPDATE => (HeapOp::Update, SIZE_OF_HEAP_UPDATE), + XLOG_HEAP_HOT_UPDATE => (HeapOp::HotUpdate, SIZE_OF_HEAP_UPDATE), + XLOG_HEAP_DELETE => (HeapOp::Delete, SIZE_OF_HEAP_DELETE), + _ => return Ok(None), + }; + if md.len() < need { + return Err(DecodeError::Truncated { + offset: 0, + need, + have: md.len(), + }); + } + let mut shape = RecordShape { + op, + tuples: 1, + partial: false, + }; + if heap2 { + shape.tuples = u16::from_le_bytes([md[2], md[3]]); + if shape.tuples == 0 { + return Err(DecodeError::Truncated { + offset: 0, + need: 1, + have: 0, + }); + } + if md[0] & XLH_INSERT_CONTAINS_NEW_TUPLE == 0 { + return Ok(None); + } + } else if matches!(op, HeapOp::Update | HeapOp::HotUpdate) { + shape.partial = md[7] & (XLH_UPDATE_PREFIX_FROM_OLD | XLH_UPDATE_SUFFIX_FROM_OLD) != 0; + } + Ok(Some(shape)) } fn parse_heap_record<'a, T: TupleBuild<'a>>( @@ -2404,6 +2452,36 @@ mod tests { assert_eq!(new.columns[1], Some(ColumnValue::Null)); } + #[test] + fn record_shape_reads_headers() { + let shape = |rm: RmId, info, main_data: &[u8]| { + record_shape(HeapRecordBytes { + rm: rm as u8, + info, + main_data, + block: None, + }) + }; + let mut update = [0u8; SIZE_OF_HEAP_UPDATE]; + update[7] = XLH_UPDATE_SUFFIX_FROM_OLD; + let s = shape(RmId::Heap, XLOG_HEAP_HOT_UPDATE, &update) + .unwrap() + .unwrap(); + assert_eq!((s.op, s.tuples, s.partial), (HeapOp::HotUpdate, 1, true)); + let multi = [XLH_INSERT_CONTAINS_NEW_TUPLE, 0, 3, 0]; + let s = shape(RmId::Heap2, XLOG_HEAP2_MULTI_INSERT, &multi) + .unwrap() + .unwrap(); + assert_eq!((s.op, s.tuples, s.partial), (HeapOp::Insert, 3, false)); + assert!(matches!( + shape(RmId::Heap2, XLOG_HEAP2_MULTI_INSERT, &[0, 0, 3, 0]), + Ok(None) + )); + assert!(matches!(shape(RmId::Heap, XLOG_HEAP_LOCK, &[]), Ok(None))); + assert!(shape(RmId::Heap2, XLOG_HEAP2_MULTI_INSERT, &[0, 0, 0, 0]).is_err()); + assert!(shape(RmId::Heap, XLOG_HEAP_DELETE, &[0; 4]).is_err()); + } + #[test] fn decode_multi_insert_three_rows() { let rel = descriptor(16410, vec![rel_attr(1, "id", INT4OID, 4, 'i')]); diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 6f9de741..d79e96cf 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -39,7 +39,7 @@ use std::sync::Arc; -use crate::decode::heap_decoder::{ColumnValue, HeapImages, TupleView, view_heap_record}; +use crate::decode::heap_decoder::{ColumnValue, view_heap_record}; use crate::emit::pipeline::plan_spool::{ DEFAULT_PLAN_MEM_MAX, EncoderCache, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, @@ -208,11 +208,7 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { self.writer.discard(table, u64::from(input.input.tuples)); continue; } - if !self.plan_inline(&input, table)? { - // Boxed: external values are rare, and awaiting them - // holds tuple views deep enough to bloat every plan future - Box::pin(self.plan_views(&input, table, spool, &ref_maps)).await?; - } + self.plan_views(&input, table, spool, &ref_maps).await?; } } } @@ -221,66 +217,6 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Ok(()) } - /// Tuple images of `input`, rejecting partial new images - fn views<'i>( - input: &'i DescribedInput, - ) -> Result>; 1]>, PlanError> { - let rel = &input.descriptor; - let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { - XactBufferError::OrdinaryFailClosed { - relid: rel.oid, - lsn: input.input.source_lsn, - rm: input.input.rm, - info: input.input.info, - reason: FailClosedReason::Malformed(e.to_string()), - } - })?; - if images - .iter() - .any(|h| h.new.as_ref().is_some_and(|t| t.partial)) - { - return Err(PlanError::PartialUpdate { - lsn: input.input.source_lsn, - }); - } - Ok(images) - } - - fn push_images( - &mut self, - input: &DescribedInput, - h: &HeapImages>, - patches: &[Vec<(usize, ColumnValue)>; 2], - table: u32, - ) -> Result<(), PlanError> { - let change = ViewChange { - op: h.op, - xid: input.input.xid, - source_lsn: input.input.source_lsn, - new: h.new.as_ref(), - old: h.old.as_ref(), - new_patch: &patches[0], - old_patch: &patches[1], - }; - Ok(self.writer.push_views(change, table)?) - } - - /// Encode an input without external values; false leaves it to - /// [`Self::plan_views`] - fn plan_inline(&mut self, input: &DescribedInput, table: u32) -> Result { - let images = Self::views(input)?; - if images - .iter() - .any(|h| !external_sites([h.new.as_ref(), h.old.as_ref()]).is_empty()) - { - return Ok(false); - } - for h in &images { - self.push_images(input, h, &Default::default(), table)?; - } - Ok(true) - } - /// Encode an input from tuple images in place, external values resolved /// into per-image patches async fn plan_views( @@ -291,12 +227,24 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { ref_maps: &[&ChunkRefMap], ) -> Result<(), PlanError> { let rel = &input.descriptor; - for h in &Self::views(input)? { + let lsn = input.input.source_lsn; + let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { + XactBufferError::OrdinaryFailClosed { + relid: rel.oid, + lsn, + rm: input.input.rm, + info: input.input.info, + reason: FailClosedReason::Malformed(e.to_string()), + } + })?; + for h in &images { + if h.new.as_ref().is_some_and(|t| t.partial) { + return Err(PlanError::PartialUpdate { lsn }); + } let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); if !sites.is_empty() { let db = rel.rfn.db_node; - let lsn = input.input.source_lsn; // Value permit drops once bytes land in the plan let (values, _permit) = detoast_sites(&sites, rel, db, lsn, spool, ref_maps, self.resolver).await?; @@ -304,7 +252,16 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { patches[site.image].push((site.idx, v)); } } - self.push_images(input, h, &patches, table)?; + let change = ViewChange { + op: h.op, + xid: input.input.xid, + source_lsn: lsn, + new: h.new.as_ref(), + old: h.old.as_ref(), + new_patch: &patches[0], + old_patch: &patches[1], + }; + self.writer.push_views(change, table)?; } Ok(()) } diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index c39fa247..6ec71d7f 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -54,7 +54,7 @@ use crate::catalog::pending::{PendingCatalog, PendingSlot}; use crate::decode::datum::DatumRef; use crate::decode::decoder_sink::{DecoderSinkError, DecoderStats, SkipReason}; use crate::decode::heap_decoder::{ - ColumnValue, HeapImages, HeapOp, ToastPointer, TupleView, decode_heap_record, scan_heap_record, + ColumnValue, HeapImages, HeapOp, ToastPointer, TupleView, decode_heap_record, record_shape, view_heap_record, }; #[cfg(any(test, feature = "test-support"))] @@ -2054,27 +2054,23 @@ impl MergedDrain { // Sink stashes heap/heap2 only; anything else broke that invariant return Err(fail(FailClosedReason::UnsupportedOperation)); } - let shapes = scan_heap_record(&rec, &rel) - .map_err(|e| fail(FailClosedReason::Malformed(e.to_string())))?; + let Some(shape) = + record_shape(&rec).map_err(|e| fail(FailClosedReason::Malformed(e.to_string())))? + else { + return Ok(None); + }; if let Some(s) = &self.stash.stats { s.raw_decode_ordinary_ops.bump(raw.rm, raw.info); s.raw_decode_rows_ops - .add(raw.rm, raw.info, shapes.len() as u64); + .add(raw.rm, raw.info, u64::from(shape.tuples)); } // New-tuple prefix/suffix elision references a same-page predecessor - // image this path cannot reconstruct. Replident-shaped old keys keep - // live-path semantics - if shapes - .iter() - .any(|t| t.new.as_ref().is_some_and(|n| n.partial)) - { + // image this path cannot reconstruct + if shape.partial { return Err(fail(FailClosedReason::PartialUpdate)); } - if shapes.is_empty() { - return Ok(None); - } Ok(Some(Box::new(DescribedInput { - input: HeapInput::from_parsed(&rec, raw.source_lsn, shapes.len() as u16), + input: HeapInput::from_parsed(&rec, raw.source_lsn, shape.tuples), descriptor: rel, descriptor_valid_from: valid_from, }))) @@ -3195,7 +3191,7 @@ impl BufferingDecoderSink { if rel.kind != 'r' && rel.kind != 'p' { continue; } - self.stats.record_op(HeapOp::Truncate, false); + self.stats.record_op(HeapOp::Truncate, 1, false); let mut buf = self.lock_buffer().await; buf.on_truncate(Truncate { xid, @@ -3352,30 +3348,19 @@ impl BufferingDecoderSink { .filter(|schema| &*rel.rel_name.namespace == *schema) .and_then(|_| ConfigTableKind::from_relname(&rel.rel_name.name)); if rel.kind != 't' && config_kind.is_none() { - // Ordinary rows stay as WAL bytes until commit; scanning now - // keeps malformed-record errors and per-op counts at record time - let shapes = { - let _decode = decode_span(); - scan_heap_record(&record.parsed, &rel).map_err(DecoderSinkError::from)? - }; - if shapes.is_empty() { + // Ordinary rows stay as WAL bytes until commit, tuple bytes + // validate where planning reads them + let Some(shape) = record_shape(&record.parsed).map_err(DecoderSinkError::from)? else { self.skip(SkipReason::NoTuple, record, rfn); return Ok(()); - } - for t in &shapes { - let partial = [&t.new, &t.old].into_iter().flatten().any(|s| s.partial); - self.stats.record_op(t.op, partial); - } + }; + self.stats.record_op(shape.op, shape.tuples, shape.partial); let input = DescribedInput { - input: HeapInput::from_parsed( - &record.parsed, - record.source_lsn, - shapes.len() as u16, - ), + input: HeapInput::from_parsed(&record.parsed, record.source_lsn, shape.tuples), descriptor: rel, descriptor_valid_from: rel_valid_from, }; - let buffer_span = trace_span!(sampled, "buffer", rows = shapes.len()); + let buffer_span = trace_span!(sampled, "buffer", rows = shape.tuples); return async { self.lock_buffer().await.on_input(input).await } .instrument(buffer_span) .await @@ -3424,7 +3409,7 @@ impl BufferingDecoderSink { .into_iter() .flatten() .any(|t| t.partial); - self.stats.record_op(img.op, partial); + self.stats.record_op(img.op, 1, partial); if img.op == HeapOp::Delete && let Some((blkno, offnum)) = tid { @@ -7284,9 +7269,17 @@ mod tests { .await .unwrap(); assert_eq!(skip_count(&sink, live, SkipReason::NoTuple), 1); - let (info, md, _) = multi_insert_parts(XLH_INSERT_CONTAINS_NEW_TUPLE, &[int4_tuple(1)]); + // Sink reads headers only, tuple bytes are checked at decode + let (info, md, data) = multi_insert_parts(XLH_INSERT_CONTAINS_NEW_TUPLE, &[int4_tuple(1)]); let err = sink - .on_record(&heap_record(RmId::Heap2, info, 7, live, md, vec![0; 3])) + .on_record(&heap_record( + RmId::Heap2, + info, + 7, + live, + md[..2].to_vec(), + data, + )) .await .unwrap_err(); assert!(err.to_string().contains("truncated"), "{err}"); From e90ebe93d8dd8f69eb9a5b4f23a5575d30cdf575 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:33:23 +0000 Subject: [PATCH 14/20] improve memory accounting --- architecture/README.md | 4 +- docs/configuration.md | 31 ++++-- plans/INDEX.md | 1 - plans/native_encoding.md | 52 ---------- plans/performance.md | 24 +++++ plans/tier2.md | 3 +- plans/verification.md | 8 ++ src/backfill/backfill_staging.rs | 4 +- src/bin/stream/metrics_publish.rs | 3 + src/budget.rs | 66 +++++++++++- src/ch/test_support.rs | 21 +++- src/crash_test.rs | 2 + src/emit/ch_ddl.rs | 2 +- src/emit/ch_emitter.rs | 15 ++- src/emit/direct_parity.rs | 148 +++++++++++++++++++++++++++ src/emit/pipeline/batcher.rs | 43 +++++--- src/emit/pipeline/inserter.rs | 45 +++++++-- src/emit/pipeline/resolver.rs | 105 ++++++++++++++++++- src/emit/pipeline/tail.rs | 27 +++-- src/ops/metrics.rs | 8 +- src/xact/spill.rs | 5 +- src/xact/xact_buffer.rs | 19 ++-- tests/control_plane_e2e.rs | 93 +++++++++++++++++ tests/emitter_budget_flush.rs | 162 ++++++++++++++++++++++++++++++ tests/kill_restart.rs | 64 ++++++++---- tests/tee_cdc.rs | 10 +- 26 files changed, 813 insertions(+), 152 deletions(-) delete mode 100644 plans/native_encoding.md diff --git a/architecture/README.md b/architecture/README.md index 06189567..b746e863 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -53,7 +53,9 @@ Each inserter owns a ClickHouse connection and takes any ready batch Sequence numbers identify work slices, not necessarily whole transactions Only a commit's final slice publishes its LSN, after all earlier work finishes Bounded queues and a shared payload budget limit work in flight; transaction -and plan data can spill to disk +and plan data can spill to disk. Each tail's output pool holds batches from +batcher to acknowledgement and reserves seal workspace apart from them, so +conversion finishes however much output waits on inserters Queue worker runs on a dedicated thread. A dispatcher scans each batch ahead of it to start TOAST store reads for values updated rows keep. Ordinary heap diff --git a/docs/configuration.md b/docs/configuration.md index 8b4ea18f..db994070 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -335,20 +335,29 @@ ClickHouse tails bound sealed encoder buffers by startup `[ch] byte_budget`, including spare vector capacity. Each batch targets that budget divided by inserter count, keeping concurrent inserts within a byte limit. Partial batches across tables share another `byte_budget`; a producer waiting to hand off a -sealed batch can hold one additional batch. A single oversized row may exceed -these limits. Oracle responses and native client buffers remain additional costs +sealed batch can hold one additional batch. A batch above `byte_budget` waits +until no other batch holds budget Sealed-buffer permits stay held through resolution, retries and insert completion. After resolution each Native block is serialized once, compressed with -`[ch] compression`, and kept in that form until acknowledged; its permit shrinks -to memory those bytes occupy, which can exceed their length. Inserters send -those bytes unchanged. A connection unable to accept them, such as a -destination configured with other compression, gets each block decoded and -rebuilt, so one block of scratch per inserter adds to these limits. An INSERT -carries several blocks, sealed at 1,048,576 rows or 16 MiB of column bytes and -when oracle request size fills, while batch limits still bound each INSERT. -`walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` -reports how far sealing shrinks resident blocks +`[ch] compression`, and kept in that form until acknowledged; its permit settles +to memory those bytes occupy, which can exceed their length. Output beyond +what a batch reserved, such as frame overhead on incompressible data, is held +without waiting. Each resolver also +reserves workspace on top of `byte_budget` for one block's compressed output +and frame scratch, plus 64 MiB for oracle request and response when the tail +has an oracle; resolvers count oracle bridge concurrency, or one without an +oracle. Blocks needing more workspace than that reserve seal one at a time +Inserters send prepared bytes unchanged. A connection unable to accept them, +such as a destination configured with other compression, gets each block +decoded and rebuilt into one block of scratch per inserter, outside the pool. +An INSERT carries several blocks, sealed at 1,048,576 rows or 16 MiB of column +bytes and when oracle request size fills, while batch limits still bound each +INSERT. `walshadow_resident_raw_bytes_total` over +`walshadow_resident_stored_bytes_total` reports how far sealing shrinks +resident blocks. `walshadow_output_resident_bytes`, +`walshadow_batcher_open_bytes`, and `walshadow_inserter_scratch_bytes` report +allocated bytes held by output pools, open batches, and rebuild scratch This pool is separate from decoded-payload admission to avoid circular waits. Live byte-budget reductions lower flush thresholds; increasing sealed-buffer capacity requires restarting walshadow diff --git a/plans/INDEX.md b/plans/INDEX.md index 118ec4a5..5b4c139b 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -39,7 +39,6 @@ writes. A documented limitation does not imply code rejects it | [Multi-database loads and metrics](multi_database.md) | Extend heap-page bootstrap and backup loads beyond primary database, attribute metrics | | [Fuzzing](fuzzing.md) | Find parser and schema-transition interactions beyond fixed regressions | | [Performance](performance.md) | Establish healthy sustained WAL baseline with exact reconciliation before tuning | -| [Direct Native encoding](native_encoding.md) | Bound memory across owners, prove resource gains end to end | | [Failover](failover.md) | Fence unplanned promotion and prove remaining archive/restart fault cases | | [Shadow TOAST reclamation](shadow_toast.md) | Keep historical values readable under lag and restart | | [Replay callback](custom_rmgr.md) | Reduce measured command-boundary capture stalls | diff --git a/plans/native_encoding.md b/plans/native_encoding.md deleted file mode 100644 index 3be50d02..00000000 --- a/plans/native_encoding.md +++ /dev/null @@ -1,52 +0,0 @@ -# Encode PostgreSQL values directly into ClickHouse blocks - -Commits, loads, and gap replay encode rows from tuple bytes or COPY fields into -column chunks, and sealed Native blocks stay compressed until acknowledgement; -see [architecture](../architecture/README.md#commit-pipeline). Remaining work -bounds memory across owners and proves resource gains end to end. Follow [shared constraints](coordination.md), -[value coercion](value_coercion.md), and [performance acceptance](performance.md) - -Share view encoders and column buffers with [container codecs](tier2.md) -Container promotion can retain existing raw pending cells in owned adapters - -## Bound memory across owners - -Track allocated capacity, not lengths, for raw inputs, open column builders, -oracle cells and responses, compression scratch, and transport buffers, and keep -them distinct from process RSS. Bound queue bytes as well as -item counts, including spare slab capacity and per-chunk metadata for small -rows across many tables. Oracle cells own one allocation each; pack them into -shared buffers if they dominate - -Reserve conversion and compression workspace separately from queued output so -unfinished work cannot block its own completion. Specify acquisition and release -order before changing pools, including incompressible output and oversized rows - -Evaluate compressing raw transaction segments if open inputs dominate RSS; keep -spill available regardless of ratio. Defer encoding before commit until -measurements justify its unresolved-cell and rollback state - -## Acceptance - -| Concern | Required evidence | -|---|---| -| Load coverage | Compare cell and direct output across COPY, direct bootstrap, base backup, object store, pending transactions, deferred TOAST, and restart using existing load fixtures | -| TOAST | Compare cell and view detoast across inline, compressed, external, unchanged, and same-transaction values under every value mode | -| Prepared send | Decode prepared blocks through existing Native reader; verify none/LZ4/ZSTD, mismatched destinations, and retries after an intermediate Data block and after server acceptance before reply | -| Recovery | Restart with open raw transactions, spilled plans, and ambiguous INSERT outcomes; verify exact destination state and resume floor across required destinations | -| Memory progress | Exhaust transaction/output budgets with many tables, a large transaction, and stalled inserters; prove spill reaches COMMIT and compression finishes without circular waits | - -Measure narrow fixed rows, short strings, wide JSONB/text, TOAST-heavy updates, -mixed oracle tables, many small transactions, and a spilling large transaction -against `more-types` (`69f1700`) using `xact_spill`, `spool_read`, and -`bootstrap_pump` allocation reports, and source-to-destination with the -[benchmark driver](../bench/README.md). Run with unchanged worker counts, -batching limits, and codec settings. Record rows/s, source WAL bytes/s, CPU, -allocation count, peak/steady RSS, retained bytes by owner, spill volume, -`walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` -(stored includes allocation slack), durable lag, blocks per INSERT, and actual -ClickHouse part creation, with compressible and incompressible payloads and -both healthy and stalled destinations. Attribute savings separately to compact -transaction storage, direct encoding, and compression. Choose block size and default compression -from measured CPU/memory/part tradeoffs; do not claim gains from wire -compression alone diff --git a/plans/performance.md b/plans/performance.md index d433c134..abca4c74 100644 --- a/plans/performance.md +++ b/plans/performance.md @@ -47,6 +47,27 @@ replay can temporarily duplicate rows Use existing stage metrics to locate missing attribution before adding counters Check process uptime and cumulative series remain valid across phase changes +## Encoding and memory baseline + +Measure narrow fixed rows, short strings, wide JSONB/text, TOAST-heavy updates, +mixed oracle tables, many small transactions, and a spilling large transaction +against `more-types` (`69f1700`), before direct encoding, using `xact_spill`, +`spool_read`, and `bootstrap_pump` allocation reports, and source-to-destination +with [benchmark driver](../bench/README.md). Keep worker counts, batching +limits, and codec settings unchanged. Use compressible and incompressible +payloads, healthy and stalled destinations + +Record rows/s, source WAL bytes/s, CPU, allocation count, peak/steady RSS, spill +volume, durable lag, blocks per INSERT, and ClickHouse part creation. Record +retained bytes by owner from `walshadow_` metrics `xact_bytes_in_memory`, +`resident_payload_bytes`, `output_resident_bytes`, `batcher_open_bytes`, +`inserter_scratch_bytes`, and `resident_raw_bytes_total` over +`resident_stored_bytes_total`, then compare their +sum against RSS to size allocator and client overhead. Attribute savings +separately to compact transaction storage, direct encoding, and compression +Choose block size, seal workspace reserve, and default compression from measured +CPU, memory, and part tradeoffs; do not claim gains from wire compression alone + ## Candidate changes | Measured limit | Candidate | @@ -59,6 +80,9 @@ Check process uptime and cumulative series remain valid across phase changes | Per-record allocation dominates | Evaluate inline storage for small block-reference lists | | Catalog refresh work dominates | Narrow invalidation only after measuring recapture and cache cost | | TOAST fetch or materialization dominates | Compare ClickHouse and [shadow modes](../architecture/shadow-toast.md) | +| Oracle cells dominate output memory | Pack one column's cells into a shared buffer instead of one allocation each | +| Open transaction inputs dominate RSS | Compress raw transaction segments in memory; keep spill regardless of ratio | +| Commit-time encoding dominates latency | Encode before commit, after defining unresolved-value and rollback state | Bootstrap workers change ordering: assign acknowledgement sequence before parallel decode and account for concurrent deferred-TOAST writers. Preserve diff --git a/plans/tier2.md b/plans/tier2.md index 409cb14f..5153d553 100644 --- a/plans/tier2.md +++ b/plans/tier2.md @@ -30,8 +30,7 @@ Commit planning already reads ordinary WAL tuples through borrowed datums and writes encoded column chunks. Large plans spill those chunks; conversion workers combine local and oracle columns, then seal prepared Native blocks. Extend this path with container buffers and a borrowed element walk. Nested owned values -are not a prerequisite. See [direct encoding](native_encoding.md) for remaining -load and TOAST work +are not a prerequisite ## Dependencies need no change diff --git a/plans/verification.md b/plans/verification.md index d09abcf7..274ca414 100644 --- a/plans/verification.md +++ b/plans/verification.md @@ -38,6 +38,12 @@ Extend existing TOAST and type suites with: and MD5s after WAL replay. Test incompressible bodies, cap boundaries, explicit native JSON mapping, and restart. Do not infer native JSON semantics from automatic String mapping or row count +- Compressed external, unchanged, and same-transaction values under shadow and + disabled modes, matching ClickHouse-mode cases in + [TOAST harness](../tests/toast_e2e.rs) +- One fixture loaded by COPY, direct bootstrap, base backup, and object store, + with pending transactions, deferred TOAST, and restart; compare destination + rows across modes as well as against source For a fixture containing 8 KiB, 32 MiB, and 128 MiB TEXT rows, assert total length of `167780352` bytes after all inserts succeed. Compare every expected @@ -89,6 +95,8 @@ retry exhaustion, supervisor restart, and recovery after connectivity returns Cover a transaction spanning WAL segments, partial batch success, and later inserts completing before an earlier insert. Compare final rows and durable progress with uninterrupted execution. Verify missing retained WAL fails clearly +Add crash points while a spilled plan is written and while it replays, then +assert restart clears scratch and converges Prepared COMMIT and ABORT already dispatch through ordinary drain/discard logic, and open transactions constrain restart floor. Verify full daemon restart after diff --git a/src/backfill/backfill_staging.rs b/src/backfill/backfill_staging.rs index 45f1de02..56c0c88e 100644 --- a/src/backfill/backfill_staging.rs +++ b/src/backfill/backfill_staging.rs @@ -337,7 +337,7 @@ mod tests { let sql = "DROP TABLE IF EXISTS staging"; for retries in 0..=2 { let (config, server) = - crate::ch::test_support::retry_server(retries, sql, 0, false).await; + crate::ch::test_support::retry_server(retries, sql, 0, false, None).await; let mut session = StagingSession::connect(DestEmitter::new(Arc::new(config), None)) .await .unwrap(); @@ -349,7 +349,7 @@ mod tests { #[tokio::test] async fn exchange_ambiguity_does_not_retry() { let sql = "EXCHANGE TABLES staging AND live"; - let (config, server) = crate::ch::test_support::retry_server(2, sql, 0, false).await; + let (config, server) = crate::ch::test_support::retry_server(2, sql, 0, false, None).await; let mut session = StagingSession::connect(DestEmitter::new(Arc::new(config), None)) .await .unwrap(); diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 154796ee..62146820 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -613,6 +613,9 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M resident_seal_seconds_total: emitter_seconds(|s| &s.resident_seal_nanos), resident_raw_bytes_total: emitter(|s| &s.resident_raw_bytes), resident_stored_bytes_total: emitter(|s| &s.resident_stored_bytes), + output_resident_bytes: emitter(|s| &s.output_resident_bytes), + batcher_open_bytes: emitter(|s| &s.batcher_open_bytes), + inserter_scratch_bytes: emitter(|s| &s.inserter_scratch_bytes), oracle_resolve_seconds_total: emitter_seconds(|s| &s.oracle_resolve_nanos), process_cpu_seconds_total: proc_cpu, process_resident_memory_bytes: proc_rss, diff --git a/src/budget.rs b/src/budget.rs index a09723e2..f1c4e4d1 100644 --- a/src/budget.rs +++ b/src/budget.rs @@ -156,6 +156,8 @@ struct Inner { /// (mixed pool), the whole pool otherwise (leaf-only pool) leaf_max: usize, cur: AtomicU64, + /// Process-wide sum over pools of one kind, eg every tail's output pool + gauge: Option>, peak: AtomicU64, waits: AtomicU64, overshoots: AtomicU64, @@ -178,6 +180,15 @@ impl MemoryBudget { /// `leaf_reserve` bytes stay out of admission's reach so per-value /// leaf acquires under held admission permits always complete pub fn with_leaf_reserve(total: usize, leaf_reserve: usize) -> Self { + Self::build(total, leaf_reserve, None) + } + + /// [`Self::with_leaf_reserve`] also mirroring held bytes into `gauge` + pub fn gauged(total: usize, leaf_reserve: usize, gauge: Arc) -> Self { + Self::build(total, leaf_reserve, Some(gauge)) + } + + fn build(total: usize, leaf_reserve: usize, gauge: Option>) -> Self { let total = total.min(MAX_BYTES); let admission_max = total.saturating_sub(leaf_reserve); Self { @@ -194,6 +205,7 @@ impl MemoryBudget { leaf_reserve.min(total) }, cur: AtomicU64::new(0), + gauge, peak: AtomicU64::new(0), waits: AtomicU64::new(0), overshoots: AtomicU64::new(0), @@ -208,6 +220,11 @@ impl MemoryBudget { self.inner.total } + /// Bytes admission can hold, `total` minus leaf reserve + pub fn admission_max(&self) -> usize { + self.inner.admission_max + } + /// Share a single leaf acquire waits on: the reserve on a mixed pool, /// the whole pool on a leaf-only one. A holder staying inside it /// never withholds bytes admission needs @@ -369,6 +386,16 @@ impl Inner { fn meter(&self, bytes: u64) { let now = self.cur.fetch_add(bytes, Ordering::Relaxed) + bytes; self.peak.fetch_max(now, Ordering::Relaxed); + if let Some(g) = &self.gauge { + g.fetch_add(bytes, Ordering::Relaxed); + } + } + + fn unmeter(&self, bytes: u64) { + self.cur.fetch_sub(bytes, Ordering::Relaxed); + if let Some(g) = &self.gauge { + g.fetch_sub(bytes, Ordering::Relaxed); + } } /// Atomically reserve `want` if capacity is available and queues in @@ -574,16 +601,27 @@ impl MemoryPermit { freed |= BIG_FREE; } self.inner.release(freed); - self.inner - .cur - .fetch_sub(self.bytes - bytes, Ordering::Relaxed); + self.inner.unmeter(self.bytes - bytes); + self.bytes = bytes; + } + + /// Set held bytes to `bytes` once known: shrink, or account growth past + /// what was reserved as overshoot. Growth never waits, so an owner can + /// always finish output it already started + pub fn settle(&mut self, bytes: u64) { + if bytes <= self.bytes { + self.shrink(bytes); + return; + } + self.inner.overshoots.fetch_add(1, Ordering::Relaxed); + self.inner.meter(bytes - self.bytes); self.bytes = bytes; } } impl Drop for MemoryPermit { fn drop(&mut self) { - self.inner.cur.fetch_sub(self.bytes, Ordering::Relaxed); + self.inner.unmeter(self.bytes); let big = if self.big_leaf { BIG_FREE } else { 0 }; self.inner .release(word(self.units, self.admission_units) | big); @@ -948,6 +986,26 @@ mod tests { assert_eq!(b.resident_bytes(), 0); } + #[tokio::test(flavor = "current_thread")] + async fn settle_grows_unmetered_and_gauge_sums_pools() { + let gauge = Arc::new(AtomicU64::new(0)); + let a = MemoryBudget::gauged(64 << 10, 16 << 10, gauge.clone()); + let b = MemoryBudget::gauged(64 << 10, 0, gauge.clone()); + let mut p = a.admit(32 << 10).await; + let q = b.acquire(8 << 10).await; + assert_eq!(gauge.load(Ordering::Relaxed), 40 << 10); + // Growth past admission keeps admission units free for others + p.settle(48 << 10); + assert_eq!(a.overshoots_total(), 1); + assert_eq!(a.resident_bytes(), 48 << 10); + assert!(a.try_admit(16 << 10).is_some()); + p.settle(4 << 10); + assert_eq!(a.resident_bytes(), 4 << 10); + assert!(a.try_admit(44 << 10).is_some()); + drop((p, q)); + assert_eq!(gauge.load(Ordering::Relaxed), 0); + } + /// Allow only one large value above reserve at a time #[tokio::test(flavor = "current_thread")] async fn oversized_leaves_serialize() { diff --git a/src/ch/test_support.rs b/src/ch/test_support.rs index 195a31a6..931e7fbd 100644 --- a/src/ch/test_support.rs +++ b/src/ch/test_support.rs @@ -7,12 +7,14 @@ use tokio::task::JoinHandle; use crate::ch::CompressionChoice; use crate::emit::ch_emitter::{EmitterConfig, InstanceConfig, RetryConfig}; -/// Server expecting `sql` then `blocks` one-row Data blocks per attempt +/// Server expecting `sql` then `blocks` one-row Data blocks per attempt. +/// First attempt drops connection without reply, after `cut` blocks when set pub(crate) async fn retry_server( max_attempts: u32, sql: &str, blocks: usize, idle: bool, + cut: Option, ) -> (EmitterConfig, JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let config = EmitterConfig { @@ -45,11 +47,14 @@ pub(crate) async fn retry_server( const EMPTY_DATA: &[u8] = &[2, 0, 1, 0, 2, 0xff, 0xff, 0xff, 0xff, 0, 0, 0]; expected.push(0); expected.extend_from_slice(EMPTY_DATA); + // One UInt8 column x without custom serialization, one row 42 + const BLOCK: &[u8] = b"\x02\x00\x01\x00\x02\xff\xff\xff\xff\x00\x01\x01\x01x\x05UInt8\x00\x2a"; + let cut_at = cut.map(|n| { + assert!(n < blocks); + expected.len() + n * BLOCK.len() + }); for _ in 0..blocks { - // One UInt8 column x without custom serialization, one row 42 - expected.extend_from_slice( - b"\x02\x00\x01\x00\x02\xff\xff\xff\xff\x00\x01\x01\x01x\x05UInt8\x00\x2a", - ); + expected.extend_from_slice(BLOCK); } if blocks > 0 { expected.extend_from_slice(EMPTY_DATA); @@ -77,6 +82,12 @@ pub(crate) async fn retry_server( if idle && connection == 0 { continue; } + if let Some(at) = cut_at.filter(|_| connection == 0) { + let mut request = vec![0; at]; + stream.read_exact(&mut request).await.unwrap(); + assert_eq!(request, expected[..at]); + continue; + } let mut request = vec![0; expected.len()]; stream.read_exact(&mut request).await.unwrap(); assert_eq!(request, expected); diff --git a/src/crash_test.rs b/src/crash_test.rs index 1bb34334..7b5d3d40 100644 --- a/src/crash_test.rs +++ b/src/crash_test.rs @@ -14,6 +14,8 @@ pub enum Point { BeforeDdl, AfterAlter, AfterDdl, + /// One destination accepted rows, later destinations and ack pending + AfterInsert, AfterAck, AfterManifest, BootstrapHandoff, diff --git a/src/emit/ch_ddl.rs b/src/emit/ch_ddl.rs index 0b35acad..e5ac9846 100644 --- a/src/emit/ch_ddl.rs +++ b/src/emit/ch_ddl.rs @@ -1774,7 +1774,7 @@ mod tests { let sql = "CREATE DATABASE IF NOT EXISTS retry_test"; for (retries, idle) in [(0, false), (1, false), (2, false), (1, true)] { let (config, server) = - crate::ch::test_support::retry_server(retries, sql, 0, idle).await; + crate::ch::test_support::retry_server(retries, sql, 0, idle, None).await; let resolved = Arc::new(ResolvedConfig::default()); let ddl = DdlConfig::from_resolved( &resolved, diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 2ccc6101..8a6e0e87 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -2646,8 +2646,16 @@ impl EncodedChunk { self.rows } + /// Slab capacity plus chunk header and buffer slots, which dominate + /// for small rows spread across many tables pub fn allocated_bytes(&self) -> usize { - self.buffers.iter().map(ColumnBuf::allocated_bytes).sum() + std::mem::size_of::() + + self.buffers.capacity() * std::mem::size_of::() + + self + .buffers + .iter() + .map(ColumnBuf::allocated_bytes) + .sum::() } pub fn rel_oid(&self) -> u32 { @@ -3338,6 +3346,11 @@ crate::atomic_stats! { pub resident_stored_bytes, /// Resolver time serializing and compressing sealed blocks pub resident_seal_nanos, + /// Gauges: every tail's output pool (queued slabs, sealed blocks, + /// seal workspace), open batch slabs, inserter rebuild scratch + pub output_resident_bytes: Arc, + pub batcher_open_bytes, + pub inserter_scratch_bytes, } } diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs index c577f5a5..69a436ba 100644 --- a/src/emit/direct_parity.rs +++ b/src/emit/direct_parity.rs @@ -1013,3 +1013,151 @@ fn plan_views_match_cells_with_resolved_externals() { &[(7, text("same")), (8, text("same")), (9, text("other"))], ); } + +/// Inline short, long and compressed values beside an external value that +/// resolves, stays unchanged under its old pointer, or fills NULL as +/// disabled mode and null overflow do. Store, shadow and same-transaction +/// reads all arrive as resolved values, so patches must plan as inlined +/// values would for either target nullability +#[test] +fn plan_views_match_cells_across_value_forms() { + let mut r = rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, TEXTOID, -1, 'i', "text"), + attr(3, TEXTOID, -1, 'i', "text"), + ]); + r.replident = ReplIdent::Full { pk_attnums: None }; + let attrs = r.attributes.clone(); + let row = |inline: Vec, value_id: u32| { + let ext = external_id(value_id); + tuple( + &attrs, + 3, + &[ + Some(&1i32.to_le_bytes()[..]), + Some(&inline[..]), + Some(&ext[..]), + ], + ) + }; + let packed = || lz4(&b"x".repeat(64)); + for target in ["String", "Nullable(String)"] { + for filled in [false, true] { + let eight = if filled { + ColumnValue::Null + } else { + ColumnValue::Text("eight".into()) + }; + plan_both( + &r, + &["Int32", target, target], + &[ + (HeapOp::Insert, Some(row(short(b"a"), 7)), None), + ( + HeapOp::Update, + Some(row(long(b"bb"), 7)), + Some(row(short(b"a"), 7)), + ), + ( + HeapOp::Update, + Some(row(packed(), 8)), + Some(row(long(b"bb"), 7)), + ), + (HeapOp::Delete, None, Some(row(packed(), 8))), + ], + &[(7, ColumnValue::Text("seven".into())), (8, eight)], + ); + } + } +} + +/// Heap datum and COPY field of one value +type HeapAndCopy = (Vec, Vec); + +/// COPY binary and text fields encode as page-walk views of same values do +#[test] +fn copy_fields_match_heap_views() { + use crate::backfill::copy_backfill::{CopyRow, copy_slot}; + let rel = rel(vec![ + attr(1, INT4OID, 4, 'i', "int4"), + attr(2, INT8OID, 8, 'd', "int8"), + attr(3, FLOAT8OID, 8, 'd', "float8"), + attr(4, BOOLOID, 1, 'c', "bool"), + attr(5, DATEOID, 4, 'i', "date"), + attr(6, TEXTOID, -1, 'i', "text"), + attr(7, NUMERICOID, -1, 'i', "numeric"), + attr(8, NUMERICOID, -1, 'i', "numeric"), + ]); + let targets = [ + "Int32", + "Nullable(Int64)", + "Float64", + "Bool", + "Date32", + "Nullable(String)", + "String", + "Decimal(10, 2)", + ]; + let rows: Vec>> = vec![ + vec![ + Some((7i32.to_le_bytes().into(), 7i32.to_be_bytes().into())), + Some((i64::MIN.to_le_bytes().into(), i64::MIN.to_be_bytes().into())), + Some(( + (-0.5f64).to_le_bytes().into(), + (-0.5f64).to_be_bytes().into(), + )), + Some((vec![1], vec![1])), + Some((8000i32.to_le_bytes().into(), 8000i32.to_be_bytes().into())), + Some((lz4(&b"y".repeat(80)), b"y".repeat(80))), + Some((short(&numeric_short(false, 2, &[1, 5000])), b"1.50".into())), + Some((short(&numeric_short(true, 0, &[42])), b"-42".into())), + ], + vec![ + Some((0i32.to_le_bytes().into(), 0i32.to_be_bytes().into())), + None, + Some(( + f64::INFINITY.to_le_bytes().into(), + f64::INFINITY.to_be_bytes().into(), + )), + Some((vec![0], vec![0])), + Some(((-1i32).to_le_bytes().into(), (-1i32).to_be_bytes().into())), + None, + Some((short(&0xC000u16.to_le_bytes()), b"NaN".into())), + Some((short(&numeric_short(false, 0, &[])), b"0".into())), + ], + ]; + let mut views = TableEncoder::new(plan(&rel, &targets, &ColumnRules::default())).unwrap(); + let mut copy = TableEncoder::new(plan(&rel, &targets, &ColumnRules::default())).unwrap(); + for (i, row) in rows.iter().enumerate() { + let stamp = RowStamp { + source_lsn: 0x1000 + i as u64, + xid: 700, + commit_ts: 0, + op_code: OP_INSERT, + leaf: None, + }; + let heap: Vec<_> = row + .iter() + .map(|c| c.as_ref().map(|(h, _)| &h[..])) + .collect(); + let data = tuple(&rel.attributes, rel.attributes.len(), &heap); + let view = view_block_data(&data, &rel).unwrap(); + views.append_view(&rel, &view.datums, &[], &stamp).unwrap(); + let fields: Vec<_> = row + .iter() + .map(|c| c.as_ref().map(|(_, f)| &f[..])) + .collect(); + let fields = CopyRow::new(&fields); + copy.append_slots( + |src| copy_slot(&fields, &rel, src.index).map_err(crate::ch::EmitterError::Type), + &stamp, + ) + .unwrap(); + } + let wire = |enc: &mut TableEncoder| { + let (bufs, rows) = enc.take_block().unwrap(); + let wire: Vec<_> = bufs.iter().map(|b| format!("{:?}", b.wire())).collect(); + (rows, wire) + }; + assert_eq!(wire(&mut views), wire(&mut copy)); +} diff --git a/src/emit/pipeline/batcher.rs b/src/emit/pipeline/batcher.rs index 95825b07..4ebc641a 100644 --- a/src/emit/pipeline/batcher.rs +++ b/src/emit/pipeline/batcher.rs @@ -316,17 +316,18 @@ pub(crate) fn spawn( mut msg_rx: mpsc::Receiver, out: async_channel::Sender, cfg: BatcherConfig, - alloc: Allocator, + budget: crate::budget::MemoryBudget, fatal: Fatal, stats: Arc, mut config_rx: Option>>, ) -> JoinHandle<()> { tokio::spawn(async move { + let alloc = Allocator::global(&mimalloc::MiMalloc); let out = BatchOutput { sender: out, - budget: crate::budget::MemoryBudget::new(cfg.byte_budget.max(1)), + batch_limit: (budget.admission_max() / cfg.inserters.max(1)).max(1), + budget, partial_bytes: AtomicUsize::new(0), - batch_limit: (cfg.byte_budget / cfg.inserters.max(1)).max(1), }; let mut tables = Tables::default(); let mut epoch: u64 = 0; @@ -508,16 +509,18 @@ async fn appended( .iter() .map(ColumnBuf::allocated_bytes) .sum::(); - ctx.out - .partial_bytes - .fetch_add(bytes - t.allocated_bytes, Ordering::Relaxed); + let grown = bytes - t.allocated_bytes; + ctx.out.partial_bytes.fetch_add(grown, Ordering::Relaxed); + ctx.stats + .batcher_open_bytes + .fetch_add(grown as u64, Ordering::Relaxed); t.allocated_bytes = bytes; let batch_limit = ctx.cfg.byte_budget.min(ctx.out.batch_limit); if t.rows() >= ctx.cfg.row_budget || bytes >= batch_limit { emit_batch(t, ctx.out, ctx.stats).await?; } if ctx.out.partial_bytes.load(Ordering::Relaxed) - >= ctx.cfg.byte_budget.min(ctx.out.budget.total()) + >= ctx.cfg.byte_budget.min(ctx.out.budget.admission_max()) { for table in &mut tables.open { emit_batch(table, ctx.out, ctx.stats).await?; @@ -531,8 +534,11 @@ async fn appended( /// is on the inserter channel (the inserter bumps `inserter_batches_in` once it /// drains). async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> Result<(), String> { - out.partial_bytes - .fetch_sub(std::mem::take(&mut t.allocated_bytes), Ordering::Relaxed); + let open = std::mem::take(&mut t.allocated_bytes); + out.partial_bytes.fetch_sub(open, Ordering::Relaxed); + stats + .batcher_open_bytes + .fetch_sub(open as u64, Ordering::Relaxed); t.seal_block()?; t.deadline = None; let blocks = std::mem::take(&mut t.sealed); @@ -547,8 +553,13 @@ async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> R } let per_seq = std::mem::take(&mut t.seq_counts); let permits = std::mem::take(&mut t.slice_permits); - // Separate pool avoids waiting on decoded payload held by this batch - let encoded = out.budget.acquire(bytes).await; + // Separate pool avoids waiting on decoded payload held by this batch. + // Oversized batch waits for whole compartment, then holds rest unmetered + let mut encoded = out + .budget + .admit(bytes.min(out.budget.admission_max())) + .await; + encoded.settle(bytes as u64); let batch = InsertBatch { meta: t.meta.clone(), blocks, @@ -745,7 +756,7 @@ mod tests { byte_budget: 1 << 30, flush_timeout: Duration::from_secs(3600), }, - Allocator::global(&mimalloc::MiMalloc), + crate::budget::MemoryBudget::new(1 << 30), fatal.clone(), Arc::new(EmitterStats::default()), None, @@ -790,7 +801,7 @@ mod tests { byte_budget: 1 << 30, flush_timeout: Duration::from_secs(3600), }, - Allocator::global(&mimalloc::MiMalloc), + crate::budget::MemoryBudget::new(1 << 30), fatal.clone(), Arc::new(EmitterStats::default()), None, @@ -854,7 +865,7 @@ mod tests { byte_budget: 1 << 30, flush_timeout: Duration::from_secs(3600), }, - Allocator::global(&mimalloc::MiMalloc), + crate::budget::MemoryBudget::new(1 << 30), fatal.clone(), Arc::new(EmitterStats::default()), None, @@ -920,7 +931,7 @@ mod tests { // Huge deadline: only FlushAll, not the timer, can seal flush_timeout: Duration::from_secs(3600), }, - Allocator::global(&mimalloc::MiMalloc), + crate::budget::MemoryBudget::new(1 << 30), fatal.clone(), Arc::new(EmitterStats::default()), None, @@ -958,7 +969,7 @@ mod tests { msg_rx, batches_tx, cfg, - Allocator::global(&mimalloc::MiMalloc), + crate::budget::MemoryBudget::new(cfg.byte_budget.max(1)), fatal.clone(), Arc::new(EmitterStats::default()), config_rx, diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index d2c85f1e..ce32ee90 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -68,7 +68,13 @@ impl Inserter { let block = { let mut scratch = scratch.lock().expect("inserter scratch poisoned"); - sealed.open(alloc, &mut scratch)? + let held = scratch.capacity(); + let block = sealed.open(alloc, &mut scratch); + stats.inserter_scratch_bytes.fetch_add( + (scratch.capacity() - held) as u64, + Ordering::Relaxed, + ); + block? }; let bb = builder_of(&block)?; stats.inserter_rebuilds.fetch_add(1, Ordering::Relaxed); @@ -112,6 +118,8 @@ impl Inserter { for query in &meta.inserts { self.client.select(query.instance.as_deref())?; durable = Some(self.send_with_retry(&query.sql, blocks).await?); + #[cfg(feature = "test-support")] + crate::crash_test::hit(crate::crash_test::Point::AfterInsert, u64::MAX); } durable.ok_or_else(|| EmitterError::Config("batch has no INSERT target".into())) } @@ -148,6 +156,15 @@ impl Inserter { } } +impl Drop for Inserter { + fn drop(&mut self) { + let held = self.scratch.get_mut().map_or(0, |s| s.capacity()); + self.stats + .inserter_scratch_bytes + .fetch_sub(held as u64, Ordering::Relaxed); + } +} + /// Connect `n` inserters and spawn drain loops pub(crate) async fn spawn_pool( n: usize, @@ -192,20 +209,23 @@ mod tests { } /// Every attempt resends every block of the INSERT through failed and - /// idle reconnects; blocks prepared for other compression than the - /// uncompressed server connection rebuild + /// idle reconnects, and after a drop between Data blocks; blocks + /// prepared for other compression than the uncompressed server + /// connection rebuild #[tokio::test] async fn insert_retry_preserves_payload_through_failed_reconnect() { let sql = "INSERT INTO retry_test (x) VALUES"; - for (retries, idle, blocks, codec) in [ - (0, false, 1, CompressionChoice::Lz4), - (1, false, 1, CompressionChoice::None), - (2, false, 1, CompressionChoice::Zstd), - (2, false, 3, CompressionChoice::Lz4), - (1, true, 2, CompressionChoice::Zstd), + for (retries, idle, blocks, codec, cut) in [ + (0, false, 1, CompressionChoice::Lz4, None), + (1, false, 1, CompressionChoice::None, None), + (2, false, 1, CompressionChoice::Zstd, None), + (2, false, 3, CompressionChoice::Lz4, None), + (1, true, 2, CompressionChoice::Zstd, None), + (2, false, 3, CompressionChoice::None, Some(1)), + (2, false, 3, CompressionChoice::Zstd, Some(2)), ] { let (config, server) = - crate::ch::test_support::retry_server(retries, sql, blocks, idle).await; + crate::ch::test_support::retry_server(retries, sql, blocks, idle, cut).await; let dest = DestEmitter::new(Arc::new(config), None); let client = ChConn::connect(dest).await.unwrap(); let (ack, collector) = ack::spawn(Arc::default(), Fatal::new()); @@ -235,8 +255,13 @@ mod tests { codec != CompressionChoice::None, "{codec:?}" ); + assert_eq!( + stats.inserter_scratch_bytes.load(Ordering::Relaxed), + inserter.scratch.lock().unwrap().capacity() as u64 + ); server.await.unwrap(); drop(inserter); + assert_eq!(stats.inserter_scratch_bytes.load(Ordering::Relaxed), 0); collector.await.unwrap(); } } diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index 26a34f26..6a5433c5 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -12,6 +12,18 @@ //! //! A batch with no oracle column crosses unchanged, without touching the //! bridge. +//! +//! ## Output pool +//! +//! Each tail owns one pool, separate from the global admission pool its +//! batches already hold. Batcher admits slab bytes per batch; permit rides +//! batch through sealing until insert ack. Resolvers draw seal workspace +//! from leaf reserve, which queued batches never hold, so sealing finishes +//! however full the queue is. Order per block: take workspace, resolve, +//! seal, drop slabs and workspace. Output past admitted slab bytes, such as +//! frame overhead on incompressible data, settles as unmetered overshoot +//! instead of waiting. Blocks whose workspace exceeds reserve seal one at a +//! time. Inserters take nothing from pool, so acks always release it use std::collections::hash_map::Entry; use std::sync::Arc; @@ -22,15 +34,18 @@ use clickhouse_c::{Allocator, BlockBuilder, ColumnBuilder, TypeAst}; use tokio::sync::watch; use tokio::task::JoinHandle; +use crate::budget::MemoryBudget; use crate::ch::{CompressionChoice, EmitterError}; use crate::config::ResolvedConfig; use crate::emit::ch_emitter::{ ColumnBuf, EmitterStats, RetryConfig, build_leaf, build_root, literal_column, }; use crate::emit::pipeline::Fatal; -use crate::emit::pipeline::batcher::{BatchMeta, BlockSlabs, InsertBatch}; +use crate::emit::pipeline::batcher::{BatchMeta, BlockSlabs, InsertBatch, NATIVE_BLOCK_BYTES}; use crate::emit::pipeline::sealed::SealedBlock; -use crate::ops::oracle::{Oracle, OracleBlock, OracleError, OracleRequestColumn}; +use crate::ops::oracle::{ + ORACLE_BATCH_SEAL_BYTES, Oracle, OracleBlock, OracleError, OracleRequestColumn, +}; use crate::schema::TableKey; use foldhash::HashMap; @@ -60,6 +75,43 @@ pub(crate) struct ResolverOptions { pub stats: Arc, pub fatal: Fatal, pub config_rx: Option>>, + /// Tail's output pool, see [`output_pool`] + pub budget: MemoryBudget, +} + +/// clickhouse-c frame stage plus one compressed frame, +/// `CHC_COMPRESS_MAX_CHUNK` in `clickhouse-compression.h` +const SEAL_SCRATCH: usize = 160 << 10; + +/// Pool admitting `queued` bytes of batches, plus seal workspace for each +/// of `resolvers`: largest block's output, and with `oracle` its request +/// copy and decoded response +pub(crate) fn output_pool( + queued: usize, + resolvers: usize, + oracle: bool, + gauge: Arc, +) -> MemoryBudget { + let oracle = if oracle { + 2 * ORACLE_BATCH_SEAL_BYTES + } else { + 0 + }; + let reserve = resolvers.max(1) * (SEAL_SCRATCH + NATIVE_BLOCK_BYTES + oracle); + MemoryBudget::gauged(queued.max(1) + reserve, reserve, gauge) +} + +/// Bytes sealing `block` holds beside its slabs: compressed output before +/// slabs drop, frame scratch, oracle request and response +fn workspace_bytes(block: &BlockSlabs) -> usize { + let slabs: usize = block.buffers.iter().map(ColumnBuf::allocated_bytes).sum(); + let oracle: usize = block + .buffers + .iter() + .filter(|b| matches!(b, ColumnBuf::Oracle(_))) + .map(ColumnBuf::allocated_bytes) + .sum(); + slabs + slabs / 128 + SEAL_SCRATCH + 2 * oracle } /// Spawn `n` resolvers over the shared batch queue. Drops the sender it was @@ -166,6 +218,7 @@ async fn run( stats, fatal, config_rx, + budget, } = opts; let alloc = Allocator::global(&mimalloc::MiMalloc); let mut sealer = match Sealer::new(codec) { @@ -190,6 +243,7 @@ async fn run( let mut sealed = Vec::with_capacity(blocks.len()); for mut block in blocks { resolve_local_columns(&mut block, &stats); + let _workspace = budget.acquire(workspace_bytes(&block)).await; let started = std::time::Instant::now(); let resolved = match resolve_oracle_with_retry( &oracle, alloc, &meta, &block, &retry, &stats, @@ -227,7 +281,7 @@ async fn run( .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed); } // Slabs and oracle output already dropped - encoded.shrink(sealed.iter().map(|b| b.allocated_bytes() as u64).sum()); + encoded.settle(sealed.iter().map(|b| b.allocated_bytes() as u64).sum()); let ready = ResolvedBatch { meta, blocks: sealed, @@ -418,7 +472,7 @@ pub(crate) mod tests { inserters: 1, flush_timeout: Duration::from_secs(3600), }, - Allocator::global(&mimalloc::MiMalloc), + crate::budget::MemoryBudget::new(1 << 30), fatal.clone(), Arc::new(EmitterStats::default()), None, @@ -490,6 +544,49 @@ pub(crate) mod tests { assert_eq!(stats.oracle_local_columns.load(Ordering::Relaxed), 1); } + /// Queued batches holding all admitted bytes leave sealing its + /// workspace, so a stalled inserter cannot block its own queue + #[tokio::test] + async fn sealing_finishes_while_queued_output_fills_admission() { + let gauge = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let budget = output_pool(4 << 10, 1, false, gauge.clone()); + let queued = budget.admit(budget.admission_max()).await; + let batch = seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await; + let (batch_tx, batch_rx) = async_channel::bounded(1); + let (resolved_tx, resolved_rx) = async_channel::bounded(1); + let fatal = Fatal::new(); + let task = tokio::spawn(run( + batch_rx, + resolved_tx, + ResolverOptions { + oracle: None, + retry: RetryConfig::default(), + codec: CompressionChoice::Lz4, + stats: Arc::default(), + fatal: fatal.clone(), + config_rx: None, + budget: budget.clone(), + }, + )); + batch_tx.send(batch).await.unwrap(); + let resolved = tokio::time::timeout(Duration::from_secs(5), resolved_rx.recv()) + .await + .expect("sealed under full admission") + .unwrap(); + assert_eq!(resolved.blocks.len(), 1); + assert!(budget.peak_bytes() > queued.bytes(), "workspace taken"); + assert_eq!( + budget.resident_bytes(), + queued.bytes(), + "workspace returned" + ); + drop(batch_tx); + task.await.unwrap(); + assert!(fatal.message().is_none()); + drop(queued); + assert_eq!(gauge.load(Ordering::Relaxed), 0); + } + /// A cell PG must convert keeps the whole column in the request, even /// when its neighbours are rendered #[tokio::test] diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index 1989521b..fac415bf 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -13,7 +13,6 @@ use std::sync::Arc; -use clickhouse_c::Allocator; use tokio::sync::{mpsc, oneshot, watch}; use tokio_util::task::AbortOnDropHandle; @@ -264,15 +263,13 @@ pub fn spawn_discard( let (ack, collector) = ack::spawn(Arc::new(Monotone::default()), fatal.clone()); let (msg_tx, msg_rx) = mpsc::channel::(256); let (batches_tx, batches_rx) = async_channel::bounded::(4); - let batcher = batcher::spawn( - msg_rx, - batches_tx, - cfg, - Allocator::global(&mimalloc::MiMalloc), - fatal, - stats, - None, + let budget = resolver::output_pool( + cfg.byte_budget, + 1, + false, + stats.output_resident_bytes.clone(), ); + let batcher = batcher::spawn(msg_rx, batches_tx, cfg, budget, fatal, stats, None); let discard_ack = ack.clone(); let discard = tokio::spawn(async move { while let Ok(batch) = batches_rx.recv().await { @@ -328,8 +325,15 @@ pub async fn spawn_with_config( // As wide as the bridge: a narrower pool leaves shadow workers idle, a // wider one only queues on their sockets + let width = oracle.as_ref().map_or(1, |o| o.concurrency()); + let budget = resolver::output_pool( + emitter.byte_budget, + width, + oracle.is_some(), + stats.output_resident_bytes.clone(), + ); let resolvers = resolver::spawn_pool( - oracle.as_ref().map_or(1, |o| o.concurrency()), + width, batches_rx, resolved_tx, resolver::ResolverOptions { @@ -339,6 +343,7 @@ pub async fn spawn_with_config( stats: stats.clone(), fatal: fatal.clone(), config_rx: config_rx.clone(), + budget: budget.clone(), }, ); @@ -357,7 +362,7 @@ pub async fn spawn_with_config( inserters: n, flush_timeout, }, - Allocator::global(&mimalloc::MiMalloc), + budget, fatal, stats.clone(), config_rx, diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index d976ff76..6a9fea32 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -227,7 +227,7 @@ snapshot! { lsn emitter_ack_lsn: Pos = "Contiguous-done watermark from the insert pipeline.", lsn floor_lsn: Pos = "Durable resume floor: restart resumes here and pruners cut to it.", gauge xact_active: u64 = "Active transactions buffered in memory or on spill.", - gauge xact_bytes_in_memory: u64 = "Bytes held in memory across all buffered xacts.", + gauge xact_bytes_in_memory: u64 = "Allocated bytes held in memory across all buffered xacts.", gauge spill_xacts_active: u64 = "Xacts with at least one entry currently in their spill file.", gauge spill_bytes_active: u64 = "Bytes currently held across all active xact spill files.", gauge drain_resident_bytes: u64 = @@ -426,6 +426,12 @@ snapshot! { "Serialized Native block bytes sealed for insertion, before resident compression.", counter resident_stored_bytes_total: u64 = "Bytes allocated for those sealed blocks while awaiting insertion, including growth slack.", + gauge output_resident_bytes: u64 = + "Bytes held in insert tail output pools: queued column slabs, sealed blocks awaiting acknowledgement, and seal workspace.", + gauge batcher_open_bytes: u64 = + "Allocated column slab bytes in batches still open for more rows.", + gauge inserter_scratch_bytes: u64 = + "Allocated inserter buffers for decoding sealed blocks a connection cannot send as prepared.", counter oracle_resolve_seconds_total: f64 = "Cumulative resolver time inside the oracle round trip. Overlaps inserter_ch_seconds_total, so the larger of the two is the tail's limiter.", counter process_cpu_seconds_total: f64 = diff --git a/src/xact/spill.rs b/src/xact/spill.rs index 8d41a01e..1a5c3483 100644 --- a/src/xact/spill.rs +++ b/src/xact/spill.rs @@ -429,11 +429,12 @@ impl RawRecord { pub fn approx_bytes(&self) -> usize { std::mem::size_of::() - + self.main_data.len() + + self.main_data.capacity() + + self.blocks.capacity() * std::mem::size_of::() + self .blocks .iter() - .map(|b| std::mem::size_of::() + b.image.len() + b.data.len()) + .map(|b| b.image.capacity() + b.data.capacity()) .sum::() } } diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index 6ec71d7f..6d63f86c 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -548,16 +548,17 @@ impl StashMark { } } -/// Approximate byte cost for in-memory accounting; estimate, not exact -/// heap allocation. Good enough for the eviction threshold +/// Allocated bytes of an entry and its buffer slot, excluding allocator +/// overhead and spare slot capacity fn approximate_size(entry: &SpillEntry) -> usize { - match entry { - SpillEntry::Truncate(_) => std::mem::size_of::(), - SpillEntry::Chunk(c) => std::mem::size_of::() + c.chunk_data.len(), - SpillEntry::ToastDelete(_) => std::mem::size_of::(), - SpillEntry::Raw(r) => r.approx_bytes(), - SpillEntry::Input(i) => i.approx_bytes(), - } + std::mem::size_of::() + + match entry { + SpillEntry::Truncate(_) => std::mem::size_of::(), + SpillEntry::Chunk(c) => std::mem::size_of::() + c.chunk_data.len(), + SpillEntry::ToastDelete(_) => 0, + SpillEntry::Raw(r) => r.approx_bytes(), + SpillEntry::Input(i) => i.approx_bytes(), + } } #[derive(Debug, Eq, Ord, PartialEq, PartialOrd)] diff --git a/tests/control_plane_e2e.rs b/tests/control_plane_e2e.rs index d3d6d439..ccf0cfcc 100644 --- a/tests/control_plane_e2e.rs +++ b/tests/control_plane_e2e.rs @@ -3191,6 +3191,99 @@ async fn check_ddl_crash(point: Point, applied_columns: u64) { } } +/// Crash after main destination accepted an INSERT, before its tee did or +/// its ack: restart replays rows to both, and `_lsn` dedup keeps each exact +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn insert_crash_between_destinations_before_ack() { + use std::os::unix::process::ExitStatusExt; + use walshadow::manifest::Manifest; + + if !gated() { + return; + } + let mut h = Harness::up(&fx::Ports::alloc()) + .await + .expect("bring up harness"); + let result = async { + h.wait_ready(Duration::from_secs(60)).await?; + h.ctl_body( + &["apply"], + "[table.demo.crash_t]\nreplicate = true\ntee = [{ table = \"crash_copy\" }]", + )?; + h.ctl(&["reload"])?; + h.psql( + "CREATE TABLE demo.crash_t (id bigint PRIMARY KEY, v text); \ + INSERT INTO demo.crash_t VALUES (1, 'before')", + )?; + let rows = + |table: &str| format!("SELECT id, v FROM demo.{table} FINAL ORDER BY id FORMAT TSV"); + for table in ["demo_crash_t", "crash_copy"] { + h.wait_ch(&rows(table), "1\tbefore", Duration::from_secs(60)) + .await?; + } + let arm = h.stderr_path.with_extension("arm"); + fs::write(&arm, format!("{:?}", Point::AfterInsert))?; + // Catalog change arms capture; next INSERT after it crashes + h.psql("ALTER TABLE demo.crash_t ADD COLUMN a bigint")?; + h.psql("INSERT INTO demo.crash_t VALUES (2, 'after', 8)")?; + let deadline = Instant::now() + Duration::from_secs(60); + while h.alive() { + ensure!(Instant::now() < deadline, "crash point never fired"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let status = h.child.take().context("daemon missing")?.wait()?; + ensure!( + status.signal() == Some(libc::SIGKILL), + "unexpected exit: {status}" + ); + let landed = |h: &Harness, table: &str| -> Result { + h.ch_get(&format!("SELECT count() FROM demo.{table} WHERE id = 2")) + }; + ensure!( + landed(&h, "demo_crash_t")? == "1", + "main INSERT did not land" + ); + ensure!( + landed(&h, "crash_copy")? == "0", + "tee INSERT ran before crash" + ); + let spill = h.tmp.path().join("spill"); + let manifest: Manifest = toml::from_str(&fs::read_to_string(spill.join("manifest.toml"))?)?; + let landed_lsn: u64 = h + .ch_get("SELECT max(_lsn) FROM demo.demo_crash_t WHERE id = 2")? + .parse()?; + ensure!( + manifest.lsn.emitter_ack.get() < landed_lsn, + "ack reached manifest before crash" + ); + // Merges would collapse replayed duplicate + for table in ["demo_crash_t", "crash_copy"] { + h.ch_get(&format!("SYSTEM STOP MERGES demo.{table}"))?; + } + fs::remove_file(arm)?; + h.start_daemon(Duration::from_secs(120)).await?; + h.psql("INSERT INTO demo.crash_t VALUES (3, 'live', 9)")?; + for table in ["demo_crash_t", "crash_copy"] { + h.wait_ch( + &format!("SELECT id, v, a FROM demo.{table} FINAL ORDER BY id FORMAT TSV"), + "1\tbefore\t\\N\n2\tafter\t8\n3\tlive\t9", + Duration::from_secs(60), + ) + .await?; + } + ensure!( + (landed(&h, "demo_crash_t")?, landed(&h, "crash_copy")?) == ("2".into(), "1".into()), + "restart did not replay unacknowledged INSERT to every destination" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + let stderr = h.teardown(); + if let Err(e) = result { + panic!("{e:#}\n--- daemon stderr ---\n{stderr}"); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn ddl_crash_before_descriptor_append() { check_ddl_crash(Point::BeforeDescriptor, 0).await; diff --git a/tests/emitter_budget_flush.rs b/tests/emitter_budget_flush.rs index 2a966091..88512981 100644 --- a/tests/emitter_budget_flush.rs +++ b/tests/emitter_budget_flush.rs @@ -229,3 +229,165 @@ async fn budget_trips_seal_complete_inserts() { .expect("ch distinct"); assert_eq!(distinct, N.to_string()); } + +/// Forward to `upstream`; client bytes wait while `gate` reads false, so +/// every INSERT stalls mid-request until it opens +async fn stall_proxy(upstream: u16, gate: tokio::sync::watch::Receiver) -> u16 { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((client, _)) = listener.accept().await { + let server = tokio::net::TcpStream::connect(("127.0.0.1", upstream)) + .await + .unwrap(); + let (mut client_rx, mut client_tx) = client.into_split(); + let (mut server_rx, mut server_tx) = server.into_split(); + let mut gate = gate.clone(); + tokio::spawn(async move { + let mut buf = vec![0; 64 << 10]; + while let Ok(n @ 1..) = client_rx.read(&mut buf).await { + if gate.wait_for(|open| *open).await.is_err() + || server_tx.write_all(&buf[..n]).await.is_err() + { + break; + } + } + }); + tokio::spawn(async move { + let _ = tokio::io::copy(&mut server_rx, &mut client_tx).await; + }); + } + }); + port +} + +/// Rows of many small commits spread over many tables, sent while every +/// inserter stalls, exceed byte budget: producer must block, then all +/// rows land and output memory returns once inserts resume +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stalled_inserters_backpressure_many_tables_then_drain() { + if !fx::tools::clickhouse_available() { + return; + } + const TABLES: usize = 16; + const ROWS: usize = 1_280; + + let slot = fx::Ports::alloc(); + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + ch.query("CREATE DATABASE IF NOT EXISTS walshadow_test") + .expect("create db"); + for t in 0..TABLES { + ch.query(&format!( + "CREATE TABLE walshadow_test.foo_{t} (id Int32, val Nullable(String), \ + _lsn UInt64, _xid UInt32, _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool) \ + ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id" + )) + .expect("create dest table"); + } + + let (gate_tx, gate_rx) = tokio::sync::watch::channel(true); + let port = stall_proxy(slot.ch_tcp, gate_rx).await; + let cfg = EmitterConfig { + conn: InstanceConfig { + host: "127.0.0.1".into(), + port, + database: "walshadow_test".into(), + compression: CompressionChoice::Lz4, + ..Default::default() + }, + row_budget: 16, + byte_budget: 16 << 10, + ..Default::default() + }; + let stats = Arc::new(EmitterStats::default()); + let emitter_ack = Arc::new(Monotone::::default()); + let fatal = Fatal::new(); + let (msg_tx, ack, tail_parts) = tail::spawn( + walshadow::config::DestEmitter::new(Arc::new(cfg), None), + 2, + stats.clone(), + emitter_ack.clone(), + fatal.clone(), + ) + .await + .expect("spawn tail"); + gate_tx.send(false).unwrap(); + + let tables: Vec<_> = (0..TABLES) + .map(|t| { + let mut rel = (*rel_descriptor()).clone(); + rel.rfn.rel_node += t as u32; + rel.oid += t as u32; + rel.rel_name = RelName::new("public", &format!("foo_{t}")); + let mut map = (*mapping()).clone(); + map.target = TableTarget::new("walshadow_test", &format!("foo_{t}")); + let route = walshadow::emit::route::RouteSnapshot::freeze( + Arc::new(map), + Arc::default(), + Default::default(), + ); + (Arc::new(rel), route) + }) + .collect(); + let commit = |i: usize| 0x10_0000 + 0x100 * i as u64; + let producer = { + let (msg_tx, ack) = (msg_tx.clone(), ack.clone()); + tokio::spawn(async move { + let mut seqs = ack.seqs(0); + for i in 0..ROWS { + let (rel, route) = &tables[i % TABLES]; + let seq = seqs.open(commit(i), Publish::Commit); + let mut row = tuple(i as i32, commit(i) - 1, commit(i)); + row.decoded.rfn = rel.rfn; + let segment = + PlacedSegment::of_rows(seq.seq(), rel, route, &[row]).expect("encode row"); + msg_tx + .send(BatcherMsg::Segments(vec![segment])) + .await + .expect("route row"); + seqs.place(seq, 1); + } + }) + }; + + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !producer.is_finished(), + "producer blocks behind stalled inserts" + ); + assert_eq!( + emitter_ack.get(), + Pos::new(0), + "nothing durable while stalled" + ); + assert!(stats.output_resident_bytes.load(Ordering::Relaxed) > 0); + + gate_tx.send(true).unwrap(); + producer.await.unwrap(); + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + msg_tx + .send(BatcherMsg::FlushAll(reply_tx)) + .await + .expect("send flush"); + reply_rx.await.expect("flush ack"); + ack.wait_through(ROWS as u64) + .await + .expect("ack collector alive"); + tail_parts.close(msg_tx, ack).await; + assert!(fatal.message().is_none(), "no fatal: {:?}", fatal.message()); + assert_eq!(emitter_ack.get(), Pos::new(commit(ROWS - 1))); + for gauge in [&stats.batcher_open_bytes, &*stats.output_resident_bytes] { + assert_eq!(gauge.load(Ordering::Relaxed), 0); + } + + for t in 0..TABLES { + let count = ch + .query(&format!( + "SELECT count() FROM walshadow_test.foo_{t} FINAL WHERE _is_deleted = 0" + )) + .expect("ch count"); + assert_eq!(count, (ROWS / TABLES).to_string(), "foo_{t}"); + } +} diff --git a/tests/kill_restart.rs b/tests/kill_restart.rs index 1dac9f4f..da9120b1 100644 --- a/tests/kill_restart.rs +++ b/tests/kill_restart.rs @@ -62,6 +62,9 @@ enum Strategy { const RUNS_PER_STRATEGY: u32 = 5; +/// Below one large xact, so it spills while open +const XACT_BUFFER_MAX: usize = 1 << 20; + /// Splitmix-style LCG; deterministic, no dep cost. fn next_seeded(state: &mut u64) -> u64 { *state = state @@ -178,6 +181,8 @@ impl DaemonFlags { "0".into(), "--ch-config".into(), self.ch_config.to_string_lossy().into_owned(), + "--xact-buffer-max".into(), + XACT_BUFFER_MAX.to_string(), "--bootstrap-shadow-data-dir".into(), self.shadow_data.to_string_lossy().into_owned(), "--keep-shadow-running".into(), @@ -249,13 +254,14 @@ async fn small_insert_loop( } } -/// Run one large `BEGIN; INSERT × 10000; COMMIT` xact. Sized to spill -/// to disk via `XactBuffer`'s largest-first eviction (each row carries -/// a 200-byte payload; 10k rows × ~250 bytes ≈ 2.5 MiB). +/// Run one large `BEGIN; INSERT × 10000; COMMIT` xact, open two seconds +/// after its rows. 10k rows × ~250 bytes ≈ 2.5 MiB spills past +/// [`XACT_BUFFER_MAX`] via `XactBuffer`'s largest-first eviction async fn large_xact(source_sock: PathBuf, start_id: i64) { let sql = format!( "BEGIN; \ INSERT INTO kr.t SELECT g::int4, repeat('x', 200) FROM generate_series({}, {}) g; \ + SELECT pg_sleep(2); \ COMMIT", start_id, start_id + 9999, @@ -299,22 +305,19 @@ async fn wait_for_source_rows(source: &Shadow, target: i64, deadline: Duration) bail!("source never reached {target} rows in {deadline:?}"); } -/// Poll the metrics endpoint until `walshadow_xacts_committed_total` -/// crosses zero. Used by the `PostCommit` strategy to fire immediately -/// after the first commit drain returns (simpler shape per plan -/// §"Risks"). -async fn wait_for_first_commit(metrics_addr: SocketAddr, deadline: Duration) -> Result<()> { +/// Poll the metrics endpoint until counter `name` crosses zero +async fn wait_for_metric(metrics_addr: SocketAddr, name: &str, deadline: Duration) -> Result<()> { let start = Instant::now(); while start.elapsed() < deadline { if let Ok(body) = fx::http_get(metrics_addr, "/metrics") - && let Some(v) = fx::parse_metric(&body, "walshadow_xacts_committed_total") + && let Some(v) = fx::parse_metric(&body, name) && v > 0 { return Ok(()); } tokio::time::sleep(Duration::from_millis(20)).await; } - bail!("walshadow_xacts_committed_total never crossed 0 in {deadline:?}"); + bail!("{name} never crossed 0 in {deadline:?}"); } /// Poll source's `pg_current_wal_lsn` + daemon's @@ -522,9 +525,8 @@ async fn run_cycle( stop.clone(), next_id.clone(), )); - // Strategy 2's parallel large xact runs alongside the small loop; - // its window overlaps the kill_delay naturally since the xact takes - // ~hundreds of ms to commit. + // Strategy 2's parallel large xact runs alongside the small loop and + // stays open past the kill let large_handle = if matches!(strategy, Strategy::MidXact) { let large_start = next_id.fetch_add(10_000, Ordering::Relaxed); Some(tokio::spawn(large_xact(source_sock.clone(), large_start))) @@ -537,15 +539,30 @@ async fn run_cycle( .await .context("source never reached 100 rows")?; - // 4. Strategy-specific kill trigger. + // 4. Strategy-specific kill trigger. MidXact kills while large xact + // sits spilled and uncommitted match strategy { - Strategy::MidSegment | Strategy::MidXact => { + Strategy::MidSegment => { tokio::time::sleep(Duration::from_millis(kill_delay_ms)).await; } + Strategy::MidXact => { + wait_for_metric( + flags.metrics_addr, + "walshadow_spill_evictions_total", + Duration::from_secs(30), + ) + .await + .context("large xact never spilled")?; + tokio::time::sleep(Duration::from_millis(kill_delay_ms / 10)).await; + } Strategy::PostCommit => { - wait_for_first_commit(flags.metrics_addr, Duration::from_secs(30)) - .await - .context("first commit drain never happened")?; + wait_for_metric( + flags.metrics_addr, + "walshadow_xacts_committed_total", + Duration::from_secs(30), + ) + .await + .context("first commit drain never happened")?; } } @@ -590,6 +607,17 @@ async fn run_cycle( .await .context("restart pump catchup")?; + // Large xact was open at kill, so restart reads and spills it again + if matches!(strategy, Strategy::MidXact) { + wait_for_metric( + flags.metrics_addr, + "walshadow_spill_evictions_total", + Duration::from_secs(5), + ) + .await + .context("restart never spilled reopened large xact")?; + } + // 10. Wait for source/CH parity fx::wait_ch_matches_source(ch, source, "kr.t", "default.kr_t", Duration::from_secs(60)) .context("source vs CH parity")?; diff --git a/tests/tee_cdc.rs b/tests/tee_cdc.rs index ee0ef8a6..7a203c2f 100644 --- a/tests/tee_cdc.rs +++ b/tests/tee_cdc.rs @@ -1,6 +1,8 @@ //! Tee end-to-end: one source table feeds a `ReplacingMergeTree` primary, a //! `MergeTree` audit log, and a replica in another database. Rows, deletes -//! and `ADD COLUMN` must reach all three. +//! and `ADD COLUMN` must reach all three. Named instances differ in +//! compression, so one INSERT sends prepared blocks to one and rebuilds +//! them for other #![cfg(target_os = "linux")] @@ -103,6 +105,7 @@ async fn run(named: bool) { host: "127.0.0.1".into(), port: remote_ports.ch_tcp, database: "walshadow_test".into(), + compression: walshadow::ch::CompressionChoice::None, ..Default::default() }, ); @@ -174,6 +177,11 @@ async fn run(named: bool) { .wait_for_replay(target, Duration::from_secs(30)) .expect("shadow replay"); assert!(observed >= target); + let rebuilds = pipeline + .stats + .inserter_rebuilds + .load(std::sync::atomic::Ordering::Relaxed); + assert_eq!(rebuilds > 0, named, "rebuilds {rebuilds}"); pipeline.shutdown().await.expect("pipeline drains clean"); let live = "SELECT arrayStringConcat(groupArray(concat(toString(id), ':', body, ':', \ From a2e31645e3cd4523c439e04696fa4082704563be Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:41:34 +0000 Subject: [PATCH 15/20] remove remaining ColumnValue usage, don't pass numeric through text --- architecture/values.md | 10 + benches/bootstrap_pump.rs | 6 +- benches/jsonb.rs | 11 +- benches/spool_read.rs | 15 +- plans/value_coercion.md | 2 +- src/backfill/backfill_bootstrap.rs | 6 +- src/backfill/backup_page_walk.rs | 15 +- src/backfill/copy_backfill.rs | 405 +++++----- src/backfill/spool.rs | 8 +- src/backfill/visibility_gate.rs | 22 +- src/backfill/visibility_pending.rs | 21 +- src/backfill/wal_replay.rs | 16 +- src/bin/stream/session.rs | 2 +- src/catalog/type_bridge.rs | 394 ++++----- src/decode/codecs/fixed.rs | 10 + src/decode/codecs/jsonb.rs | 15 +- src/decode/codecs/mod.rs | 11 +- src/decode/codecs/numeric.rs | 305 +++++-- src/decode/datum.rs | 254 +++--- src/decode/heap_decoder.rs | 1018 ++++++++---------------- src/emit/ch_ddl.rs | 17 +- src/emit/ch_emitter.rs | 991 +++++++++++++---------- src/emit/direct_parity.rs | 102 +-- src/emit/pipeline/batcher.rs | 16 +- src/emit/pipeline/bootstrap.rs | 83 +- src/emit/pipeline/plan_spool.rs | 102 ++- src/emit/pipeline/planner.rs | 56 +- src/emit/pipeline/resolver.rs | 44 +- src/lib.rs | 2 +- src/ops/oracle.rs | 75 +- src/runtime_config.rs | 191 +++-- src/schema.rs | 2 +- src/toast/read_ahead.rs | 48 +- src/toast/resolver.rs | 128 +-- src/xact/spill.rs | 7 +- src/xact/xact_buffer.rs | 258 ++---- tests/backfill_gap_across_promotion.rs | 10 +- tests/bootstrap_direct_e2e.rs | 5 +- tests/bootstrap_object_store_e2e.rs | 5 +- tests/bootstrap_pipeline_ch.rs | 4 +- tests/bridge.rs | 9 +- tests/copy_chunk_resume_e2e.rs | 6 +- tests/emitter_budget_flush.rs | 7 +- tests/emitter_native_types.rs | 57 +- tests/emitter_tls.rs | 34 +- tests/xact_buffer.rs | 53 +- 46 files changed, 2332 insertions(+), 2526 deletions(-) diff --git a/architecture/values.md b/architecture/values.md index cbe701ed..60cf97a7 100644 --- a/architecture/values.md +++ b/architecture/values.md @@ -33,6 +33,16 @@ acknowledgement alone is insufficient because restart could still reread a pre-drop row. Rewrite barriers handle changed physical generations without unconditionally wiping history needed by earlier reads +## Value forms + +Values stay in forms PostgreSQL produced until they become ClickHouse Native +column bytes: stored datums read in place from WAL, page images and TOAST +chunks, or `typoutput` text from SQL-read fast defaults and COPY casts. COPY's +binary send forms convert to stored datums on arrival. Each destination column +compiles one conversion from its source type, so there is no decoded +intermediate value. Detoasted values hold their stored bytes, so resident +accounting counts what was fetched rather than a rendering + ## Convert unfamiliar types After decompression, values outside local codec set are sent to PostgreSQL diff --git a/benches/bootstrap_pump.rs b/benches/bootstrap_pump.rs index 4c61cc7b..30deeac0 100644 --- a/benches/bootstrap_pump.rs +++ b/benches/bootstrap_pump.rs @@ -51,8 +51,8 @@ use walshadow::backfill::backup_source::{ }; use walshadow::backfill_bootstrap::{BootstrapConfig, spawn_greenfield_bootstrap}; use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, PAGE_BYTES, TupleData}; +use walshadow::datum::build; use walshadow::decode::datum::DatumRef; -use walshadow::heap_decoder::ColumnValue; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::bootstrap; use walshadow::pos::{EmitterAck, Monotone}; @@ -443,10 +443,10 @@ async fn bench_drain(shape: Shape) -> Report { let slab: Vec = (slab_start..end) .map(|i| { let mut cells: Vec<_> = (0..shape.columns) - .map(|c| Some(ColumnValue::Int4(i as i32 + c as i32))) + .map(|c| Some(build::int4(i as i32 + c as i32))) .collect(); if shape.text > 0 { - cells.push(Some(ColumnValue::Text(text.clone()))); + cells.push(Some(build::text(&text))); } BackfillTuple { rfn: r.rfn, diff --git a/benches/jsonb.rs b/benches/jsonb.rs index cdd31066..4e687f0d 100644 --- a/benches/jsonb.rs +++ b/benches/jsonb.rs @@ -5,7 +5,16 @@ use std::hint::black_box; use std::time::{Duration, Instant}; use clap::Parser; -use walshadow::codecs::{decode_jsonb_with_capacity, jsonb_text_len}; +use walshadow::codecs::{jsonb_text_len, render_jsonb_text}; + +fn decode_jsonb_with_capacity( + body: &[u8], + len: usize, +) -> Result { + let mut out = String::with_capacity(len); + render_jsonb_text(body, &mut out)?; + Ok(out) +} #[derive(Debug, Parser)] struct Args { diff --git a/benches/spool_read.rs b/benches/spool_read.rs index 9b1af82a..d905abf8 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -16,8 +16,9 @@ use walrus::pg::walparser::{ }; use walshadow::backup_page_walk::{BackfillTuple, TupleData}; use walshadow::ch_emitter::EncodedChunk; +use walshadow::datum::build; use walshadow::emit::route::RouteSnapshot; -use walshadow::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; +use walshadow::heap_decoder::{DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::plan_spool::{PlanItem, PlanWriter, ViewChange}; use walshadow::pipeline::planner::{PlanRouteView, Planner}; @@ -103,11 +104,8 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let base = GLOBAL.reset_peak(); let start = Instant::now(); let data = |i: usize| { - let cells = [ - Some(ColumnValue::Int4(i as i32)), - Some(ColumnValue::Text(payload.clone())), - ]; - walshadow::heap_decoder::form_block_data(&cells).into_boxed_slice() + let cells = [Some(build::int4(i as i32)), Some(build::text(&payload))]; + walshadow::heap_decoder::form_block_data(&[], &cells).into_boxed_slice() }; let data_len = data(0).len(); for i in 0..args.rows { @@ -290,10 +288,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { source_lsn: i as u64, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![ - Some(ColumnValue::Int4(i as i32)), - Some(ColumnValue::Text(payload.clone())), - ], + columns: vec![Some(build::int4(i as i32)), Some(build::text(&payload))], partial: false, }), old: None, diff --git a/plans/value_coercion.md b/plans/value_coercion.md index a3b13349..1c5068b2 100644 --- a/plans/value_coercion.md +++ b/plans/value_coercion.md @@ -112,7 +112,7 @@ containers for unsupported elements - Verify automatic numeric mappings at precision 76/77/1000, negative scale, scale greater than precision, and unconstrained numeric against source text Include trailing fractional zeros, SQL NULL, and String fast default `'NaN'`; - compare decoded default semantics, not SQL spelling such as `unhex('4e614e')` + compare decoded default semantics, not SQL spelling - Preserve finite calendar boundaries; reject adjacent out-of-range values - Reject Decimal values exceeding declared precision even when wire width fits - Accept `1.230` at scale 2; reject `1.234`, including negative equivalents diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index c7555dca..5f965aa5 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -417,6 +417,7 @@ mod tests { use super::*; use crate::backfill::backup_page_walk::{PAGE_BYTES, make_rel, synth_single_tuple_page}; use crate::backfill::backup_source::{BackupSink, BackupSource, FileMeta}; + use crate::decode::datum::build; use async_trait::async_trait; use walrus::pg::backup::tarball::{Entry, Kind, Reader, Writer}; @@ -585,9 +586,6 @@ mod tests { assert_eq!(tuple.source_lsn, 0xDEAD_BEEF); assert_eq!(tuple.xid, 99); let columns = tuple.cells(&make_rel()).unwrap(); - assert_eq!( - columns, - [Some(crate::decode::heap_decoder::ColumnValue::Int4(42))] - ); + assert_eq!(columns, [Some(build::int4(42))]); } } diff --git a/src/backfill/backup_page_walk.rs b/src/backfill/backup_page_walk.rs index 364363d9..a8aa29a6 100644 --- a/src/backfill/backup_page_walk.rs +++ b/src/backfill/backup_page_walk.rs @@ -30,12 +30,12 @@ use crate::backfill::backup_source::{ use crate::backfill::copy_backfill::CopyRow; use crate::backfill::pg_path::{BaseRelFile, RelFork, parse_base_path}; use crate::backfill::walk_barrier::WalkBarrier; -#[cfg(any(test, feature = "test-support"))] -use crate::decode::heap_decoder::{ColumnValue, decode_block_data}; use crate::decode::heap_decoder::{DecodeError, TupleView, view_block_data}; use crate::decode::visibility::{ PendingXids, PgMultiXactAccum, PgXactAccum, PgXactView, SealedPatch, }; +#[cfg(any(test, feature = "test-support"))] +use crate::decode::{datum::Datum, heap_decoder::decode_block_data}; use crate::schema::RelDescriptor; use foldhash::{HashMap, HashMapExt, HashSet}; @@ -121,8 +121,8 @@ impl TupleData { /// Heap image of `values`, see /// [`form_block_data`](crate::decode::heap_decoder::form_block_data) #[cfg(any(test, feature = "test-support"))] - pub fn from_cells(values: &[Option]) -> Self { - Self::Heap(crate::decode::heap_decoder::form_block_data(values).into()) + pub fn from_cells(values: &[Option]) -> Self { + Self::Heap(crate::decode::heap_decoder::form_block_data(&[], values).into()) } pub fn len(&self) -> usize { @@ -148,12 +148,12 @@ impl BackfillTuple { /// Owned cells, attnum-1 indexed #[cfg(any(test, feature = "test-support"))] - pub fn cells(&self, rel: &RelDescriptor) -> Result>, String> { + pub fn cells(&self, rel: &RelDescriptor) -> Result>, String> { match &self.data { TupleData::Heap(b) => decode_block_data(b, rel) .map(|t| t.columns) .map_err(|e| e.to_string()), - TupleData::Copy(row) => crate::backfill::copy_backfill::copy_cells(row, rel), + TupleData::Copy(row) => Ok(crate::backfill::copy_backfill::copy_cells(row, rel)), } } } @@ -1187,6 +1187,7 @@ pub(crate) fn synth_single_tuple_page(value: i32) -> [u8; PAGE_BYTES] { #[cfg(test)] mod tests { use super::*; + use crate::decode::datum::Datum; use crate::backfill::backup_source::EndInfo; use crate::backfill::backup_source::testing::expect_tap; @@ -1228,7 +1229,7 @@ mod tests { /// Leading int4 of a tuple walked against [`make_rel`]'s layout fn int4(t: &BackfillTuple) -> Option { match t.cells(&make_rel()).unwrap().first() { - Some(Some(ColumnValue::Int4(v))) => Some(*v), + Some(Some(Datum::Fixed(b))) => Some(i32::from_le_bytes(b.as_slice().try_into().ok()?)), _ => None, } } diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 9b12a8bd..2ae6a109 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -34,13 +34,12 @@ //! ## Field decode //! //! Binary COPY carries each field in `typsend` wire form (big-endian), not -//! the on-disk datum form the WAL heap decoder reads. Fixed-width types and -//! byte/text strings decode natively into the same [`ColumnValue`] variants -//! the WAL path produces; `numeric` and `jsonb` select as `::text` -//! (`numeric_out` and `jsonb_out` are the forms the WAL path renders too); -//! every out-of-matrix type also selects -//! as `::text` and ships as [`ColumnValue::PgPendingText`], so the oracle -//! converts it through `typinput` exactly as it does a WAL-path default +//! the on-disk datum form the WAL heap decoder reads. Each row converts to +//! on-disk form as it arrives: fixed-width fields byte-swap, `name` pads to +//! `NAMEDATALEN`, `numeric_send` rebuilds the stored `numeric` body, and +//! byte/text strings pass unchanged. `jsonb` and every out-of-matrix type +//! select as `::text` and stay `typoutput` text, which the oracle converts +//! through `typinput` exactly as it does a WAL-path default //! (architecture/values.md). //! //! ## Resume ledger @@ -83,10 +82,8 @@ use crate::backfill::opt_in::Backfiller; use crate::backfill::visibility_pending::{self, SharedPendingLedger}; use crate::catalog::shadow_catalog::ShadowCatalog; use crate::config::{DestEmitter, ResolvedConfig}; -use crate::decode::codecs::NumericKind; -use crate::decode::datum::ValueRef; -use crate::decode::heap_decoder::ColumnValue; -use crate::emit::ch_emitter::{Cell, EmitterStats, Slot, ToastMode}; +use crate::decode::datum::DatumRef; +use crate::emit::ch_emitter::{EmitterStats, ToastMode}; use crate::emit::pipeline::tail::OwnedTail; use crate::emit::pipeline::{Fatal, bootstrap}; use crate::emit::route::{Routing, freeze_routes, routing}; @@ -97,9 +94,8 @@ use crate::pos::{Pos, Snapshot}; use crate::runtime_config::InitialLoadMode; use crate::schema::{ BOOLOID, BPCHAROID, BYTEAOID, CHAROID, CIDOID, DATEOID, FLOAT4OID, FLOAT8OID, INT2OID, INT4OID, - INT8OID, JSONBOID, JSONOID, MONEYOID, NAMEOID, NUMERICOID, OID8OID, OIDOID, PG_LSNOID, - RelDescriptor, RelName, TEXTOID, TIMEOID, TIMESTAMPOID, TIMESTAMPTZOID, UUIDOID, VARCHAROID, - XID8OID, XIDOID, + INT8OID, JSONOID, MONEYOID, NAMEOID, NUMERICOID, OID8OID, OIDOID, PG_LSNOID, RelDescriptor, + RelName, TEXTOID, TIMEOID, TIMESTAMPOID, TIMESTAMPTZOID, UUIDOID, VARCHAROID, XID8OID, XIDOID, }; use crate::source::source_feed::open_sql_client; use crate::toast::{ToastResolver, ToastRow}; @@ -406,7 +402,7 @@ impl Ledger { // --------------------------------------------------------------------------- /// How one selected column decodes from the wire. Native kinds read `typsend` -/// output; `NumericText`/`CastText` columns were cast to `text` in the SELECT. +/// output; `Cast` columns were cast to `text` in the SELECT #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum WireKind { Bool, @@ -427,8 +423,8 @@ enum WireKind { Text, Name, Json, - NumericText, - CastText, + Numeric, + Cast, } struct ColPlan { @@ -461,19 +457,15 @@ fn wire_kind(type_oid: u32) -> Option { TEXTOID | VARCHAROID | BPCHAROID => WireKind::Text, NAMEOID => WireKind::Name, JSONOID => WireKind::Json, + NUMERICOID => WireKind::Numeric, _ => return None, }) } /// Wire form a column selects as; out-of-matrix types cast to `text`. -/// jsonb's send form prefixes a version byte; its text is the document the -/// WAL path renders, so take that instead +/// jsonb's send form is its text behind a version byte, so cast it too fn copy_kind(type_oid: u32) -> WireKind { - wire_kind(type_oid).unwrap_or(match type_oid { - JSONBOID => WireKind::Json, - NUMERICOID => WireKind::NumericText, - _ => WireKind::CastText, - }) + wire_kind(type_oid).unwrap_or(WireKind::Cast) } fn column_plan(desc: &RelDescriptor) -> CopyPlan { @@ -567,7 +559,8 @@ pub async fn copy_rows_into( row.try_get(i).context("backfill: COPY field")?; } let payload_bytes = fields.iter().flatten().map(|f| f.len() as u64).sum::(); - let data = CopyRow::new(&fields); + let data = + CopyRow::from_send(desc, &fields).map_err(|e| anyhow::anyhow!("backfill: {e}"))?; slab_bytes += data.bytes().len(); slab.push(BackfillTuple { rfn: desc.rfn, @@ -708,6 +701,26 @@ impl CopyRow { Self(out.into()) } + /// Row of `typsend` fields indexed by attnum - 1, each converted to its + /// on-disk form per `rel`'s attribute types + pub fn from_send(rel: &RelDescriptor, fields: &[Option<&[u8]>]) -> Result { + let head = 2 + 4 * fields.len(); + let body = fields.iter().flatten().map(|f| f.len()).sum::(); + let mut out = Vec::with_capacity(head + body); + out.extend_from_slice(&(fields.len() as u16).to_le_bytes()); + out.resize(head, 0); + for (i, f) in fields.iter().enumerate() { + if let Some(raw) = f { + let type_oid = rel.attributes.get(i).map_or(0, |a| a.type_oid); + store_field(copy_kind(type_oid), raw, &mut out)?; + } + let end = u32::try_from(out.len() - head).map_err(|_| "COPY row past 4 GiB")?; + let mark = if f.is_none() { COPY_NULL } else { 0 }; + out[2 + 4 * i..][..4].copy_from_slice(&(end | mark).to_le_bytes()); + } + Ok(Self(out.into())) + } + /// Spooled bytes, see [`Self::from_bytes`] pub fn bytes(&self) -> &[u8] { &self.0 @@ -749,83 +762,113 @@ impl CopyRow { } } -fn text_or_bytea<'a>(raw: &'a [u8], wrap: fn(&'a str) -> ValueRef<'a>) -> ValueRef<'a> { - simdutf8::basic::from_utf8(raw).map_or(ValueRef::Bytea(raw), wrap) -} - -/// Value of attribute index `i` in heap-decoder shape; NULL past the row -pub(crate) fn copy_slot<'a>( - row: &'a CopyRow, - rel: &RelDescriptor, - i: usize, -) -> Result, String> { +/// Attribute index `i` of a row [`CopyRow::from_send`] stored; NULL past the row +pub(crate) fn copy_slot<'a>(row: &'a CopyRow, rel: &RelDescriptor, i: usize) -> DatumRef<'a> { let (Some(att), Some(raw)) = (rel.attributes.get(i), row.field(i)) else { - return Ok(Slot::Cell(Cell::Value(ValueRef::Null))); + return DatumRef::Null; }; - decode_field(copy_kind(att.type_oid), att.type_oid, raw) + match copy_kind(att.type_oid) { + WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Numeric => { + DatumRef::Varlena(raw) + } + WireKind::Cast => DatumRef::Text(raw), + _ => DatumRef::Fixed(raw), + } } -/// Owned cells of a COPY row, attnum-1 indexed +/// Owned datums of a COPY row, attnum-1 indexed #[cfg(any(test, feature = "test-support"))] pub(crate) fn copy_cells( row: &CopyRow, rel: &RelDescriptor, -) -> Result>, String> { +) -> Vec> { (0..rel.attributes.len()) - .map(|i| copy_slot(row, rel, i).map(|v| Some(owned(v)))) + .map(|i| crate::decode::heap_decoder::resolve(&rel.attributes[i], copy_slot(row, rel, i))) .collect() } -#[cfg(any(test, feature = "test-support"))] -fn owned(slot: Slot<'_>) -> ColumnValue { - match slot { - Slot::Cell(Cell::Value(v)) => v.to_owned_value(), - Slot::Owned(v) => v, - Slot::Cell(_) => unreachable!("COPY fields decode to values"), +/// Append non-NULL wire field `raw` in its on-disk form +fn store_field(kind: WireKind, raw: &[u8], out: &mut Vec) -> Result<(), String> { + let swapped = |out: &mut Vec, width: usize, name: &str| { + if raw.len() != width { + return Err(format!("{name}: want {width} bytes, got {}", raw.len())); + } + out.extend(raw.iter().rev()); + Ok(()) + }; + match kind { + WireKind::Bool => swapped(out, 1, "bool"), + WireKind::Char => swapped(out, 1, "char"), + WireKind::Int2 => swapped(out, 2, "int2"), + WireKind::Int4 => swapped(out, 4, "int4"), + WireKind::Oid => swapped(out, 4, "oid"), + WireKind::Float4 => swapped(out, 4, "float4"), + WireKind::Date => swapped(out, 4, "date"), + WireKind::Int8 => swapped(out, 8, "int8"), + WireKind::Oid8 => swapped(out, 8, "oid8"), + WireKind::Float8 => swapped(out, 8, "float8"), + WireKind::Time => swapped(out, 8, "time"), + WireKind::Timestamp => swapped(out, 8, "timestamp"), + WireKind::TimestampTz => swapped(out, 8, "timestamptz"), + // Network byte order on disk too + WireKind::Uuid if raw.len() == 16 => { + out.extend_from_slice(raw); + Ok(()) + } + WireKind::Uuid => Err(format!("uuid: want 16 bytes, got {}", raw.len())), + WireKind::Name if raw.len() < NAMEDATALEN => { + out.extend_from_slice(raw); + out.resize(out.len() + NAMEDATALEN - raw.len(), 0); + Ok(()) + } + WireKind::Name => Err(format!("name: {} bytes reach NAMEDATALEN", raw.len())), + WireKind::Numeric => { + out.extend_from_slice(&numeric_from_send(raw)?); + Ok(()) + } + WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Cast => { + out.extend_from_slice(raw); + Ok(()) + } } } -/// Decode non-NULL wire field into heap-decoder value shape -fn decode_field(kind: WireKind, type_oid: u32, raw: &[u8]) -> Result, String> { - let v = match kind { - WireKind::Bool => ValueRef::Bool(fixed::<1>(raw, "bool")?[0] != 0), - WireKind::Char => ValueRef::Char(fixed::<1>(raw, "char")?[0] as i8), - WireKind::Int2 => ValueRef::Int2(i16::from_be_bytes(fixed(raw, "int2")?)), - WireKind::Int4 => ValueRef::Int4(i32::from_be_bytes(fixed(raw, "int4")?)), - WireKind::Int8 => ValueRef::Int8(i64::from_be_bytes(fixed(raw, "int8")?)), - WireKind::Oid => ValueRef::Oid(u32::from_be_bytes(fixed(raw, "oid")?)), - WireKind::Oid8 => ValueRef::Oid8(u64::from_be_bytes(fixed(raw, "oid8")?)), - WireKind::Float4 => ValueRef::Float4(f32::from_be_bytes(fixed(raw, "float4")?)), - WireKind::Float8 => ValueRef::Float8(f64::from_be_bytes(fixed(raw, "float8")?)), - WireKind::Date => ValueRef::Date(i32::from_be_bytes(fixed(raw, "date")?)), - WireKind::Time => ValueRef::Time(i64::from_be_bytes(fixed(raw, "time")?)), - WireKind::Timestamp => ValueRef::Timestamp(i64::from_be_bytes(fixed(raw, "timestamp")?)), - WireKind::TimestampTz => { - ValueRef::TimestampTz(i64::from_be_bytes(fixed(raw, "timestamptz")?)) - } - WireKind::Uuid => ValueRef::Uuid(fixed(raw, "uuid")?), - WireKind::Bytea => ValueRef::Bytea(raw), - WireKind::Text => text_or_bytea(raw, ValueRef::Text), - // Preserve source type for typinput - WireKind::CastText => match simdutf8::basic::from_utf8(raw) { - Ok(text) => ValueRef::PgPendingText { type_oid, text }, - Err(_) => ValueRef::Bytea(raw), - }, - WireKind::Name => text_or_bytea(raw, ValueRef::Name), - WireKind::Json => text_or_bytea(raw, ValueRef::Json), - // numeric_out text form; specials carry their flag - WireKind::NumericText => { - let s = simdutf8::basic::from_utf8(raw) - .map_err(|_| "numeric::text not utf8".to_string())?; - return Ok(Slot::Owned(ColumnValue::Numeric(match s { - "NaN" => NumericKind::NaN, - "Infinity" => NumericKind::PInf, - "-Infinity" => NumericKind::NInf, - _ => NumericKind::Finite(s.into()), - }))); - } +/// PostgreSQL `NAMEDATALEN`, `name`'s stored width +const NAMEDATALEN: usize = 64; + +/// `numeric_send` (`ndigits`, `weight`, `sign`, `dscale`, digits, all +/// big-endian 16-bit) to the stored body +fn numeric_from_send(raw: &[u8]) -> Result, String> { + use crate::decode::codecs::{Special, numeric_body, numeric_special_body}; + let word = |i: usize| { + raw.get(2 * i..2 * i + 2) + .map(|b| u16::from_be_bytes([b[0], b[1]])) + .ok_or_else(|| format!("numeric: {} bytes truncate word {i}", raw.len())) }; - Ok(Slot::Cell(Cell::Value(v))) + let ndigits = usize::from(word(0)?); + let weight = i32::from(word(1)? as i16); + let sign = word(2)?; + let dscale = word(3)?; + if raw.len() != 8 + 2 * ndigits { + return Err(format!( + "numeric: {ndigits} digits need {} bytes, got {}", + 8 + 2 * ndigits, + raw.len() + )); + } + let special = |s| Ok(numeric_special_body(s)); + match sign { + 0xC000 => special(Special::NaN), + 0xD000 => special(Special::PInf), + 0xF000 => special(Special::NInf), + 0x0000 | 0x4000 => { + let digits = (0..ndigits) + .map(|i| word(4 + i).map(|d| d as i16)) + .collect::, _>>()?; + Ok(numeric_body(sign == 0x4000, weight, dscale, &digits)) + } + other => Err(format!("numeric: sign word {other:#06x}")), + } } // --------------------------------------------------------------------------- @@ -2152,6 +2195,7 @@ struct CopyOutcome { #[cfg(test)] mod tests { use super::*; + use crate::schema::JSONBOID; use crate::schema::{RelAttr, RelName, ReplIdent}; use walrus::pg::walparser::RelFileNode; @@ -2204,202 +2248,161 @@ mod tests { let plan = column_plan(&d); assert_eq!( plan.select, - "\"id\", \"price\"::text, \"tags\"::text, \"doc\"::text" + "\"id\", \"price\", \"tags\"::text, \"doc\"::text" ); assert_eq!(plan.natts, 5); assert_eq!(plan.cols.len(), 4, "dropped column not selected"); assert_eq!(plan.cols[2].attnum, 4); assert_eq!(copy_kind(INT8OID), WireKind::Int8); - assert_eq!(copy_kind(NUMERICOID), WireKind::NumericText); - assert_eq!(copy_kind(1009), WireKind::CastText); + assert_eq!(copy_kind(NUMERICOID), WireKind::Numeric); + assert_eq!(copy_kind(1009), WireKind::Cast); // jsonb_out text, the document shape the WAL path yields - assert_eq!(copy_kind(JSONBOID), WireKind::Json); + assert_eq!(copy_kind(JSONBOID), WireKind::Cast); } #[test] - fn decode_matches_wal_variants() { - assert_eq!( - owned(decode_field(WireKind::Int8, 0, &7i64.to_be_bytes()).unwrap()), - ColumnValue::Int8(7), - ); - assert_eq!( - owned(decode_field(WireKind::Bool, 0, &[1]).unwrap()), - ColumnValue::Bool(true), - ); - assert_eq!( - owned(decode_field(WireKind::Date, 0, &8000i32.to_be_bytes()).unwrap()), - ColumnValue::Date(8000), - ); - assert_eq!( - owned(decode_field(WireKind::Text, 0, b"abc").unwrap()), - ColumnValue::Text("abc".into()), - ); - assert_eq!( - owned(decode_field(WireKind::Name, 0, b"orders").unwrap()), - ColumnValue::Name("orders".into()), - ); - assert_eq!( - owned(decode_field(WireKind::Json, 0, b"{\"a\": 1}").unwrap()), - ColumnValue::Json("{\"a\": 1}".into()), - ); - assert_eq!( - owned(decode_field(WireKind::NumericText, 0, b"12.50").unwrap()), - ColumnValue::Numeric(NumericKind::Finite("12.50".into())), - ); - assert_eq!( - owned(decode_field(WireKind::NumericText, 0, b"NaN").unwrap()), - ColumnValue::Numeric(NumericKind::NaN), - ); - assert_eq!( - owned(decode_field(WireKind::NumericText, 0, b"Infinity").unwrap()), - ColumnValue::Numeric(NumericKind::PInf), - ); + fn send_fields_store_as_wal_datums() { + use crate::decode::datum::{Datum, build}; + let store = |kind, raw: &[u8]| { + let mut out = Vec::new(); + store_field(kind, raw, &mut out).map(|()| out) + }; assert_eq!( - owned(decode_field(WireKind::NumericText, 0, b"-Infinity").unwrap()), - ColumnValue::Numeric(NumericKind::NInf), + store(WireKind::Int8, &7i64.to_be_bytes()).unwrap(), + 7i64.to_le_bytes() ); + assert_eq!(store(WireKind::Bool, &[1]).unwrap(), [1]); assert_eq!( - owned(decode_field(WireKind::CastText, 1007, b"{1,2}").unwrap()), - ColumnValue::PgPendingText { - type_oid: 1007, - text: "{1,2}".into(), - }, + store(WireKind::Date, &8000i32.to_be_bytes()).unwrap(), + 8000i32.to_le_bytes() ); + assert_eq!(store(WireKind::Text, b"abc").unwrap(), b"abc"); + let Datum::Fixed(name) = build::name("orders") else { + unreachable!() + }; + assert_eq!(store(WireKind::Name, b"orders").unwrap(), name.as_slice()); + // numeric_send of 12.50: 2 digits, weight 0, positive, dscale 2 + let send: Vec = [2u16, 0, 0, 2, 12, 5000] + .iter() + .flat_map(|w| w.to_be_bytes()) + .collect(); + let Datum::Varlena(want) = build::numeric("12.50") else { + unreachable!() + }; + assert_eq!(store(WireKind::Numeric, &send).unwrap(), want); + for (sign, text) in [ + (0xC000u16, "NaN"), + (0xD000, "Infinity"), + (0xF000, "-Infinity"), + ] { + let send: Vec = [0u16, 0, sign, 0] + .iter() + .flat_map(|w| w.to_be_bytes()) + .collect(); + let Datum::Varlena(want) = build::numeric(text) else { + unreachable!() + }; + assert_eq!(store(WireKind::Numeric, &send).unwrap(), want, "{text}"); + } // Wrong width is an error, not a silent misread - assert!(decode_field(WireKind::Int4, 0, &[0, 1]).is_err()); - // Invalid UTF-8 degrades to Bytea like the heap decoder - assert_eq!( - owned(decode_field(WireKind::Text, 0, &[0xFF, 0xFE]).unwrap()), - ColumnValue::Bytea(vec![0xFF, 0xFE]), - ); - assert_eq!( - owned(decode_field(WireKind::CastText, 1007, &[0xFF]).unwrap()), - ColumnValue::Bytea(vec![0xFF]), - ); - assert!(decode_field(WireKind::NumericText, 0, &[0xFF]).is_err()); + assert!(store(WireKind::Int4, &[0, 1]).is_err()); + assert!(store(WireKind::Numeric, &[0, 1, 0]).is_err()); + // Cast columns store text output verbatim, invalid UTF-8 included + assert_eq!(store(WireKind::Cast, &[0xFF]).unwrap(), [0xFF]); } /// PostgreSQL binary COPY uses each type's `typsend` format #[test] - fn every_native_kind_decodes_its_send_bytes() { + fn every_native_kind_stores_its_send_bytes() { + let store = |kind, raw: &[u8]| { + let mut out = Vec::new(); + store_field(kind, raw, &mut out).map(|()| out) + }; let uuid = *b"0123456789abcdef"; + let name = [b"rel".as_slice(), &[0; NAMEDATALEN - 3]].concat(); for (oid, kind, raw, want) in [ - (BOOLOID, WireKind::Bool, vec![0], ColumnValue::Bool(false)), - ( - CHAROID, - WireKind::Char, - vec![b'x'], - ColumnValue::Char(b'x' as i8), - ), + (BOOLOID, WireKind::Bool, vec![0], vec![0]), + (CHAROID, WireKind::Char, vec![b'x'], vec![b'x']), ( INT2OID, WireKind::Int2, (-2i16).to_be_bytes().to_vec(), - ColumnValue::Int2(-2), + (-2i16).to_le_bytes().to_vec(), ), ( INT4OID, WireKind::Int4, 5i32.to_be_bytes().to_vec(), - ColumnValue::Int4(5), + 5i32.to_le_bytes().to_vec(), ), ( MONEYOID, WireKind::Int8, 1234i64.to_be_bytes().to_vec(), - ColumnValue::Int8(1234), + 1234i64.to_le_bytes().to_vec(), ), ( XIDOID, WireKind::Oid, 77u32.to_be_bytes().to_vec(), - ColumnValue::Oid(77), + 77u32.to_le_bytes().to_vec(), ), ( PG_LSNOID, WireKind::Oid8, (1u64 << 40).to_be_bytes().to_vec(), - ColumnValue::Oid8(1 << 40), + (1u64 << 40).to_le_bytes().to_vec(), ), ( FLOAT4OID, WireKind::Float4, 1.5f32.to_be_bytes().to_vec(), - ColumnValue::Float4(1.5), + 1.5f32.to_le_bytes().to_vec(), ), ( FLOAT8OID, WireKind::Float8, (-0.25f64).to_be_bytes().to_vec(), - ColumnValue::Float8(-0.25), + (-0.25f64).to_le_bytes().to_vec(), ), ( DATEOID, WireKind::Date, (-1i32).to_be_bytes().to_vec(), - ColumnValue::Date(-1), + (-1i32).to_le_bytes().to_vec(), ), ( TIMEOID, WireKind::Time, 3_600_000_000i64.to_be_bytes().to_vec(), - ColumnValue::Time(3_600_000_000), + 3_600_000_000i64.to_le_bytes().to_vec(), ), ( TIMESTAMPOID, WireKind::Timestamp, 9i64.to_be_bytes().to_vec(), - ColumnValue::Timestamp(9), + 9i64.to_le_bytes().to_vec(), ), ( TIMESTAMPTZOID, WireKind::TimestampTz, 10i64.to_be_bytes().to_vec(), - ColumnValue::TimestampTz(10), - ), - ( - UUIDOID, - WireKind::Uuid, - uuid.to_vec(), - ColumnValue::Uuid(uuid), - ), - ( - BYTEAOID, - WireKind::Bytea, - vec![0, 0xFF], - ColumnValue::Bytea(vec![0, 0xFF]), - ), - ( - BPCHAROID, - WireKind::Text, - b"ab ".to_vec(), - ColumnValue::Text("ab ".into()), - ), - ( - NAMEOID, - WireKind::Name, - b"rel".to_vec(), - ColumnValue::Name("rel".into()), - ), - ( - JSONOID, - WireKind::Json, - b"[]".to_vec(), - ColumnValue::Json("[]".into()), + 10i64.to_le_bytes().to_vec(), ), + (UUIDOID, WireKind::Uuid, uuid.to_vec(), uuid.to_vec()), + (BYTEAOID, WireKind::Bytea, vec![0, 0xFF], vec![0, 0xFF]), + (BPCHAROID, WireKind::Text, b"ab ".to_vec(), b"ab ".to_vec()), + (NAMEOID, WireKind::Name, b"rel".to_vec(), name), + (JSONOID, WireKind::Json, b"[]".to_vec(), b"[]".to_vec()), ] { assert_eq!(wire_kind(oid), Some(kind), "oid {oid}"); - assert_eq!( - owned(decode_field(kind, oid, &raw).unwrap()), - want, - "oid {oid}" - ); + assert_eq!(store(kind, &raw).unwrap(), want, "oid {oid}"); let variable = matches!( kind, WireKind::Bytea | WireKind::Text | WireKind::Name | WireKind::Json ); assert_eq!( - decode_field(kind, oid, &[0; 17]).is_ok(), + store(kind, &[0; 17]).is_ok(), variable, "oid {oid}: only fixed-width kinds reject a wrong width" ); @@ -2410,7 +2413,7 @@ mod tests { for oid in [OID8OID, XID8OID] { assert_eq!(wire_kind(oid), Some(WireKind::Oid8)); } - assert_eq!(wire_kind(NUMERICOID), None); + assert_eq!(wire_kind(NUMERICOID), Some(WireKind::Numeric)); } #[tokio::test] diff --git a/src/backfill/spool.rs b/src/backfill/spool.rs index d8a57c83..24c66a85 100644 --- a/src/backfill/spool.rs +++ b/src/backfill/spool.rs @@ -562,7 +562,7 @@ fn decode_record(buf: &[u8]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::ColumnValue; + use crate::decode::datum::build; use tempfile::tempdir; use tokio::io::AsyncWriteExt; @@ -659,11 +659,7 @@ mod tests { page_lsn: lsn + 0x100, blkno: 3, offnum: 7, - data: TupleData::from_cells(&[ - Some(ColumnValue::Int4(1)), - None, - Some(ColumnValue::Bytea(payload.to_vec())), - ]), + data: TupleData::from_cells(&[Some(build::int4(1)), None, Some(build::bytes(payload))]), pending: None, } } diff --git a/src/backfill/visibility_gate.rs b/src/backfill/visibility_gate.rs index 541d4c0a..480b3596 100644 --- a/src/backfill/visibility_gate.rs +++ b/src/backfill/visibility_gate.rs @@ -546,6 +546,7 @@ mod tests { use crate::backfill::backup_source::{BackupSink, FileMeta, StartInfo}; use crate::backfill::spool::DEFERRED_SPOOL_MEM_MAX; use crate::backfill::spool::SpoolMark; + use crate::decode::datum::{Datum, build}; use crate::decode::visibility::{ HEAP_XMAX_COMMITTED, HEAP_XMAX_INVALID, HEAP_XMAX_IS_MULTI, HEAP_XMAX_LOCK_ONLY, HEAP_XMIN_COMMITTED, HEAP_XMIN_INVALID, @@ -820,7 +821,6 @@ mod tests { /// route to two tails at once, each resuming its own seq space #[tokio::test] async fn deferred_lanes_replay_on_their_own_tails() { - use crate::decode::heap_decoder::ColumnValue; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; let tmp = tempfile::tempdir().unwrap(); @@ -853,7 +853,7 @@ mod tests { }; let row = |id| BackfillTuple { - data: TupleData::from_cells(&[Some(ColumnValue::Int4(id))]), + data: TupleData::from_cells(&[Some(build::int4(id))]), ..tuple(100, 0, HEAP_XMIN_COMMITTED) }; let mut tails = Vec::new(); @@ -962,7 +962,7 @@ mod tests { /// them for chunk-store resolution #[tokio::test] async fn mapped_external_values_stream_untouched() { - use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; + use crate::decode::heap_decoder::ToastPointer; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::RelName; @@ -978,7 +978,7 @@ mod tests { d.attributes.push(body); let mut catalog = CatalogMap::new(); catalog.insert(desc.clone()); - let external = ColumnValue::ExternalToast(ToastPointer { + let external = Datum::External(ToastPointer { va_rawsize: 8004, va_extinfo: 8000, va_valueid: 100, @@ -1005,22 +1005,20 @@ mod tests { tees: Vec::new(), }; let row = |value, infomask| BackfillTuple { - data: TupleData::from_cells(&[Some(ColumnValue::Int4(1)), Some(value)]), + data: TupleData::from_cells(&[Some(build::int4(1)), Some(value)]), ..tuple(100, 0, infomask) }; let has_mapped_external = |t: &BackfillTuple| { let cells = t.cells(&desc).unwrap(); - mapping.columns.iter().any(|c| { - matches!( - cells[c.src_attnum as usize - 1], - Some(ColumnValue::ExternalToast(_)) - ) - }) + mapping + .columns + .iter() + .any(|c| matches!(cells[c.src_attnum as usize - 1], Some(Datum::External(_)))) }; let (stats, passed) = run_stream_phase( &catalog, vec![ - row(ColumnValue::Text("inline".into()), HEAP_XMIN_COMMITTED), + row(build::text("inline"), HEAP_XMIN_COMMITTED), row(external.clone(), HEAP_XMIN_INVALID), row(external.clone(), HEAP_XMIN_COMMITTED), ], diff --git a/src/backfill/visibility_pending.rs b/src/backfill/visibility_pending.rs index f04036ec..5248602e 100644 --- a/src/backfill/visibility_pending.rs +++ b/src/backfill/visibility_pending.rs @@ -27,7 +27,7 @@ use crate::backfill::backup_page_walk::{BOOTSTRAP_TUPLE_CHANNEL_CAP, BackfillTup use crate::backfill::spool::{DEFERRED_SPOOL_MEM_MAX, DeferredSpool}; use crate::ch::quote_ident; use crate::config::ResolvedConfig; -use crate::decode::heap_decoder::ColumnValue; +use crate::decode::datum::Datum; use crate::decode::visibility::{PendingXids, PgXactView, XidStatus}; use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::tail::OwnedTail; @@ -164,16 +164,16 @@ fn pending_table_mapping( pub(crate) fn metadata_patch( tuple: &BackfillTuple, rel: &RelDescriptor, - out: &mut Vec<(usize, ColumnValue)>, + out: &mut Vec<(usize, Datum)>, ) { let Some(pending) = tuple.pending else { return; }; let base = metadata_base(rel); out.extend([ - (base, ColumnValue::Oid(pending.insert)), - (base + 1, ColumnValue::Oid(pending.delete)), - (base + 2, ColumnValue::Int2(tuple.infomask as i16)), + (base, Datum::fixed(&pending.insert.to_le_bytes())), + (base + 1, Datum::fixed(&pending.delete.to_le_bytes())), + (base + 2, Datum::fixed(&tuple.infomask.to_le_bytes())), ]); } @@ -942,6 +942,7 @@ fn xid_list(xids: &[u32]) -> String { mod tests { use super::*; use crate::backfill::backup_page_walk::{TupleData, make_rel_named}; + use crate::decode::datum::build; fn manifest(rel: &str, xids: Vec) -> PendingManifest { PendingManifest { @@ -1000,7 +1001,7 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - data: TupleData::from_cells(&[Some(ColumnValue::Int4(1))]), + data: TupleData::from_cells(&[Some(build::int4(1))]), pending: Some(PendingXids { insert: 100, delete: 200, @@ -1028,10 +1029,10 @@ mod tests { assert_eq!( chunks[0].1.cells(), [[ - ColumnValue::Int4(1), - ColumnValue::Int4(100), - ColumnValue::Int4(200), - ColumnValue::Int2(0x0100), + build::int4(1), + build::int4(100), + build::int4(200), + build::int2(0x0100), ]], ); } diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index 8ad559e8..bd0af8e8 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -15,9 +15,10 @@ use walrus::pg::walparser::{Oid, RelFileNode, RmId}; use crate::budget::{MemoryBudget, MemoryPermit, acquire_opt}; use crate::catalog::desc_log::{DescriptorLog, DescriptorLogs}; use crate::config::ResolvedConfig; +use crate::decode::datum::Datum; use crate::decode::heap_decoder::{ - ColumnValue, HeapOp, XLH_DELETE_IS_SUPER, XLOG_HEAP_DELETE, XLOG_HEAP_HOT_UPDATE, - XLOG_HEAP_INSERT, XLOG_HEAP_OPMASK, XLOG_HEAP_UPDATE, view_heap_record, + HeapOp, XLH_DELETE_IS_SUPER, XLOG_HEAP_DELETE, XLOG_HEAP_HOT_UPDATE, XLOG_HEAP_INSERT, + XLOG_HEAP_OPMASK, XLOG_HEAP_UPDATE, view_heap_record, }; use crate::decode::visibility::PgXactPatch; use crate::decode::wal_xact::{ @@ -716,7 +717,7 @@ impl WalReplaySink { continue; } let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); - let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); + let mut patches: [Vec<(usize, Datum)>; 2] = Default::default(); let mut permit = None; if !sites.is_empty() { // Make earlier commits available before resolving values. @@ -725,7 +726,7 @@ impl WalReplaySink { self.flush_rows().await?; let db = rel.rfn.db_node; let (values, leaf) = - detoast_sites(&sites, rel, db, lsn, at.spool, at.ref_maps, &self.resolver) + detoast_sites(&sites, db, lsn, at.spool, at.ref_maps, &self.resolver) .await .map_err(SinkError::from)?; for (site, v) in sites.iter().zip(values) { @@ -948,6 +949,7 @@ pub async fn pump_segments_through( mod tests { use super::*; use crate::catalog::desc_log::DescLogIdentity; + use crate::decode::datum::build; use crate::decode::heap_decoder::SIZE_OF_HEAP_UPDATE; use crate::decode::visibility::{PgXactAccum, PgXactView, SealedPatch, XidStatus}; use crate::decode::wal_xact::{ @@ -1641,9 +1643,7 @@ mod tests { #[tokio::test] async fn error_between_commits_ships_committed_rows() { - use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, - }; + use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{RelName, ReplIdent}; use walrus::pg::walparser::RelFileNode; @@ -1693,7 +1693,7 @@ mod tests { source_lsn, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(1))], + columns: vec![Some(build::int4(1))], partial: false, }), old: None, diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index edc33543..0052f8fa 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -1283,7 +1283,7 @@ async fn session_body( } else { // Metrics-only (no CH): the identical pipeline with a null tail — // zero CH connections, no DDL applicator, no oracle (nothing ships, - // PgPending stays raw). The empty mapping routes nothing, so seqs + // out-of-matrix values stay stored bytes). The empty mapping routes nothing, so seqs // complete at placement and the watermark + slot advance move as in // a CH run. Emitter stats stay unexported (`emitter_stats_handle` // None), matching the old serial surface. diff --git a/src/catalog/type_bridge.rs b/src/catalog/type_bridge.rs index 3da91b4e..2d0898dd 100644 --- a/src/catalog/type_bridge.rs +++ b/src/catalog/type_bridge.rs @@ -38,21 +38,19 @@ //! ## Default expression //! //! `attmissingval` arrives as on-disk PG bytes (pinned worker) or element -//! text (SQL path). Resolve tier-3 bytes through shadow PG before rendering -//! `ADD COLUMN` defaults with [`column_value_to_sql_literal`] +//! text (SQL path). Tier-3 bytes resolve to text through shadow PG first. +//! The default encodes into its column as a row would, and +//! `column_literal` renders that cell as the `ADD COLUMN` default use crate::ascii_buf::{AsciiArray, AsciiBuf}; use crate::ch::types; -use crate::decode::codecs::{ - format_time_us, macaddr_to_text, macaddr8_to_text, tid_to_text, timetz_to_text, -}; -use crate::decode::heap_decoder::{self, ColumnValue}; -use crate::emit::ch_emitter::DATE32_PG_EPOCH_DAYS; +use crate::decode::codecs::format_time_us; +use crate::emit::ch_emitter::{ColumnBuf, decimal_le_to_text, encode_default}; use crate::schema::{ BOOLOID, BPCHAROID, BYTEAOID, CHAROID, CIDOID, CIDROID, DATEOID, FLOAT4OID, FLOAT8OID, INETOID, - INT2OID, INT4OID, INT8OID, INTERVALOID, JSONBOID, JSONOID, MONEYOID, NAMEOID, NUMERICOID, - OID8OID, OIDOID, PG_LSNOID, RelAttr, TEXTOID, TIMEOID, TIMESTAMPOID, TIMESTAMPTZOID, TIMETZOID, - UUIDOID, VARCHAROID, XID8OID, XIDOID, + INT2OID, INT4OID, INT8OID, INTERVALOID, JSONBOID, JSONOID, MONEYOID, MissingDefault, NAMEOID, + NUMERICOID, OID8OID, OIDOID, PG_LSNOID, RelAttr, TEXTOID, TIMEOID, TIMESTAMPOID, + TIMESTAMPTZOID, TIMETZOID, UUIDOID, VARCHAROID, XID8OID, XIDOID, }; use clickhouse_c::{Allocator, Kind, TypeAst}; use walrus::time::civil_from_days; @@ -155,12 +153,13 @@ fn array_ch_type(type_name: &str) -> Option { /// `ALTER TABLE ... ADD COLUMN ... DEFAULT k`). `None` when no fast-path /// default or value not cleanly expressible as CH literal fn render_default(att: &RelAttr, ch_inner: &str) -> Option { - let _ = att.missing_default.as_ref()?; - let value = heap_decoder::missing_value_for(att); - if matches!(value, ColumnValue::Null) { - return None; + if let Some(buf) = encode_default(att, ch_inner) { + return column_literal(&buf, ch_inner); } - column_value_to_sql_literal(&value, ch_inner) + let Some(MissingDefault::Text(text)) = &att.missing_default else { + return None; + }; + pending_text_literal(text, ch_inner) } /// Decode `pg_attribute.atttypmod` for `numeric(p, s)`. Falls back to @@ -200,79 +199,78 @@ fn temporal_precision(typmod: i32) -> i32 { } } -/// Render PG-decoded value as CH SQL literal for after `DEFAULT `. `None` -/// for shapes not cleanly expressible (e.g. `Unsupported.raw` needing -/// typmod-aware reconstruction). `ch_inner` (non-nullable, from -/// [`base_type_for`]) drives timestamp formatting + cast shape -pub fn column_value_to_sql_literal(v: &ColumnValue, ch_inner: &str) -> Option { - match v { - ColumnValue::Null => Some("NULL".into()), - ColumnValue::Bool(b) => Some(if *b { "true".into() } else { "false".into() }), - ColumnValue::Char(i) => Some(i.to_string()), - ColumnValue::Int2(i) => Some(i.to_string()), - ColumnValue::Int4(i) => Some(i.to_string()), - ColumnValue::Int8(i) => Some(i.to_string()), - ColumnValue::Oid(i) => Some(i.to_string()), - ColumnValue::Oid8(i) => Some(i.to_string()), - ColumnValue::Float4(f) => Some(format_float(*f as f64)), - ColumnValue::Float8(f) => Some(format_float(*f)), - ColumnValue::Numeric(n) => { - use crate::decode::codecs::NumericKind; - match n { - NumericKind::Finite(s) => { - if matches!( - types::kind(ch_inner), - Some( - Kind::Decimal32 | Kind::Decimal64 | Kind::Decimal128 | Kind::Decimal256 - ) - ) { - Some(s.clone()) - } else { - Some(sql_string_literal(s)) - } - } - NumericKind::NaN => Some("nan".into()), - NumericKind::PInf => Some("inf".into()), - NumericKind::NInf => Some("-inf".into()), - } - } - ColumnValue::Text(s) | ColumnValue::Name(s) | ColumnValue::Json(s) => { - Some(sql_string_literal(s)) +/// First cell of `buf`, a non-nullable `ch_inner` column, as a CH SQL +/// literal for after `DEFAULT `. `None` for shapes a literal can't spell +pub(crate) fn column_literal(buf: &ColumnBuf, ch_inner: &str) -> Option { + let bytes = match buf { + ColumnBuf::String { offsets, data, .. } => { + let cell = &data[..usize::try_from(*offsets.first()?).ok()?]; + return Some(sql_bytes_literal(cell)); } - ColumnValue::Bytea(b) => Some(sql_bytes_literal(b)), - // PG counts days from 2000-01-01, `toDate32` from 1970-01-01 - ColumnValue::Date(days) => Some(format!( + ColumnBuf::Fixed { width, bytes } => bytes.get(..*width)?, + _ => return None, + }; + let ast = TypeAst::parse(ch_inner, Allocator::global(&mimalloc::MiMalloc)).ok()?; + let view = ast.view(); + let int = |n: i128| Some(n.to_string()); + let le = |n: usize| -> Option<[u8; 16]> { + let mut wide = [0u8; 16]; + wide[..n].copy_from_slice(bytes.get(..n)?); + Some(wide) + }; + let signed = |n: usize| { + let shift = 128 - 8 * n as u32; + le(n).map(|w| (i128::from_le_bytes(w) << shift) >> shift) + }; + let unsigned = |n: usize| le(n).map(|w| u128::from_le_bytes(w) as i128); + // Ticks at `scale` digits to microseconds + let micros = |ticks: i64, scale: i32| match 6 - scale { + up @ 0.. => ticks.saturating_mul(10i64.pow(up as u32)), + down => ticks / 10i64.pow(down.unsigned_abs()), + }; + match view.kind()? { + Kind::Bool => Some(if bytes[0] != 0 { "true" } else { "false" }.into()), + Kind::Int8 => int(signed(1)?), + Kind::Int16 => int(signed(2)?), + Kind::Int32 => int(signed(4)?), + Kind::Int64 => int(signed(8)?), + Kind::UInt8 => int(unsigned(1)?), + Kind::UInt16 => int(unsigned(2)?), + Kind::UInt32 => int(unsigned(4)?), + Kind::UInt64 => int(unsigned(8)?), + Kind::Float32 => Some(format_float(f64::from(f32::from_le_bytes( + bytes.try_into().ok()?, + )))), + Kind::Float64 => Some(format_float(f64::from_le_bytes(bytes.try_into().ok()?))), + Kind::Decimal32 | Kind::Decimal64 | Kind::Decimal128 | Kind::Decimal256 => Some( + decimal_le_to_text(bytes, u8::try_from(view.decimal_scale()?).ok()?), + ), + // `toDate32` counts days from 1970-01-01 + Kind::Date32 => Some(format!( "toDate32({})", - i64::from(*days) + i64::from(DATE32_PG_EPOCH_DAYS) + i32::from_le_bytes(bytes.try_into().ok()?) )), - ColumnValue::Time(micros) => { - let txt = format_time_us(*micros); - Some(sql_string_literal(&txt)) - } - ColumnValue::TimeTz { micros, tz_seconds } => { - let txt = timetz_to_text(*micros, *tz_seconds); + Kind::Time64 => { + let ticks = i64::from_le_bytes(bytes.try_into().ok()?); + let txt = format_time_us(micros(ticks, view.datetime64_scale()? as i32)); Some(sql_string_literal(&txt)) } - ColumnValue::Timestamp(micros) | ColumnValue::TimestampTz(micros) => { - let txt = render_pg_timestamp(*micros); - let prec = datetime64_scale(ch_inner).unwrap_or(6); + Kind::DateTime64 => { + let prec = view.datetime64_scale()? as i32; + let unix = micros(i64::from_le_bytes(bytes.try_into().ok()?), prec); + let txt = render_pg_timestamp(unix - walrus::pg::replication::PG_EPOCH_USEC); Some(format!( "toDateTime64({}, {prec}, 'UTC')", sql_string_literal(&txt) )) } - ColumnValue::Interval(v) => Some(sql_string_literal(&v.to_text())), - ColumnValue::Inet(v) => Some(sql_string_literal(&v.to_text())), - ColumnValue::MacAddr(b) => Some(sql_string_literal(&macaddr_to_text(b))), - ColumnValue::MacAddr8(b) => Some(sql_string_literal(&macaddr8_to_text(b))), - ColumnValue::Tid { block, offset } => { - Some(sql_string_literal(&tid_to_text(*block, *offset))) + Kind::Uuid => { + let mut pg: [u8; 16] = bytes.try_into().ok()?; + pg[..8].reverse(); + pg[8..].reverse(); + Some(format!("toUUID({})", sql_string_literal(&format_uuid(&pg)))) } - ColumnValue::Uuid(b) => Some(format!("toUUID({})", sql_string_literal(&format_uuid(b)))), - ColumnValue::PgPendingText { text, .. } => pending_text_literal(text, ch_inner), - ColumnValue::PgPending { .. } - | ColumnValue::ExternalToast(_) - | ColumnValue::Unsupported { .. } => None, + _ => None, } } @@ -338,16 +336,28 @@ fn format_uuid(b: &[u8; 16]) -> AsciiArray<36> { AsciiArray::new(out) } -/// CH SQL string-literal escaping. Double backslash + single-quote; other -/// control chars stay literal, CH parser accepts raw UTF-8 inside `'…'` +/// CH SQL string literal. Doubles single quotes and backslashes; raw UTF-8 +/// stays literal inside `'…'` fn sql_string_literal(s: &str) -> String { - let mut out = String::with_capacity(s.len() + 2); + sql_bytes_literal(s.as_bytes()) +} + +/// [`sql_string_literal`] of arbitrary bytes, those outside UTF-8 spelled +/// `\xHH`, which CH string literals decode to the raw byte +fn sql_bytes_literal(b: &[u8]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(b.len() + 2); out.push('\''); - for ch in s.chars() { - match ch { - '\'' => out.push_str("''"), - '\\' => out.push_str("\\\\"), - other => out.push(other), + for chunk in b.utf8_chunks() { + for ch in chunk.valid().chars() { + match ch { + '\'' => out.push_str("''"), + '\\' => out.push_str("\\\\"), + other => out.push(other), + } + } + for byte in chunk.invalid() { + write!(out, "\\x{byte:02x}").expect("String write"); } } out.push('\''); @@ -424,22 +434,28 @@ fn pg_array_items(text: &str) -> Option>> { } } -/// Raw bytes as CH binary literal via `unhex('…')`; round-trip safe for -/// non-UTF-8 payloads into CH `String` columns -fn sql_bytes_literal(b: &[u8]) -> String { - let mut hex = String::with_capacity(b.len() * 2); - for byte in b { - hex.push_str(&format!("{byte:02x}")); - } - format!("unhex('{hex}')") -} - #[cfg(test)] mod tests { use super::*; - use crate::decode::codecs::{IntervalValue, NumericKind}; + use crate::decode::datum::{Datum, build}; + use crate::decode::heap_decoder::{raw_missing_array, short_varlena}; use crate::schema::RelAttr; + /// Literal of `v` stored as a `type_oid` fast default, for `ch_inner` + fn literal(type_oid: u32, type_len: i16, v: &Datum, ch_inner: &str) -> Option { + let elem = match v { + Datum::Fixed(b) => b.to_vec(), + Datum::Varlena(b) => short_varlena(b), + other => panic!("no stored default for {other:?}"), + }; + let mut att = attr(type_oid, -1, true, None); + att.type_len = type_len; + att.missing_default = Some(crate::schema::MissingDefault::Raw(raw_missing_array( + type_oid, &elem, + ))); + render_default(&att, ch_inner) + } + fn attr(oid: u32, typmod: i32, not_null: bool, missing: Option<&str>) -> RelAttr { RelAttr { attnum: 1, @@ -469,7 +485,7 @@ mod tests { ]; for (micros, expected) in cases { assert_eq!( - column_value_to_sql_literal(&ColumnValue::Time(micros), "Time64(6)"), + literal(TIMEOID, 8, &build::micros(micros), "Time64(6)"), Some(sql_string_literal(expected)), "micros={micros}" ); @@ -489,13 +505,7 @@ mod tests { ]; for (micros, tz, expected) in cases { assert_eq!( - column_value_to_sql_literal( - &ColumnValue::TimeTz { - micros, - tz_seconds: tz, - }, - "String" - ), + literal(TIMETZOID, 12, &build::timetz(micros, tz), "String"), Some(sql_string_literal(expected)), "micros={micros} tz={tz}" ); @@ -719,12 +729,9 @@ mod tests { #[test] fn unresolved_disk_default_never_renders_as_string() { - let value = ColumnValue::PgPending { - type_oid: TSVECTOROID, - raw: vec![0, 0, 0, 0x20], - }; + let value = build::bytes([0, 0, 0, 0x20]); for ch_type in ["JSON", "String", "Array(Nullable(Int32))"] { - assert_eq!(column_value_to_sql_literal(&value, ch_type), None); + assert_eq!(literal(TSVECTOROID, -1, &value, ch_type), None); } } @@ -737,40 +744,45 @@ mod tests { #[test] fn column_value_text_escapes_singles() { - let lit = column_value_to_sql_literal(&ColumnValue::Text("a'b".into()), "String").unwrap(); + let lit = literal(TEXTOID, -1, &build::text("a'b"), "String").unwrap(); assert_eq!(lit, "'a''b'"); } #[test] fn column_value_int_round_trips() { - let lit = column_value_to_sql_literal(&ColumnValue::Int4(-42), "Int32").unwrap(); + let lit = literal(INT4OID, 4, &build::int4(-42), "Int32").unwrap(); assert_eq!(lit, "-42"); } #[test] fn column_value_numeric_to_decimal_drops_quotes() { - let lit = column_value_to_sql_literal( - &ColumnValue::Numeric(NumericKind::Finite("3.14".into())), - "Decimal(10, 2)", - ) - .unwrap(); - assert_eq!(lit, "3.14"); - let lit = column_value_to_sql_literal( - &ColumnValue::Numeric(NumericKind::Finite("3.14".into())), - "String", - ) - .unwrap(); - assert_eq!(lit, "'3.14'"); + let pi = build::numeric("3.14"); + assert_eq!( + literal(NUMERICOID, -1, &pi, "Decimal(10, 2)").unwrap(), + "3.14" + ); + assert_eq!( + literal(NUMERICOID, -1, &pi, "Decimal(10, 4)").unwrap(), + "3.1400" + ); + assert_eq!( + literal(NUMERICOID, -1, &build::numeric("-0.05"), "Decimal(10, 2)").unwrap(), + "-0.05" + ); + assert_eq!(literal(NUMERICOID, -1, &pi, "String").unwrap(), "'3.14'"); + assert_eq!( + literal(NUMERICOID, -1, &build::numeric("NaN"), "Decimal(10, 2)"), + None + ); } #[test] fn column_value_interval_renders_pg_text() { - let v = ColumnValue::Interval(IntervalValue { - months: 0, - days: 1, - micros: 0, - }); - let lit = column_value_to_sql_literal(&v, "String").unwrap(); + // micros, days, months + let mut v = 0i64.to_le_bytes().to_vec(); + v.extend_from_slice(&1i32.to_le_bytes()); + v.extend_from_slice(&0i32.to_le_bytes()); + let lit = literal(INTERVALOID, 16, &Datum::fixed(&v), "String").unwrap(); assert!(lit.starts_with('\''), "{lit}"); assert!(lit.contains("day"), "{lit}"); } @@ -781,7 +793,7 @@ mod tests { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, ]; - let lit = column_value_to_sql_literal(&ColumnValue::Uuid(uuid), "UUID").unwrap(); + let lit = literal(UUIDOID, 16, &build::uuid(uuid), "UUID").unwrap(); assert_eq!( lit, "toUUID('01234567-89ab-cdef-fedc-ba9876543210')", "uuid format mismatch: {lit}" @@ -790,8 +802,7 @@ mod tests { #[test] fn column_value_timestamp_routes_through_to_datetime64() { - let lit = column_value_to_sql_literal(&ColumnValue::TimestampTz(0), "DateTime64(6, 'UTC')") - .unwrap(); + let lit = literal(TIMESTAMPTZOID, 8, &build::micros(0), "DateTime64(6, 'UTC')").unwrap(); assert!( lit.starts_with("toDateTime64('2000-01-01"), "PG epoch should render as 2000-01-01: {lit}" @@ -802,35 +813,26 @@ mod tests { #[test] fn column_value_date_shifts_to_unix_epoch() { // 2024-01-01 is day 8766 after 2000-01-01, day 19723 after 1970-01-01 - let lit = column_value_to_sql_literal(&ColumnValue::Date(8766), "Date32").unwrap(); + let lit = literal(DATEOID, 4, &build::date(8766), "Date32").unwrap(); assert_eq!(lit, "toDate32(19723)"); - let lit = column_value_to_sql_literal(&ColumnValue::Date(0), "Date32").unwrap(); + let lit = literal(DATEOID, 4, &build::date(0), "Date32").unwrap(); assert_eq!(lit, "toDate32(10957)"); } #[test] - fn column_value_bytea_renders_unhex_literal() { - let lit = - column_value_to_sql_literal(&ColumnValue::Bytea(vec![0xDE, 0xAD]), "String").unwrap(); - assert_eq!(lit, "unhex('dead')"); - } - - #[test] - fn column_value_null_is_explicit_null() { - let lit = column_value_to_sql_literal(&ColumnValue::Null, "Int32").unwrap(); - assert_eq!(lit, "NULL"); + fn bytea_default_escapes_invalid_utf8() { + let lit = literal(BYTEAOID, -1, &build::bytes([0xFF, b'\'', 0xFE]), "String").unwrap(); + assert_eq!(lit, r"'\xff''\xfe'"); + let lit = literal(BYTEAOID, -1, &build::bytes(b"ok"), "String").unwrap(); + assert_eq!(lit, "'ok'"); } #[test] - fn external_toast_does_not_render() { - use crate::decode::heap_decoder::ToastPointer; - let v = ColumnValue::ExternalToast(ToastPointer { - va_rawsize: 1, - va_extinfo: 0, - va_valueid: 0, - va_toastrelid: 0, - }); - assert!(column_value_to_sql_literal(&v, "String").is_none()); + fn absent_default_renders_nothing() { + assert_eq!( + render_default(&attr(INT4OID, -1, true, None), "Int32"), + None + ); } #[test] @@ -954,50 +956,80 @@ mod tests { #[test] fn scalar_values_render_as_ch_literals() { - use crate::decode::codecs::InetValue; - let mut addr = [0u8; 16]; - addr[..4].copy_from_slice(&[10, 0, 0, 1]); + use crate::decode::codecs::encode_tid; + use crate::schema::{ + CHAROID, FLOAT4OID, FLOAT8OID, INETOID, INT2OID, INT8OID, MACADDR8OID, MACADDROID, + OID8OID, OIDOID, TIDOID, + }; let cases = [ - (ColumnValue::Char(-3), "-3"), - (ColumnValue::Int2(7), "7"), - (ColumnValue::Int8(-1 << 40), "-1099511627776"), - (ColumnValue::Oid(4_000_000_000), "4000000000"), - (ColumnValue::Oid8(u64::MAX), "18446744073709551615"), - (ColumnValue::Float4(2.0), "2.0"), - (ColumnValue::Float8(f64::NEG_INFINITY), "-inf"), - (ColumnValue::Numeric(NumericKind::NaN), "nan"), - (ColumnValue::Numeric(NumericKind::PInf), "inf"), - (ColumnValue::Numeric(NumericKind::NInf), "-inf"), + (CHAROID, 1, build::char_(-3), "Int8", "-3"), + (INT2OID, 2, build::int2(7), "Int16", "7"), + (INT8OID, 8, build::int8(-1 << 40), "Int64", "-1099511627776"), + (OIDOID, 4, build::oid(4_000_000_000), "UInt32", "4000000000"), + ( + OID8OID, + 8, + build::oid8(u64::MAX), + "UInt64", + "18446744073709551615", + ), + (FLOAT4OID, 4, build::float4(2.0), "Float32", "2.0"), ( - ColumnValue::Inet(InetValue { - family: 2, - bits: 32, - is_cidr: false, - addr, - }), + FLOAT8OID, + 8, + build::float8(f64::NEG_INFINITY), + "Float64", + "-inf", + ), + (NUMERICOID, -1, build::numeric("NaN"), "String", "'NaN'"), + ( + NUMERICOID, + -1, + build::numeric("Infinity"), + "String", + "'Infinity'", + ), + ( + NUMERICOID, + -1, + build::numeric("-Infinity"), + "String", + "'-Infinity'", + ), + ( + INETOID, + -1, + build::bytes([2, 32, 10, 0, 0, 1]), + "String", "'10.0.0.1'", ), ( - ColumnValue::MacAddr([0x08, 0, 0x2b, 1, 2, 3]), + MACADDROID, + 6, + Datum::fixed(&[0x08, 0, 0x2b, 1, 2, 3]), + "String", "'08:00:2b:01:02:03'", ), ( - ColumnValue::MacAddr8([0x08, 0, 0x2b, 1, 2, 3, 4, 5]), + MACADDR8OID, + 8, + Datum::fixed(&[0x08, 0, 0x2b, 1, 2, 3, 4, 5]), + "String", "'08:00:2b:01:02:03:04:05'", ), ( - ColumnValue::Tid { - block: 12, - offset: 3, - }, + TIDOID, + 6, + Datum::fixed(&encode_tid(12, 3)), + "String", "'(12,3)'", ), ]; - for (value, want) in cases { + for (type_oid, type_len, value, ch_inner, want) in cases { assert_eq!( - column_value_to_sql_literal(&value, "String").as_deref(), + literal(type_oid, type_len, &value, ch_inner).as_deref(), Some(want), - "{value:?}" + "{type_oid} {value:?}" ); } } diff --git a/src/decode/codecs/fixed.rs b/src/decode/codecs/fixed.rs index c7efbc70..8014abdd 100644 --- a/src/decode/codecs/fixed.rs +++ b/src/decode/codecs/fixed.rs @@ -42,6 +42,15 @@ pub(crate) fn decode_tid(b: &[u8; 6]) -> (u32, u16) { ) } +/// Inverse of [`decode_tid`] +pub(crate) fn encode_tid(block: u32, offset: u16) -> [u8; 6] { + let mut out = [0u8; 6]; + out[..2].copy_from_slice(&((block >> 16) as u16).to_le_bytes()); + out[2..4].copy_from_slice(&(block as u16).to_le_bytes()); + out[4..].copy_from_slice(&offset.to_le_bytes()); + out +} + /// `tid` as `tidout` prints it pub(crate) fn tid_to_text(block: u32, offset: u16) -> AsciiBuf<18> { let mut out = AsciiBuf::new(); @@ -78,6 +87,7 @@ mod tests { #[test] fn tid_joins_block_halves() { assert_eq!(decode_tid(&[1, 0, 2, 0, 3, 0]), (65538, 3)); + assert_eq!(encode_tid(65538, 3), [1, 0, 2, 0, 3, 0]); let (block, offset) = decode_tid(&[0xff; 6]); assert_eq!(tid_to_text(block, offset).as_str(), "(4294967295,65535)"); assert_eq!(parse_tid("(4294967295,65535)"), Some((block, offset))); diff --git a/src/decode/codecs/jsonb.rs b/src/decode/codecs/jsonb.rs index 890a8d2d..aa7d968b 100644 --- a/src/decode/codecs/jsonb.rs +++ b/src/decode/codecs/jsonb.rs @@ -36,14 +36,14 @@ const JSONB_MAX_DEPTH: usize = 1000; /// Decode `jsonb` varlena body to PG `jsonb_out` text (`JsonbToCString`) pub fn decode_jsonb(body: &[u8]) -> Result { - decode_jsonb_with_capacity(body, body.len()) + let mut out = String::with_capacity(body.len()); + render_jsonb_text(body, &mut out)?; + Ok(out) } -/// [`decode_jsonb`] into a buffer sized by [`jsonb_text_len`] -pub fn decode_jsonb_with_capacity(body: &[u8], capacity: usize) -> Result { - let mut out = String::with_capacity(capacity); - render_jsonb_container(body, 0, &mut out)?; - Ok(out) +/// `jsonb_out` text appended to `out` +pub fn render_jsonb_text(body: &[u8], out: &mut String) -> Result<(), CodecError> { + render_jsonb_container(body, 0, out) } /// [`decode_jsonb`] length without rendering. Text may outgrow its body, @@ -453,7 +453,8 @@ mod tests { for (hex, want) in JSONB_VECTORS { let body = varlena_body(hex); let len = jsonb_text_len(&body).expect(hex); - let text = decode_jsonb_with_capacity(&body, len).expect(hex); + let mut text = String::with_capacity(len); + render_jsonb_text(&body, &mut text).expect(hex); assert_eq!( (text.as_str(), text.capacity()), (*want, want.len()), diff --git a/src/decode/codecs/mod.rs b/src/decode/codecs/mod.rs index fe2fb6c3..e9b334f9 100644 --- a/src/decode/codecs/mod.rs +++ b/src/decode/codecs/mod.rs @@ -11,12 +11,17 @@ mod time; mod uuid; pub(crate) use fixed::{ - decode_tid, macaddr_to_text, macaddr8_to_text, parse_macaddr, parse_tid, tid_to_text, + decode_tid, encode_tid, macaddr_to_text, macaddr8_to_text, parse_macaddr, parse_tid, + tid_to_text, }; pub use inet::{InetValue, PGSQL_AF_INET, PGSQL_AF_INET6, decode_inet}; pub use interval::{IntervalValue, decode_interval}; -pub use jsonb::{decode_jsonb, decode_jsonb_with_capacity, jsonb_text_len}; -pub use numeric::{NumericKind, decode_numeric, numeric_text_len}; +pub use jsonb::{decode_jsonb, jsonb_text_len, render_jsonb_text}; +pub(crate) use numeric::{Numeric, parse_numeric, scaled_groups}; +pub use numeric::{ + Special, numeric_body, numeric_from_text, numeric_special_body, numeric_text_len, + render_numeric_text, +}; pub use text_array::decode_text_array; pub(crate) use time::{format_time_us, timetz_to_text}; pub(crate) use uuid::uuid_to_ch_wire; diff --git a/src/decode/codecs/numeric.rs b/src/decode/codecs/numeric.rs index 30613599..f899e9be 100644 --- a/src/decode/codecs/numeric.rs +++ b/src/decode/codecs/numeric.rs @@ -40,31 +40,37 @@ const NUMERIC_SHORT_WEIGHT_MASK: u16 = 0x003F; const NUMERIC_DSCALE_MASK: u16 = 0x3FFF; -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum NumericKind { +/// Non-finite `numeric`, a header with no digits +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Special { NaN, /// `+Infinity` (PG 14+) PInf, /// `-Infinity` (PG 14+) NInf, - /// PG-text form - Finite(String), } -impl NumericKind { - pub(crate) fn as_text(&self) -> &str { +impl Special { + pub(crate) fn as_text(self) -> &'static str { match self { - Self::Finite(s) => s, Self::NaN => "NaN", Self::PInf => "Infinity", Self::NInf => "-Infinity", } } + + fn header(self) -> u16 { + match self { + Self::NaN => NUMERIC_NAN, + Self::PInf => NUMERIC_PINF, + Self::NInf => NUMERIC_NINF, + } + } } /// `numeric` header fields plus its base-10000 digits, still in the datum -enum Numeric<'a> { - Special(NumericKind), +pub(crate) enum Numeric<'a> { + Special(Special), Finite { neg: bool, weight: i32, @@ -74,7 +80,7 @@ enum Numeric<'a> { }, } -fn parse_numeric(body: &[u8]) -> Result, CodecError> { +pub(crate) fn parse_numeric(body: &[u8]) -> Result, CodecError> { if body.len() < 2 { return Err(CodecError::Truncated { offset: 0, @@ -89,10 +95,10 @@ fn parse_numeric(body: &[u8]) -> Result, CodecError> { if is_special { return Ok(Numeric::Special(match n_header { - NUMERIC_NAN => NumericKind::NaN, - NUMERIC_PINF => NumericKind::PInf, - NUMERIC_NINF => NumericKind::NInf, - _ => NumericKind::NaN, // unknown special, treat as NaN (lossy-safe) + NUMERIC_PINF => Special::PInf, + NUMERIC_NINF => Special::NInf, + // Unknown special reads as NaN, lossy but safe + _ => Special::NaN, })); } @@ -141,26 +147,8 @@ fn parse_numeric(body: &[u8]) -> Result, CodecError> { }) } -/// Decode `numeric` varlena body to PG `numeric_out` text or special value -pub fn decode_numeric(body: &[u8]) -> Result { - match parse_numeric(body)? { - Numeric::Special(kind) => Ok(kind), - Numeric::Finite { - neg, - weight, - dscale, - digits, - } => { - let mut out = String::with_capacity(finite_text_len(neg, weight, dscale, digits)); - render_numeric(neg, weight, dscale, digits, &mut out); - Ok(NumericKind::Finite(out)) - } - } -} - -/// `numeric_out` text into a caller's buffer, for callers holding one already: -/// `jsonb`'s numeric children render straight into the enclosing document -pub(crate) fn render_numeric_text(body: &[u8], out: &mut String) -> Result<(), CodecError> { +/// `numeric_out` text appended to `out` +pub fn render_numeric_text(body: &[u8], out: &mut String) -> Result<(), CodecError> { match parse_numeric(body)? { Numeric::Special(kind) => out.push_str(kind.as_text()), Numeric::Finite { @@ -173,7 +161,7 @@ pub(crate) fn render_numeric_text(body: &[u8], out: &mut String) -> Result<(), C Ok(()) } -/// [`decode_numeric`] text length without rendering +/// [`render_numeric_text`] length without rendering pub fn numeric_text_len(body: &[u8]) -> Result { Ok(match parse_numeric(body)? { Numeric::Special(kind) => kind.as_text().len(), @@ -262,6 +250,121 @@ fn render_numeric(neg: bool, weight: i32, dscale: i32, digits: &[[u8; 2]], out: } } +const NUMERIC_SHORT_DSCALE_MAX: i32 = 0x3F; +const NUMERIC_SHORT_WEIGHT_MAX: i32 = 63; +const NUMERIC_SHORT_WEIGHT_MIN: i32 = -64; +const NBASE: u32 = 10_000; + +/// Body PostgreSQL stores for a non-finite value +pub fn numeric_special_body(special: Special) -> Vec { + special.header().to_le_bytes().to_vec() +} + +/// Body `make_result` stores: zero groups trimmed from both ends, short +/// header when weight and dscale fit +pub fn numeric_body(neg: bool, weight: i32, dscale: u16, digits: &[i16]) -> Vec { + let lead = digits.iter().take_while(|&&d| d == 0).count(); + let digits = &digits[lead..]; + let digits = &digits[..digits.len() - digits.iter().rev().take_while(|&&d| d == 0).count()]; + let (neg, weight) = if digits.is_empty() { + (false, 0) + } else { + (neg, weight - lead as i32) + }; + let dscale = i32::from(dscale & NUMERIC_DSCALE_MASK); + let mut out = Vec::with_capacity(4 + 2 * digits.len()); + if dscale <= NUMERIC_SHORT_DSCALE_MAX + && (NUMERIC_SHORT_WEIGHT_MIN..=NUMERIC_SHORT_WEIGHT_MAX).contains(&weight) + { + let sign = if neg { NUMERIC_SHORT_SIGN_MASK } else { 0 }; + let weight_sign = if weight < 0 { + NUMERIC_SHORT_WEIGHT_SIGN_MASK + } else { + 0 + }; + let header = NUMERIC_SHORT + | sign + | ((dscale as u16) << NUMERIC_SHORT_DSCALE_SHIFT) + | weight_sign + | (weight as u16 & NUMERIC_SHORT_WEIGHT_MASK); + out.extend_from_slice(&header.to_le_bytes()); + } else { + let sign = if neg { NUMERIC_NEG } else { NUMERIC_POS }; + out.extend_from_slice(&(sign | dscale as u16).to_le_bytes()); + out.extend_from_slice(&(weight as i16).to_le_bytes()); + } + for d in digits { + out.extend_from_slice(&d.to_le_bytes()); + } + out +} + +/// `numeric_in` for plain decimal text and the special spellings +pub fn numeric_from_text(text: &str) -> Option> { + let text = text.trim(); + match text.to_ascii_lowercase().as_str() { + "nan" => return Some(numeric_special_body(Special::NaN)), + "infinity" | "+infinity" | "inf" | "+inf" => { + return Some(numeric_special_body(Special::PInf)); + } + "-infinity" | "-inf" => return Some(numeric_special_body(Special::NInf)), + _ => {} + } + let (neg, body) = match text.as_bytes().first()? { + b'-' => (true, &text[1..]), + b'+' => (false, &text[1..]), + _ => (false, text), + }; + let (int, frac) = body.split_once('.').unwrap_or((body, "")); + let all_digits = |s: &str| s.bytes().all(|b| b.is_ascii_digit()); + if (int.is_empty() && frac.is_empty()) || !all_digits(int) || !all_digits(frac) { + return None; + } + let dscale = u16::try_from(frac.len()).ok()?; + let int_pad = (4 - int.len() % 4) % 4; + let frac_pad = (4 - frac.len() % 4) % 4; + let decimal: Vec = std::iter::repeat_n(b'0', int_pad) + .chain(int.bytes()) + .chain(frac.bytes()) + .chain(std::iter::repeat_n(b'0', frac_pad)) + .collect(); + let digits: Vec = decimal + .chunks(4) + .map(|g| g.iter().fold(0i16, |acc, b| acc * 10 + i16::from(b - b'0'))) + .collect(); + let weight = i32::try_from((int_pad + int.len()) / 4).ok()? - 1; + Some(numeric_body(neg, weight, dscale, &digits)) +} + +/// Finite value times `10^scale`, fractional digits past `scale` dropped. +/// `step(m, a)` sets the caller's accumulator to `acc * m + a`, `false` on +/// overflow +pub(crate) fn scaled_groups( + weight: i32, + digits: &[[u8; 2]], + scale: u32, + mut step: impl FnMut(u32, u32) -> bool, +) -> bool { + let digit = |g: i32| { + usize::try_from(g) + .ok() + .and_then(|i| digits.get(i)) + .map_or(0, |raw| i16::from_le_bytes(*raw).max(0) as u32) + }; + let whole = (scale / 4) as i32; + let rem = scale % 4; + for e in (-whole..=weight.max(0)).rev() { + if !step(NBASE, digit(weight - e)) { + return false; + } + } + if rem == 0 { + return true; + } + let tail = digit(weight + whole + 1) / 10u32.pow(4 - rem); + step(10u32.pow(rem), tail) +} + #[cfg(test)] mod tests { use super::*; @@ -294,86 +397,71 @@ mod tests { out } + fn text(body: &[u8]) -> Result { + let mut out = String::new(); + render_numeric_text(body, &mut out)?; + Ok(out) + } + #[test] fn numeric_short_one_digit() { let body = short_numeric(false, 0, 0, &[42]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("42".into()) - ); + assert_eq!(text(&body).unwrap(), "42"); } #[test] fn numeric_short_negative() { let body = short_numeric(true, 0, 0, &[7]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("-7".into()) - ); + assert_eq!(text(&body).unwrap(), "-7"); } #[test] fn numeric_short_with_scale() { // 1.5: digits base-10000 → [1, 5000] let body = short_numeric(false, 0, 1, &[1, 5000]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("1.5".into()) - ); + assert_eq!(text(&body).unwrap(), "1.5"); } #[test] fn numeric_zero() { let body = short_numeric(false, 0, 0, &[]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("0".into()) - ); + assert_eq!(text(&body).unwrap(), "0"); } #[test] fn numeric_long_form_large() { // 12345: base-10000 blocks [1, 2345], weight 1 let body = long_numeric(false, 1, 0, &[1, 2345]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("12345".into()) - ); + assert_eq!(text(&body).unwrap(), "12345"); } #[test] fn numeric_specials() { let nan = NUMERIC_NAN.to_le_bytes(); - assert_eq!(decode_numeric(&nan).unwrap(), NumericKind::NaN); + assert_eq!(text(&nan).unwrap(), "NaN"); let pinf = NUMERIC_PINF.to_le_bytes(); - assert_eq!(decode_numeric(&pinf).unwrap(), NumericKind::PInf); + assert_eq!(text(&pinf).unwrap(), "Infinity"); let ninf = NUMERIC_NINF.to_le_bytes(); - assert_eq!(decode_numeric(&ninf).unwrap(), NumericKind::NInf); + assert_eq!(text(&ninf).unwrap(), "-Infinity"); } #[test] fn numeric_truncated_returns_error() { let one_byte = [0u8]; - assert!(matches!( - decode_numeric(&one_byte), - Err(CodecError::Truncated { .. }) - )); + assert!(matches!(text(&one_byte), Err(CodecError::Truncated { .. }))); } #[test] fn numeric_short_negative_weight_renders_leading_zero() { let body = short_numeric(false, -1, 1, &[5000]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("0.5".into()) - ); + assert_eq!(text(&body).unwrap(), "0.5"); } #[test] fn numeric_long_form_truncated_body() { let body = NUMERIC_POS.to_le_bytes().to_vec(); assert_eq!( - decode_numeric(&body), + text(&body), Err(CodecError::Truncated { offset: 2, need: 4, @@ -386,16 +474,13 @@ mod tests { fn numeric_unknown_special_falls_back_to_nan() { let header: u16 = NUMERIC_SPECIAL | 0x0001; let body = header.to_le_bytes(); - assert_eq!(decode_numeric(&body).unwrap(), NumericKind::NaN); + assert_eq!(text(&body).unwrap(), "NaN"); } #[test] fn numeric_dscale_trailing_zero_pad() { let body = short_numeric(false, 0, 4, &[5]); - assert_eq!( - decode_numeric(&body).unwrap(), - NumericKind::Finite("5.0000".into()) - ); + assert_eq!(text(&body).unwrap(), "5.0000"); } #[test] @@ -415,18 +500,71 @@ mod tests { NUMERIC_NINF.to_le_bytes().to_vec(), ]; for body in bodies { - let text = decode_numeric(&body).unwrap(); - assert_eq!( - numeric_text_len(&body), - Ok(text.as_text().len()), - "{text:?}" - ); - let mut rendered = String::new(); - render_numeric_text(&body, &mut rendered).unwrap(); - assert_eq!(rendered, text.as_text()); + let rendered = text(&body).unwrap(); + assert_eq!(numeric_text_len(&body), Ok(rendered.len()), "{rendered:?}"); } } + #[test] + fn numeric_from_text_round_trips_canonically() { + for (input, want) in [ + ("0", "0"), + ("-0.00", "0.00"), + ("12345", "12345"), + ("-12.340", "-12.340"), + ("0.0005", "0.0005"), + ("100000000", "100000000"), + (".5", "0.5"), + ("NaN", "NaN"), + ("-Infinity", "-Infinity"), + ] { + let body = numeric_from_text(input).unwrap(); + assert_eq!(text(&body).unwrap(), want, "{input}"); + } + assert_eq!( + numeric_from_text("12345").unwrap(), + short_numeric(false, 1, 0, &[1, 2345]) + ); + assert_eq!( + numeric_from_text("1.5").unwrap(), + short_numeric(false, 0, 1, &[1, 5000]) + ); + assert_eq!( + numeric_from_text("0").unwrap(), + short_numeric(false, 0, 0, &[]) + ); + let wide = format!("1.{}", "0".repeat(70)); + assert_eq!( + numeric_from_text(&wide).unwrap(), + long_numeric(false, 0, 70, &[1]) + ); + for bad in ["", "-", ".", "1e5", "1.2.3", "abc"] { + assert_eq!(numeric_from_text(bad), None, "{bad}"); + } + } + + #[test] + fn scaled_groups_truncates_past_scale() { + let scaled = |input: &str, scale: u32| { + let body = numeric_from_text(input).unwrap(); + let Numeric::Finite { weight, digits, .. } = parse_numeric(&body).unwrap() else { + panic!("finite"); + }; + let mut acc = 0u128; + assert!(scaled_groups(weight, digits, scale, |m, a| { + acc = acc * u128::from(m) + u128::from(a); + true + })); + acc + }; + assert_eq!(scaled("12.34", 2), 1234); + assert_eq!(scaled("12.3456789", 3), 12345); + assert_eq!(scaled("12", 5), 1_200_000); + assert_eq!(scaled("0.0005", 4), 5); + assert_eq!(scaled("0.0005", 3), 0); + assert_eq!(scaled("123456789.5", 0), 123_456_789); + } + #[test] fn numeric_rejects_oversized_digit_array() { let mut body = NUMERIC_POS.to_le_bytes().to_vec(); @@ -435,9 +573,6 @@ mod tests { 0u8, ((NUMERIC_DSCALE_MASK as usize) + 5) * 2, )); - assert!(matches!( - decode_numeric(&body), - Err(CodecError::BadNumeric { .. }) - )); + assert!(matches!(text(&body), Err(CodecError::BadNumeric { .. }))); } } diff --git a/src/decode/datum.rs b/src/decode/datum.rs index 8eb98d86..e8a260f6 100644 --- a/src/decode/datum.rs +++ b/src/decode/datum.rs @@ -1,8 +1,8 @@ -//! Borrowed tuple attributes and values, so encoders read source bytes in -//! place instead of building owned cells per row +//! Tuple attributes as PostgreSQL stores or sends them. Encoders read source +//! bytes in place; owned copies hold only bytes that outlive their record -use crate::decode::codecs::{InetValue, IntervalValue, NumericKind}; -use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; +use crate::decode::heap_decoder::ToastPointer; +use smallvec::SmallVec; /// One attribute as stored in a tuple image, storage framing stripped #[derive(Clone, Copy, Debug, PartialEq)] @@ -25,137 +25,141 @@ pub enum DatumRef<'a> { Cstring(&'a [u8]), /// Varlena tag PostgreSQL never writes to disk; rest of buffer BadTag(&'a [u8]), + /// `typoutput` text: fast defaults read over SQL, COPY fields cast to text + Text(&'a [u8]), } -/// Borrowed counterpart of [`ColumnValue`], one variant per variant -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum ValueRef<'a> { +/// Owned [`DatumRef`] for values outliving their record: TOAST values, +/// synthetic columns, COPY and test cells +#[derive(Clone, Debug, PartialEq)] +pub enum Datum { Null, - Bool(bool), - Char(i8), - Int2(i16), - Int4(i32), - Int8(i64), - Float4(f32), - Float8(f64), - Oid(u32), - Oid8(u64), - MacAddr([u8; 6]), - MacAddr8([u8; 8]), - Tid { block: u32, offset: u16 }, - Date(i32), - Time(i64), - Timestamp(i64), - TimestampTz(i64), - TimeTz { micros: i64, tz_seconds: i32 }, - Uuid([u8; 16]), - Name(&'a str), - Bytea(&'a [u8]), - Text(&'a str), - Numeric(&'a NumericKind), - Inet(InetValue), - Interval(IntervalValue), - Json(&'a str), - PgPending { type_oid: u32, raw: &'a [u8] }, - PgPendingText { type_oid: u32, text: &'a str }, - ExternalToast(ToastPointer), - Unsupported { type_oid: u32, raw: &'a [u8] }, + Fixed(SmallVec<[u8; 16]>), + Varlena(Vec), + External(ToastPointer), + Cstring(Vec), + BadTag(Vec), + Text(Vec), } -impl ColumnValue { - pub fn view(&self) -> ValueRef<'_> { +impl Datum { + pub fn view(&self) -> DatumRef<'_> { match self { - Self::Null => ValueRef::Null, - Self::Bool(v) => ValueRef::Bool(*v), - Self::Char(v) => ValueRef::Char(*v), - Self::Int2(v) => ValueRef::Int2(*v), - Self::Int4(v) => ValueRef::Int4(*v), - Self::Int8(v) => ValueRef::Int8(*v), - Self::Float4(v) => ValueRef::Float4(*v), - Self::Float8(v) => ValueRef::Float8(*v), - Self::Oid(v) => ValueRef::Oid(*v), - Self::Oid8(v) => ValueRef::Oid8(*v), - Self::MacAddr(v) => ValueRef::MacAddr(*v), - Self::MacAddr8(v) => ValueRef::MacAddr8(*v), - Self::Tid { block, offset } => ValueRef::Tid { - block: *block, - offset: *offset, - }, - Self::Date(v) => ValueRef::Date(*v), - Self::Time(v) => ValueRef::Time(*v), - Self::Timestamp(v) => ValueRef::Timestamp(*v), - Self::TimestampTz(v) => ValueRef::TimestampTz(*v), - Self::TimeTz { micros, tz_seconds } => ValueRef::TimeTz { - micros: *micros, - tz_seconds: *tz_seconds, - }, - Self::Uuid(v) => ValueRef::Uuid(*v), - Self::Name(s) => ValueRef::Name(s), - Self::Bytea(b) => ValueRef::Bytea(b), - Self::Text(s) => ValueRef::Text(s), - Self::Numeric(n) => ValueRef::Numeric(n), - Self::Inet(v) => ValueRef::Inet(*v), - Self::Interval(v) => ValueRef::Interval(*v), - Self::Json(s) => ValueRef::Json(s), - Self::PgPending { type_oid, raw } => ValueRef::PgPending { - type_oid: *type_oid, - raw, - }, - Self::PgPendingText { type_oid, text } => ValueRef::PgPendingText { - type_oid: *type_oid, - text, - }, - Self::ExternalToast(p) => ValueRef::ExternalToast(*p), - Self::Unsupported { type_oid, raw } => ValueRef::Unsupported { - type_oid: *type_oid, - raw, - }, + Self::Null => DatumRef::Null, + Self::Fixed(b) => DatumRef::Fixed(b), + Self::Varlena(b) => DatumRef::Varlena(b), + Self::External(p) => DatumRef::External(*p), + Self::Cstring(b) => DatumRef::Cstring(b), + Self::BadTag(b) => DatumRef::BadTag(b), + Self::Text(b) => DatumRef::Text(b), } } -} -impl ValueRef<'_> { - pub fn to_owned_value(self) -> ColumnValue { + /// Payload bytes held, for resident accounting + pub fn approx_bytes(&self) -> usize { match self { - Self::Null => ColumnValue::Null, - Self::Bool(v) => ColumnValue::Bool(v), - Self::Char(v) => ColumnValue::Char(v), - Self::Int2(v) => ColumnValue::Int2(v), - Self::Int4(v) => ColumnValue::Int4(v), - Self::Int8(v) => ColumnValue::Int8(v), - Self::Float4(v) => ColumnValue::Float4(v), - Self::Float8(v) => ColumnValue::Float8(v), - Self::Oid(v) => ColumnValue::Oid(v), - Self::Oid8(v) => ColumnValue::Oid8(v), - Self::MacAddr(v) => ColumnValue::MacAddr(v), - Self::MacAddr8(v) => ColumnValue::MacAddr8(v), - Self::Tid { block, offset } => ColumnValue::Tid { block, offset }, - Self::Date(v) => ColumnValue::Date(v), - Self::Time(v) => ColumnValue::Time(v), - Self::Timestamp(v) => ColumnValue::Timestamp(v), - Self::TimestampTz(v) => ColumnValue::TimestampTz(v), - Self::TimeTz { micros, tz_seconds } => ColumnValue::TimeTz { micros, tz_seconds }, - Self::Uuid(v) => ColumnValue::Uuid(v), - Self::Name(s) => ColumnValue::Name(s.to_owned()), - Self::Bytea(b) => ColumnValue::Bytea(b.to_vec()), - Self::Text(s) => ColumnValue::Text(s.to_owned()), - Self::Numeric(n) => ColumnValue::Numeric(n.clone()), - Self::Inet(v) => ColumnValue::Inet(v), - Self::Interval(v) => ColumnValue::Interval(v), - Self::Json(s) => ColumnValue::Json(s.to_owned()), - Self::PgPending { type_oid, raw } => ColumnValue::PgPending { - type_oid, - raw: raw.to_vec(), - }, - Self::PgPendingText { type_oid, text } => ColumnValue::PgPendingText { - type_oid, - text: text.to_owned(), - }, - Self::ExternalToast(p) => ColumnValue::ExternalToast(p), - Self::Unsupported { type_oid, raw } => ColumnValue::Unsupported { - type_oid, - raw: raw.to_vec(), - }, + Self::Null => 0, + Self::Fixed(b) => b.len(), + Self::External(_) => std::mem::size_of::(), + Self::Varlena(b) | Self::Cstring(b) | Self::BadTag(b) | Self::Text(b) => b.len(), } } + + pub fn fixed(le: &[u8]) -> Self { + Self::Fixed(SmallVec::from_slice(le)) + } +} + +/// Cells in their on-disk form, for tests to state tuples +#[cfg(any(test, feature = "test-support"))] +pub mod build { + use super::Datum; + use crate::decode::heap_decoder::ToastPointer; + + pub fn null() -> Datum { + Datum::Null + } + + pub fn bool_(v: bool) -> Datum { + Datum::fixed(&[u8::from(v)]) + } + + pub fn char_(v: i8) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn int2(v: i16) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn int4(v: i32) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn int8(v: i64) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn float4(v: f32) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn float8(v: f64) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn oid(v: u32) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn oid8(v: u64) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + /// Days since 2000-01-01 + pub fn date(v: i32) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + /// `time` or `timestamp[tz]` microseconds + pub fn micros(v: i64) -> Datum { + Datum::fixed(&v.to_le_bytes()) + } + + pub fn timetz(micros: i64, tz_seconds: i32) -> Datum { + let mut b = micros.to_le_bytes().to_vec(); + b.extend_from_slice(&tz_seconds.to_le_bytes()); + Datum::fixed(&b) + } + + pub fn uuid(v: [u8; 16]) -> Datum { + Datum::fixed(&v) + } + + pub fn name(v: impl AsRef) -> Datum { + let mut b = v.as_ref().as_bytes().to_vec(); + b.resize(64, 0); + Datum::fixed(&b) + } + + pub fn text(v: impl AsRef) -> Datum { + Datum::Varlena(v.as_ref().as_bytes().to_vec()) + } + + pub fn bytes(v: impl AsRef<[u8]>) -> Datum { + Datum::Varlena(v.as_ref().to_vec()) + } + + pub fn numeric(v: impl AsRef) -> Datum { + Datum::Varlena(crate::decode::codecs::numeric_from_text(v.as_ref()).expect("numeric text")) + } + + pub fn external(p: ToastPointer) -> Datum { + Datum::External(p) + } + + /// `typoutput` text + pub fn out_text(v: impl AsRef) -> Datum { + Datum::Text(v.as_ref().as_bytes().to_vec()) + } } diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index 7001ac4e..74177a32 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -41,7 +41,7 @@ //! | `relreplident` | old payload | this module's behaviour | //! |---|---|---| //! | `Full` (`'f'`) | every non-dropped column | decode every column; `old` populated | -//! | `UsingIndex` (`'i'`) | indexed columns only, others written as NULL via bitmap | decode all; non-indexed columns surface as `Some(ColumnValue::Null)` | +//! | `UsingIndex` (`'i'`) | indexed columns only, others written as NULL via bitmap | decode all; non-indexed columns surface as `Some(Datum::Null)` | //! | `Default` (`'d'`) with PK | PK columns when `XLH_UPDATE_CONTAINS_OLD_KEY` set | same as `UsingIndex` shape, with PK attnums populated | //! | `Default` (`'d'`) no PK | nothing (PG never sets OLD_KEY) | `old = None` | //! | `Nothing` (`'n'`) | empty | `old = None`; partial = true noted in stats | @@ -63,12 +63,9 @@ //! downstream by design. Every output stamped with `xid` so xact buffer //! can key on it. -use std::borrow::Cow; -use std::string::FromUtf8Error; - use smallvec::{SmallVec, smallvec}; -use crate::decode::datum::{DatumRef, ValueRef}; +use crate::decode::datum::{Datum, DatumRef}; use thiserror::Error; use walrus::pg::walparser::{RelFileNode, RmId, XLogRecord}; @@ -205,124 +202,6 @@ pub enum DecodeError { BadPrefixSuffix { need: usize, have: usize }, } -/// Tier 1/2 decoded value space. Out-of-matrix types surface as -/// [`ColumnValue::Unsupported`] or [`ColumnValue::ExternalToast`]. -#[derive(Debug, Clone, PartialEq)] -pub enum ColumnValue { - Null, - Bool(bool), - /// `char` (typname='char'). PG stores as `int8`; surface `i8` to - /// distinguish from `int2`. - Char(i8), - Int2(i16), - Int4(i32), - Int8(i64), - Float4(f32), - Float8(f64), - /// `oid`, `xid`, `cid` - Oid(u32), - /// `oid8`, `xid8`, `pg_lsn` - Oid8(u64), - MacAddr([u8; 6]), - MacAddr8([u8; 8]), - Tid { - block: u32, - offset: u16, - }, - /// `date`, days since PG epoch 2000-01-01, negative for pre-epoch. - Date(i32), - /// `time`, microseconds since midnight. Legacy float8 storage - /// (`--disable-integer-datetimes`) removed in PG 10. - Time(i64), - /// `timestamp`, microseconds since PG epoch 2000-01-01 00:00:00 UTC. - Timestamp(i64), - /// `timestamptz`, same storage as `timestamp`; tz is presentation-only - /// in PG, bytes identical. - TimestampTz(i64), - /// `timetz`, `time` storage + 4-byte UTC offset (seconds, negative for - /// east of UTC per PG sign convention). - TimeTz { - micros: i64, - tz_seconds: i32, - }, - /// `uuid`, 16 raw bytes network byte order on disk (PG `uuid_send`, memcpy). - Uuid([u8; 16]), - /// `name`, NUL-padded NAMEDATALEN(64) C string, no varlena header; - /// surface trimmed. - Name(String), - Bytea(Vec), - /// `text`/`varchar`/`bpchar`, varlena + UTF-8 decode. Invalid UTF-8 - /// surfaces as [`ColumnValue::Bytea`]. - Text(String), - /// `numeric` Tier 2. PG-text form for finite; NaN/Infinity carry their - /// flag. Emitter maps to CH `String` (PG precision is per-row, no fixed - /// CH `Decimal` fits without operator config). - Numeric(crate::decode::codecs::NumericKind), - /// `inet`/`cidr` Tier 2. Emit via [`crate::decode::codecs::InetValue::to_text`]. - Inet(crate::decode::codecs::InetValue), - /// `interval` Tier 2, 16-byte fixed-width (months, days, micros). - Interval(crate::decode::codecs::IntervalValue), - /// JSON document text: `json` is varlena text on disk and passes - /// through unchanged, `jsonb` renders through - /// [`crate::decode::codecs::decode_jsonb`]. - Json(String), - /// Unhandled on-disk Datum body, varlena header stripped - PgPending { - type_oid: u32, - raw: Vec, - }, - /// Attribute default text requiring `typinput`, not Datum reconstruction - PgPendingText { - type_oid: u32, - text: String, - }, - /// On-disk TOAST pointer; xact buffer's TOAST reassembly dereferences. - ExternalToast(ToastPointer), - /// Type OID outside decoder matrix, carries raw bytes. - Unsupported { - type_oid: u32, - raw: Vec, - }, -} - -impl ColumnValue { - /// Approximate in-memory payload bytes, to bound the decode pool's - /// coalescing buffer. Detoast has already run by route time so - /// `ExternalToast` pointers are resolved into `Text`/`Bytea`. Exhaustive - /// so a new variant forces a width. - pub fn approx_bytes(&self) -> usize { - use crate::decode::codecs::NumericKind; - match self { - ColumnValue::Null => 0, - ColumnValue::Bool(_) | ColumnValue::Char(_) => 1, - ColumnValue::Int2(_) => 2, - ColumnValue::MacAddr(_) | ColumnValue::Tid { .. } => 6, - ColumnValue::Int4(_) - | ColumnValue::Float4(_) - | ColumnValue::Oid(_) - | ColumnValue::Date(_) => 4, - ColumnValue::Int8(_) - | ColumnValue::Float8(_) - | ColumnValue::Oid8(_) - | ColumnValue::MacAddr8(_) - | ColumnValue::Time(_) - | ColumnValue::Timestamp(_) - | ColumnValue::TimestampTz(_) => 8, - ColumnValue::TimeTz { .. } => 12, - ColumnValue::Uuid(_) => 16, - ColumnValue::Interval(_) => 16, - ColumnValue::Inet(_) => 19, - ColumnValue::Numeric(NumericKind::Finite(s)) => s.len(), - ColumnValue::Numeric(_) => 0, - ColumnValue::Name(s) | ColumnValue::Text(s) | ColumnValue::Json(s) => s.len(), - ColumnValue::Bytea(b) => b.len(), - ColumnValue::PgPending { raw, .. } | ColumnValue::Unsupported { raw, .. } => raw.len(), - ColumnValue::PgPendingText { text, .. } => text.len(), - ColumnValue::ExternalToast(_) => 16, - } - } -} - /// On-disk TOAST pointer (PG `varatt.h` `struct varatt_external`), 16 bytes, /// unaligned in source tuple. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -354,7 +233,7 @@ impl DecodedHeap { t.columns .iter() .flatten() - .map(ColumnValue::approx_bytes) + .map(Datum::approx_bytes) .sum::() }) }; @@ -424,15 +303,15 @@ pub struct CommittedTuple { pub struct DecodedTuple { /// Indexed by attnum-1. `None` = absent from WAL (PG UPDATE prefix/suffix /// compression skipped it, or this `relreplident` shape never carries it). - /// `Some(ColumnValue::Null)` is explicit NULL (bitmap bit clear). - pub columns: Vec>, + /// `Some(Datum::Null)` is explicit NULL (bitmap bit clear). + pub columns: Vec>, /// True iff a column was elided by PG prefix/suffix compression; xact /// buffer backfills from previous tuple image. pub partial: bool, } -/// Builds one tuple image from walked attributes: owned cells for -/// [`DecodedTuple`], borrowed datums for [`TupleView`] +/// Builds one tuple image from walked attributes: owned datums for +/// [`DecodedTuple`], borrowed for [`TupleView`] pub trait TupleBuild<'a>: Sized { fn with_capacity(n: usize) -> Self; fn push(&mut self, att: &RelAttr, datum: DatumRef<'a>); @@ -449,7 +328,7 @@ impl<'a> TupleBuild<'a> for DecodedTuple { #[inline(always)] fn push(&mut self, att: &RelAttr, datum: DatumRef<'a>) { - self.columns.push(datum_value(att, datum)); + self.columns.push(resolve(att, datum)); } fn set_partial(&mut self) { @@ -484,14 +363,14 @@ impl<'a> TupleBuild<'a> for TupleView<'a> { #[cfg(test)] impl TupleView<'_> { - /// Owned cells, as [`decode_heap_record`] builds them + /// Owned datums, as [`decode_heap_record`] builds them pub fn to_tuple(&self, rel: &RelDescriptor) -> DecodedTuple { DecodedTuple { columns: self .datums .iter() .zip(&rel.attributes) - .map(|(d, att)| datum_value(att, *d)) + .map(|(d, att)| resolve(att, *d)) .collect(), partial: self.partial, } @@ -898,55 +777,43 @@ pub fn scan_block_data(block_data: &[u8], rel: &RelDescriptor) -> Result]) -> Vec { +pub fn form_block_data(attrs: &[RelAttr], values: &[Option]) -> Vec { let natts = values.len(); - let has_null = values - .iter() - .any(|v| matches!(v, None | Some(ColumnValue::Null))); + let has_null = values.iter().any(|v| matches!(v, None | Some(Datum::Null))); let bitmap_len = if has_null { natts.div_ceil(8) } else { 0 }; let hoff = (SIZE_OF_HEAP_TUPLE_HEADER + bitmap_len).next_multiple_of(8); let mut bitmap = vec![0u8; bitmap_len]; let mut data = Vec::new(); - let varlena = |body: &[u8]| { - if body.len() < 127 { - let mut v = vec![(((body.len() + 1) << 1) | 1) as u8]; - v.extend_from_slice(body); - (v, 1) - } else { - let mut v = (((body.len() + 4) as u32) << 2).to_le_bytes().to_vec(); - v.extend_from_slice(body); - (v, 4) - } - }; for (i, v) in values.iter().enumerate() { + let align_of = |len: usize| { + attrs + .get(i) + .map_or(if len.is_power_of_two() { len.min(8) } else { 1 }, |a| { + align_bytes(a.type_align) + }) + }; let (bytes, align) = match v { - None | Some(ColumnValue::Null) => continue, - Some(ColumnValue::Bool(b)) => (vec![u8::from(*b)], 1), - Some(ColumnValue::Char(c)) => (vec![*c as u8], 1), - Some(ColumnValue::Int2(n)) => (n.to_le_bytes().to_vec(), 2), - Some(ColumnValue::Int4(n) | ColumnValue::Date(n)) => (n.to_le_bytes().to_vec(), 4), - Some(ColumnValue::Oid(n)) => (n.to_le_bytes().to_vec(), 4), - Some(ColumnValue::Float4(f)) => (f.to_le_bytes().to_vec(), 4), - Some( - ColumnValue::Int8(n) - | ColumnValue::Time(n) - | ColumnValue::Timestamp(n) - | ColumnValue::TimestampTz(n), - ) => (n.to_le_bytes().to_vec(), 8), - Some(ColumnValue::Oid8(n)) => (n.to_le_bytes().to_vec(), 8), - Some(ColumnValue::Float8(f)) => (f.to_le_bytes().to_vec(), 8), - Some(ColumnValue::Uuid(u)) => (u.to_vec(), 1), - Some(ColumnValue::Text(t) | ColumnValue::Json(t)) => varlena(t.as_bytes()), - Some( - ColumnValue::Bytea(b) - | ColumnValue::PgPending { raw: b, .. } - | ColumnValue::Unsupported { raw: b, .. }, - ) => varlena(b), - Some(ColumnValue::ExternalToast(p)) => { + None | Some(Datum::Null) => continue, + Some(Datum::Fixed(b)) => (b.to_vec(), align_of(b.len())), + Some(Datum::Varlena(b) | Datum::BadTag(b) | Datum::Text(b)) if b.len() < 127 => { + let mut v = vec![(((b.len() + 1) << 1) | 1) as u8]; + v.extend_from_slice(b); + (v, 1) + } + Some(Datum::Varlena(b) | Datum::BadTag(b) | Datum::Text(b)) => { + let mut v = (((b.len() + 4) as u32) << 2).to_le_bytes().to_vec(); + v.extend_from_slice(b); + (v, align_of(4).max(4)) + } + Some(Datum::Cstring(b)) => { + let mut v = b.clone(); + v.push(0); + (v, 1) + } + Some(Datum::External(p)) => { let mut v = vec![0x01, 18]; for word in [ p.va_rawsize as u32, @@ -958,7 +825,6 @@ pub fn form_block_data(values: &[Option]) -> Vec { } (v, 1) } - Some(other) => unimplemented!("form_block_data: {other:?}"), }; if has_null { bitmap[i / 8] |= 1 << (i % 8); @@ -1145,111 +1011,123 @@ fn walk_tuple<'a, T: TupleBuild<'a>>( Ok(()) } -/// Resolve a column's fast default ([`RelAttr::missing_default`]) to a value; -/// `Null` when there is none. Tier-3 stays a pending value the oracle resolves. -pub fn missing_value_for(att: &RelAttr) -> ColumnValue { +/// Column's fast default ([`RelAttr::missing_default`]) as stored or as +/// `typoutput` text; `Null` when there is none +pub fn missing_datum(att: &RelAttr) -> DatumRef<'_> { match &att.missing_default { - Some(MissingDefault::Raw(arr)) => missing_value_from_array(att, arr), - Some(MissingDefault::Text(text)) => missing_value_from_text(att.type_oid, text), - None => ColumnValue::Null, + Some(MissingDefault::Raw(arr)) => missing_array_element(att, arr), + Some(MissingDefault::Text(text)) => DatumRef::Text(text.as_bytes()), + None => DatumRef::Null, } } -/// Raw and text spell one default differently but resolve to one value, so -/// compare by value; tier-3 stays pending, so equal source type counts as equal. +/// Raw and text spell one default differently, so compare parsed datums. +/// Text no local parser reads stays pending; equal source type counts as equal pub fn missing_defaults_equivalent(a: &RelAttr, b: &RelAttr) -> bool { - use ColumnValue::{PgPending, PgPendingText}; - match (missing_value_for(a), missing_value_for(b)) { - ( - PgPending { type_oid: x, .. } | PgPendingText { type_oid: x, .. }, - PgPending { type_oid: y, .. } | PgPendingText { type_oid: y, .. }, - ) => x == y, - (l, r) => l == r, - } + #[derive(PartialEq)] + enum Image { + Value(Option), + Pending(u32), + } + let image = |att: &RelAttr| match missing_datum(att) { + DatumRef::Text(text) => match text_datum(att.type_oid, text) { + TextDatum::Datum(d) => Image::Value(Some(d)), + TextDatum::Null => Image::Value(Some(Datum::Null)), + TextDatum::Opaque if is_text_type(att.type_oid) => { + Image::Value(Some(Datum::Varlena(text.to_vec()))) + } + TextDatum::Opaque => Image::Pending(att.type_oid), + }, + // Text-typed defaults compare as their text + d if local_matrix_covers(att.type_oid, att.type_len) => { + Image::Value(resolve(att, d).map(|d| { + match (d, att.type_oid) { + (Datum::Fixed(b), NAMEOID) => { + let end = b.iter().position(|&c| c == 0).unwrap_or(b.len()); + Datum::Varlena(b[..end].to_vec()) + } + (Datum::Varlena(b), JSONBOID) => crate::decode::codecs::decode_jsonb(&b) + .map_or(Datum::BadTag(b), |text| Datum::Varlena(text.into_bytes())), + (d, _) => d, + } + })) + } + DatumRef::Null => Image::Value(Some(Datum::Null)), + _ => Image::Pending(att.type_oid), + }; + image(a) == image(b) } -fn missing_value_from_array(att: &RelAttr, arr: &[u8]) -> ColumnValue { +/// `attmissingval[1]`, an `ArrayType` with one non-null element +fn missing_array_element<'a>(att: &RelAttr, arr: &'a [u8]) -> DatumRef<'a> { // ArrayType: vl_len_(4) ndim(4) dataoffset(4) elemtype(4) dims[] lbounds[] [nulls] data - let decode = || -> Option { - if arr.len() < 16 { - return None; - } - let ndim = i32::from_le_bytes(arr[4..8].try_into().ok()?); - let dataoffset = i32::from_le_bytes(arr[8..12].try_into().ok()?); + let element = || -> Option> { + let ndim = i32::from_le_bytes(arr.get(4..8)?.try_into().ok()?); + let dataoffset = i32::from_le_bytes(arr.get(8..12)?.try_into().ok()?); if ndim != 1 { return None; } let abs = if dataoffset != 0 { - dataoffset as usize + usize::try_from(dataoffset).ok()? } else { // MAXALIGN(16 + 8*ndim), no null bitmap (16 + 8 * ndim as usize).next_multiple_of(8) }; - decode_one_value(att, arr, abs).ok().map(|(v, _)| v) + read_datum(att, arr, abs).ok().map(|(d, _)| d) }; - decode().unwrap_or(ColumnValue::Null) + element().unwrap_or(DatumRef::Null) } -/// `attmissingval` element from its `anyarray_out` text form (SQL/mirror path). -pub(crate) fn missing_value_from_text(type_oid: u32, text: &str) -> ColumnValue { +fn is_text_type(type_oid: u32) -> bool { + matches!( + type_oid, + TEXTOID | VARCHAROID | BPCHAROID | NAMEOID | JSONOID | JSONBOID + ) +} + +/// `typoutput` text read back into a stored datum +#[derive(Debug, PartialEq)] +pub enum TextDatum { + Datum(Datum), + /// Text the type's parser rejects + Null, + /// No local parser; String targets take the text as is + Opaque, +} + +/// Parse `typoutput` text of types with a local reader into stored form +pub fn text_datum(type_oid: u32, text: &[u8]) -> TextDatum { + let Ok(text) = simdutf8::basic::from_utf8(text) else { + return TextDatum::Opaque; + }; + fn fixed>(b: Option) -> TextDatum { + b.map_or(TextDatum::Null, |b| { + TextDatum::Datum(Datum::fixed(b.as_ref())) + }) + } match type_oid { - BOOLOID => match text { - "t" | "true" | "yes" | "on" | "1" => ColumnValue::Bool(true), - _ => ColumnValue::Bool(false), - }, - CHAROID => text - .as_bytes() - .first() - .map(|b| ColumnValue::Char(*b as i8)) - .unwrap_or(ColumnValue::Null), - INT2OID => text - .parse::() - .map(ColumnValue::Int2) - .unwrap_or(ColumnValue::Null), - INT4OID => text - .parse::() - .map(ColumnValue::Int4) - .unwrap_or(ColumnValue::Null), - INT8OID => text - .parse::() - .map(ColumnValue::Int8) - .unwrap_or(ColumnValue::Null), - FLOAT4OID => text - .parse::() - .map(ColumnValue::Float4) - .unwrap_or(ColumnValue::Null), - FLOAT8OID => text - .parse::() - .map(ColumnValue::Float8) - .unwrap_or(ColumnValue::Null), - OIDOID | XIDOID | CIDOID => text - .parse::() - .map(ColumnValue::Oid) - .unwrap_or(ColumnValue::Null), - OID8OID | XID8OID => text - .parse::() - .map(ColumnValue::Oid8) - .unwrap_or(ColumnValue::Null), - PG_LSNOID => crate::pg::parse_pg_lsn(text) - .map(ColumnValue::Oid8) - .unwrap_or(ColumnValue::Null), - TEXTOID | VARCHAROID | BPCHAROID | NAMEOID => ColumnValue::Text(text.to_owned()), - MACADDROID => crate::decode::codecs::parse_macaddr(text) - .map(ColumnValue::MacAddr) - .unwrap_or(ColumnValue::Null), - MACADDR8OID => crate::decode::codecs::parse_macaddr(text) - .map(ColumnValue::MacAddr8) - .unwrap_or(ColumnValue::Null), - TIDOID => crate::decode::codecs::parse_tid(text) - .map(|(block, offset)| ColumnValue::Tid { block, offset }) - .unwrap_or(ColumnValue::Null), - // `jsonb` arrives as `jsonb_out` text, already what the emitter ships - JSONOID | JSONBOID => ColumnValue::Json(text.to_owned()), - // attmissingval text requires typinput - _ => ColumnValue::PgPendingText { - type_oid, - text: text.to_owned(), - }, + BOOLOID => fixed(Some([u8::from(matches!( + text, + "t" | "true" | "yes" | "on" | "1" + ))])), + CHAROID => fixed(text.as_bytes().first().map(|b| [*b])), + INT2OID => fixed(text.parse::().ok().map(i16::to_le_bytes)), + INT4OID => fixed(text.parse::().ok().map(i32::to_le_bytes)), + INT8OID => fixed(text.parse::().ok().map(i64::to_le_bytes)), + FLOAT4OID => fixed(text.parse::().ok().map(f32::to_le_bytes)), + FLOAT8OID => fixed(text.parse::().ok().map(f64::to_le_bytes)), + OIDOID | XIDOID | CIDOID => fixed(text.parse::().ok().map(u32::to_le_bytes)), + OID8OID | XID8OID => fixed(text.parse::().ok().map(u64::to_le_bytes)), + PG_LSNOID => fixed(crate::pg::parse_pg_lsn(text).ok().map(u64::to_le_bytes)), + MACADDROID => fixed(crate::decode::codecs::parse_macaddr::<6>(text)), + MACADDR8OID => fixed(crate::decode::codecs::parse_macaddr::<8>(text)), + TIDOID => fixed( + crate::decode::codecs::parse_tid(text) + .map(|(block, offset)| crate::decode::codecs::encode_tid(block, offset)), + ), + NUMERICOID => crate::decode::codecs::numeric_from_text(text) + .map_or(TextDatum::Null, |b| TextDatum::Datum(Datum::Varlena(b))), + _ => TextDatum::Opaque, } } @@ -1265,6 +1143,16 @@ fn decoded_size_or_skip(att: &RelAttr) -> Option { /// MAXALIGN dispatch over `pg_type.typalign`, for in-prefix / past-EOF where /// we can't peek to detect a short varlena header (`att_align_pointer`). +#[cfg(any(test, feature = "test-support"))] +fn align_bytes(attalign: char) -> usize { + match attalign { + 's' => 2, + 'i' => 4, + 'd' => 8, + _ => 1, + } +} + fn align_for(cur: usize, attalign: char) -> usize { match attalign { 's' => align_up(cur, 2), @@ -1354,134 +1242,40 @@ pub(crate) fn read_datum<'a>( )) } -/// Returns `(value, bytes_consumed)`, see [`read_datum`] -pub(crate) fn decode_one_value( - att: &RelAttr, - buf: &[u8], - abs: usize, -) -> Result<(ColumnValue, usize), Truncated> { - let (datum, len) = read_datum(att, buf, abs)?; - Ok((datum_value(att, datum).unwrap_or(ColumnValue::Null), len)) -} - -/// Owned cell for `datum`, `None` when absent from WAL image -#[inline(always)] -pub fn datum_value(att: &RelAttr, datum: DatumRef<'_>) -> Option { +/// Owned copy of `datum`, `None` when absent from WAL image. Fast defaults +/// resolve, stored where a local reader parses their text, and inline +/// compressed bodies inflate +pub fn resolve(att: &RelAttr, datum: DatumRef<'_>) -> Option { Some(match datum { DatumRef::Absent => return None, - DatumRef::Null | DatumRef::Dropped => ColumnValue::Null, - DatumRef::Missing => missing_value_for(att), - DatumRef::Fixed(body) => fixed_value(att, body), - DatumRef::Varlena(body) => varlena_to_value(att.type_oid, Cow::Borrowed(body)), + DatumRef::Null | DatumRef::Dropped => Datum::Null, + DatumRef::Missing => match missing_datum(att) { + DatumRef::Text(t) => match text_datum(att.type_oid, t) { + TextDatum::Datum(d) => d, + TextDatum::Null => Datum::Null, + TextDatum::Opaque => Datum::Text(t.to_vec()), + }, + d => return resolve(att, d), + }, + DatumRef::Fixed(b) => Datum::fixed(b), + DatumRef::Varlena(b) => Datum::Varlena(b.to_vec()), DatumRef::Compressed(datum) => { - // Reader checked datum spans header and va_tcinfo - let tcinfo = u32::from_le_bytes(datum[4..8].try_into().unwrap()); - let method = (tcinfo >> VARLENA_EXTSIZE_BITS) as u8; - let raw_len = (tcinfo & VARLENA_EXTSIZE_MASK) as usize; - decompress_varlena(method, &datum[8..], raw_len) - .map(|out| varlena_to_value(att.type_oid, Cow::Owned(out))) - .unwrap_or_else(|| ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: datum.to_vec(), - }) + inflate(datum).map_or_else(|| Datum::BadTag(datum.to_vec()), Datum::Varlena) } - DatumRef::External(p) => ColumnValue::ExternalToast(p), - DatumRef::Cstring(body) => simdutf8::basic::from_utf8(body) - .map(|s| ColumnValue::Text(s.to_owned())) - .unwrap_or_else(|_| ColumnValue::Bytea(body.to_vec())), - DatumRef::BadTag(raw) => ColumnValue::Unsupported { - type_oid: att.type_oid, - raw: raw.to_vec(), - }, + DatumRef::External(p) => Datum::External(p), + DatumRef::Cstring(b) => Datum::Cstring(b.to_vec()), + DatumRef::BadTag(b) => Datum::BadTag(b.to_vec()), + DatumRef::Text(b) => Datum::Text(b.to_vec()), }) } -/// Datum whose value only [`datum_value`] builds: fast defaults, compressed -/// bodies, `numeric` and `jsonb` -#[derive(Debug)] -pub struct AllocatingDecode; - -/// Borrowed value for `datum`, `Ok(None)` when absent from WAL image -#[inline] -pub fn datum_view<'a>( - att: &RelAttr, - datum: DatumRef<'a>, -) -> Result>, AllocatingDecode> { - Ok(Some(match datum { - DatumRef::Absent => return Ok(None), - DatumRef::Null | DatumRef::Dropped => ValueRef::Null, - DatumRef::Fixed(body) => fixed_view(att, body), - DatumRef::Varlena(body) => varlena_view(att.type_oid, body)?, - DatumRef::External(p) => ValueRef::ExternalToast(p), - DatumRef::Cstring(body) => match simdutf8::basic::from_utf8(body) { - Ok(s) => ValueRef::Text(s), - Err(_) => ValueRef::Bytea(body), - }, - DatumRef::BadTag(raw) => ValueRef::Unsupported { - type_oid: att.type_oid, - raw, - }, - DatumRef::Missing | DatumRef::Compressed(_) => return Err(AllocatingDecode), - })) -} - -/// Fixed-width value from its on-disk bytes. Inlined so callers construct -/// in place: returning a 40-byte enum through memory then reloading it wide -/// stalls store forwarding, which dominated fixed-width decode -#[inline(always)] -fn fixed_value(att: &RelAttr, body: &[u8]) -> ColumnValue { - fixed_view(att, body).to_owned_value() -} - -#[inline(always)] -fn fixed_view<'a>(att: &RelAttr, body: &'a [u8]) -> ValueRef<'a> { - let unsupported = || ValueRef::Unsupported { - type_oid: att.type_oid, - raw: body, - }; - match att.type_oid { - BOOLOID => ValueRef::Bool(body[0] != 0), - CHAROID => ValueRef::Char(body[0] as i8), - INT2OID => ValueRef::Int2(i16::from_le_bytes(body.try_into().unwrap())), - INT4OID => ValueRef::Int4(i32::from_le_bytes(body.try_into().unwrap())), - // `money` counts minor units, scale set by source `lc_monetary` - INT8OID | MONEYOID => ValueRef::Int8(i64::from_le_bytes(body.try_into().unwrap())), - FLOAT4OID => ValueRef::Float4(f32::from_le_bytes(body.try_into().unwrap())), - FLOAT8OID => ValueRef::Float8(f64::from_le_bytes(body.try_into().unwrap())), - OIDOID | XIDOID | CIDOID => ValueRef::Oid(u32::from_le_bytes(body.try_into().unwrap())), - OID8OID | XID8OID | PG_LSNOID => { - ValueRef::Oid8(u64::from_le_bytes(body.try_into().unwrap())) - } - // NUL-padded to NAMEDATALEN - NAMEOID => { - let end = body.iter().position(|&b| b == 0).unwrap_or(body.len()); - match simdutf8::basic::from_utf8(&body[..end]) { - Ok(s) => ValueRef::Name(s), - Err(_) => unsupported(), - } - } - MACADDROID => ValueRef::MacAddr(body.try_into().unwrap()), - MACADDR8OID => ValueRef::MacAddr8(body.try_into().unwrap()), - TIDOID => { - let (block, offset) = crate::decode::codecs::decode_tid(body.try_into().unwrap()); - ValueRef::Tid { block, offset } - } - DATEOID => ValueRef::Date(i32::from_le_bytes(body.try_into().unwrap())), - TIMEOID => ValueRef::Time(i64::from_le_bytes(body.try_into().unwrap())), - TIMESTAMPOID => ValueRef::Timestamp(i64::from_le_bytes(body.try_into().unwrap())), - TIMESTAMPTZOID => ValueRef::TimestampTz(i64::from_le_bytes(body.try_into().unwrap())), - TIMETZOID => { - // 12 bytes: 8-byte time + 4-byte tz offset - let micros = i64::from_le_bytes(body[..8].try_into().unwrap()); - let tz_seconds = i32::from_le_bytes(body[8..12].try_into().unwrap()); - ValueRef::TimeTz { micros, tz_seconds } - } - UUIDOID => ValueRef::Uuid(body.try_into().unwrap()), - INTERVALOID => crate::decode::codecs::decode_interval(body) - .map(ValueRef::Interval) - .unwrap_or_else(|_| unsupported()), - _ => unsupported(), - } +/// Body of an inline compressed varlena, `None` on a corrupt stream +pub(crate) fn inflate(datum: &[u8]) -> Option> { + // Reader checked datum spans header and va_tcinfo + let tcinfo = u32::from_le_bytes(datum.get(4..8)?.try_into().ok()?); + let method = (tcinfo >> VARLENA_EXTSIZE_BITS) as u8; + let raw_len = (tcinfo & VARLENA_EXTSIZE_MASK) as usize; + decompress_varlena(method, &datum[8..], raw_len) } /// PG `va_tcinfo`/`va_extinfo`: codec method in the top 2 bits, size in the low 30. @@ -1605,84 +1399,6 @@ fn read_varlena(buf: &[u8], abs: usize) -> Result<(DatumRef<'_>, usize), Truncat Ok((DatumRef::Varlena(&buf[abs + 4..abs + total]), total)) } -/// Map a varlena body (header stripped, decompressed) to a typed value by the -/// column's type OID. Shared by the inline decoder (`Cow::Borrowed` into the -/// WAL buffer) and the TOAST reassembler (`Cow::Owned`, so `detoasted_value` -/// moves the reassembled buffer into the value instead of copying it). A -/// detoasted value resolves identically to an inline one — Tier 3 types land -/// as `PgPending`, resolved at emit by the oracle. -pub(crate) fn varlena_to_value(type_oid: u32, body: Cow<[u8]>) -> ColumnValue { - match type_oid { - BYTEAOID => ColumnValue::Bytea(body.into_owned()), - TEXTOID | VARCHAROID | BPCHAROID | JSONOID => { - // Owned arm keeps detoasted buffer rather than copy it - let text = match body { - Cow::Borrowed(b) => simdutf8::basic::from_utf8(b) - .map(str::to_owned) - .map_err(|_| b.to_vec()), - Cow::Owned(v) => String::from_utf8(v).map_err(FromUtf8Error::into_bytes), - }; - // PG validates UTF-8 on input; surface bytea not crash if ever invalid - match text { - Ok(s) if type_oid == JSONOID => ColumnValue::Json(s), - Ok(s) => ColumnValue::Text(s), - Err(b) => ColumnValue::Bytea(b), - } - } - NUMERICOID => match crate::decode::codecs::decode_numeric(&body) { - Ok(v) => ColumnValue::Numeric(v), - Err(_) => ColumnValue::Unsupported { - type_oid, - raw: body.into_owned(), - }, - }, - INETOID | CIDROID => match crate::decode::codecs::decode_inet(&body, type_oid == CIDROID) { - Ok(v) => ColumnValue::Inet(v), - Err(_) => ColumnValue::Unsupported { - type_oid, - raw: body.into_owned(), - }, - }, - JSONBOID => match crate::decode::codecs::decode_jsonb(&body) { - Ok(s) => ColumnValue::Json(s), - Err(_) => ColumnValue::Unsupported { - type_oid, - raw: body.into_owned(), - }, - }, - // Tier 3 deferred: range types, arrays (typcategory='A'), - // tsvector etc. Carries full on-disk body so the SQL bridge - // reconstructs the varlena Datum; walshadow extension resolves at emit - _ => ColumnValue::PgPending { - type_oid, - raw: body.into_owned(), - }, - } -} - -/// [`varlena_to_value`] borrowing an inline body -fn varlena_view(type_oid: u32, body: &[u8]) -> Result, AllocatingDecode> { - Ok(match type_oid { - BYTEAOID => ValueRef::Bytea(body), - TEXTOID | VARCHAROID | BPCHAROID | JSONOID => match simdutf8::basic::from_utf8(body) { - Ok(s) if type_oid == JSONOID => ValueRef::Json(s), - Ok(s) => ValueRef::Text(s), - Err(_) => ValueRef::Bytea(body), - }, - INETOID | CIDROID => crate::decode::codecs::decode_inet(body, type_oid == CIDROID) - .map(ValueRef::Inet) - .unwrap_or(ValueRef::Unsupported { - type_oid, - raw: body, - }), - NUMERICOID | JSONBOID => return Err(AllocatingDecode), - _ => ValueRef::PgPending { - type_oid, - raw: body, - }, - }) -} - /// Match decoder paths producing typed values pub(crate) fn local_matrix_covers(type_oid: u32, type_len: i16) -> bool { match type_len { @@ -1806,6 +1522,7 @@ pub(crate) fn short_varlena(content: &[u8]) -> Vec { #[cfg(test)] mod tests { use super::*; + use crate::decode::datum::{Datum, build}; use crate::schema::RelName; use walrus::pg::walparser::{ BlockLocation, XLogRecordBlock, XLogRecordBlockHeader, XLogRecordHeader, @@ -1859,21 +1576,13 @@ mod tests { #[test] fn approx_bytes_counts_scalars_and_varlena() { - assert_eq!(ColumnValue::Null.approx_bytes(), 0); - assert_eq!(ColumnValue::Int2(0).approx_bytes(), 2); - assert_eq!(ColumnValue::Int4(0).approx_bytes(), 4); - assert_eq!(ColumnValue::Int8(0).approx_bytes(), 8); - assert_eq!(ColumnValue::Uuid([0; 16]).approx_bytes(), 16); - assert_eq!(ColumnValue::Text("héllo".into()).approx_bytes(), 6); - assert_eq!(ColumnValue::Bytea(vec![0u8; 100]).approx_bytes(), 100); - assert_eq!( - ColumnValue::PgPending { - type_oid: 1, - raw: vec![0u8; 7], - } - .approx_bytes(), - 7 - ); + assert_eq!(Datum::Null.approx_bytes(), 0); + assert_eq!(build::int2(0).approx_bytes(), 2); + assert_eq!(build::int4(0).approx_bytes(), 4); + assert_eq!(build::int8(0).approx_bytes(), 8); + assert_eq!(build::uuid([0; 16]).approx_bytes(), 16); + assert_eq!(build::text("héllo").approx_bytes(), 6); + assert_eq!(build::bytes(vec![0u8; 100]).approx_bytes(), 100); let heap = DecodedHeap { rfn: RelFileNode { @@ -1885,14 +1594,11 @@ mod tests { source_lsn: 0, op: HeapOp::Update, new: Some(DecodedTuple { - columns: vec![ - Some(ColumnValue::Int4(1)), - Some(ColumnValue::Bytea(vec![0u8; 10])), - ], + columns: vec![Some(build::int4(1)), Some(build::bytes(vec![0u8; 10]))], partial: false, }), old: Some(DecodedTuple { - columns: vec![None, Some(ColumnValue::Text("abc".into()))], + columns: vec![None, Some(build::text("abc"))], partial: false, }), }; @@ -1945,44 +1651,6 @@ mod tests { } } - #[test] - fn varlena_text_preserves_bytes_and_owned_buffers() { - let suffixes: &[&[u8]] = &[ - b"", - "é😀".as_bytes(), - &[0xff], - &[0xc0, 0xaf], - &[0xed, 0xa0, 0x80], - &[0xf0, 0x9f, 0x98], - ]; - for type_oid in [TEXTOID, VARCHAROID, BPCHAROID, JSONOID] { - for prefix in [0, 63, 64, 65, 127, 128, 129] { - for suffix in suffixes { - let mut raw = vec![b'a'; prefix]; - raw.extend_from_slice(suffix); - let expected = match String::from_utf8(raw.clone()) { - Ok(s) if type_oid == JSONOID => ColumnValue::Json(s), - Ok(s) => ColumnValue::Text(s), - Err(_) => ColumnValue::Bytea(raw.clone()), - }; - for body in [Cow::Borrowed(raw.as_slice()), Cow::Owned(raw.clone())] { - let owned = matches!(body, Cow::Owned(_)); - let ptr = body.as_ptr(); - let value = varlena_to_value(type_oid, body); - assert_eq!(value, expected); - if owned { - assert!( - matches!(&value, ColumnValue::Text(s) | ColumnValue::Json(s) if s.as_ptr() == ptr) - || matches!(&value, ColumnValue::Bytea(b) if b.as_ptr() == ptr), - "owned body reused in place" - ); - } - } - } - } - } - } - /// Every decode here also checks borrowed views build equal cells fn decode_heap_record( record: &XLogRecord, @@ -2008,43 +1676,42 @@ mod tests { owned } - fn decode_cstring(buf: &[u8], abs: usize) -> Result<(ColumnValue, usize), Truncated> { - let (datum, n) = read_cstring(buf, abs)?; - let att = rel_attr(1, "c", 0, -2, 'c'); - Ok((datum_value(&att, datum).unwrap(), n)) + /// Owned datum at `abs` and its stored length + fn decode_one_value( + att: &RelAttr, + buf: &[u8], + abs: usize, + ) -> Result<(Datum, usize), Truncated> { + let (datum, len) = read_datum(att, buf, abs)?; + Ok((resolve(att, datum).unwrap_or(Datum::Null), len)) + } + + fn missing_value_for(att: &RelAttr) -> Datum { + resolve(att, DatumRef::Missing).expect("fast default resolves") } #[test] - fn decode_cstring_text_bytea_and_truncated() { - let (v, n) = decode_cstring(b"hello\0", 0).unwrap(); - assert_eq!(v, ColumnValue::Text("hello".into())); + fn decode_cstring_bytes_and_truncated() { + let att = rel_attr(1, "c", 0, -2, 'c'); + let cstring = |s: &[u8]| Datum::Cstring(s.to_vec()); + let (v, n) = decode_one_value(&att, b"hello\0", 0).unwrap(); + assert_eq!(v, cstring(b"hello")); assert_eq!(n, 6); - let (v, n) = decode_cstring(&[0xFF, 0xFE, 0x00], 0).unwrap(); - assert_eq!(v, ColumnValue::Bytea(vec![0xFF, 0xFE])); + let (v, n) = decode_one_value(&att, &[0xFF, 0xFE, 0x00], 0).unwrap(); + assert_eq!(v, cstring(&[0xFF, 0xFE])); assert_eq!(n, 3); // non-zero offset within multi-string buffer - let (v, n) = decode_cstring(b"ab\0cd\0", 3).unwrap(); - assert_eq!(v, ColumnValue::Text("cd".into())); + let (v, n) = decode_one_value(&att, b"ab\0cd\0", 3).unwrap(); + assert_eq!(v, cstring(b"cd")); assert_eq!(n, 3); // no terminator before buffer end - let err = decode_cstring(b"hello", 0).unwrap_err(); + let err = decode_one_value(&att, b"hello", 0).unwrap_err(); assert_eq!( format!("{err:?}"), "Truncated { offset: 0, need: 1, have: 0 }" ); } - #[test] - fn decode_one_value_maps_char_to_signed_i8() { - let att = rel_attr(1, "c", CHAROID, 1, 'c'); - let (v, n) = decode_one_value(&att, &[65u8], 0).unwrap(); - assert_eq!(v, ColumnValue::Char(65)); - assert_eq!(n, 1); - // 0xFF -> i8 -1 - let (v, _) = decode_one_value(&att, &[0xFFu8], 0).unwrap(); - assert_eq!(v, ColumnValue::Char(-1)); - } - #[test] fn decode_insert_int4_int8() { let rel = descriptor( @@ -2068,8 +1735,8 @@ mod tests { let new = out.new.unwrap(); assert!(!new.partial); assert_eq!(new.columns.len(), 2); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(12345))); - assert_eq!(new.columns[1], Some(ColumnValue::Int8(999_999))); + assert_eq!(new.columns[0], Some(build::int4(12345))); + assert_eq!(new.columns[1], Some(build::int8(999_999))); } #[test] @@ -2095,8 +1762,8 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Bool(true))); - assert_eq!(new.columns[1], Some(ColumnValue::Text("hi".into()))); + assert_eq!(new.columns[0], Some(build::bool_(true))); + assert_eq!(new.columns[1], Some(build::text("hi"))); } #[test] @@ -2114,7 +1781,7 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Text("hi".into()))); + assert_eq!(new.columns[0], Some(build::text("hi"))); } fn inline_compressed_varlena(method: u8, body: &[u8], raw_len: usize) -> Vec { @@ -2136,7 +1803,7 @@ mod tests { let att = rel_attr(1, "val", TEXTOID, -1, 'i'); let (v, consumed) = decode_one_value(&att, &buf, 0).unwrap(); assert_eq!(consumed, buf.len()); - assert_eq!(v, ColumnValue::Text(text)); + assert_eq!(v, build::text(&text)); } #[test] @@ -2147,7 +1814,7 @@ mod tests { let att = rel_attr(1, "val", TEXTOID, -1, 'i'); let (v, consumed) = decode_one_value(&att, &buf, 0).unwrap(); assert_eq!(consumed, buf.len()); - assert_eq!(v, ColumnValue::Text(text)); + assert_eq!(v, build::text(&text)); } #[test] @@ -2156,13 +1823,7 @@ mod tests { let att = rel_attr(1, "val", TEXTOID, -1, 'i'); let (v, consumed) = decode_one_value(&att, &buf, 0).unwrap(); assert_eq!(consumed, buf.len()); - assert!(matches!( - v, - ColumnValue::Unsupported { - type_oid: TEXTOID, - .. - } - )); + assert!(matches!(v, Datum::BadTag(_))); } #[test] @@ -2207,9 +1868,9 @@ mod tests { let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); assert_eq!(new.columns.len(), 3); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(100))); - assert_eq!(new.columns[1], Some(ColumnValue::Null)); - assert_eq!(new.columns[2], Some(ColumnValue::Int4(300))); + assert_eq!(new.columns[0], Some(build::int4(100))); + assert_eq!(new.columns[1], Some(Datum::Null)); + assert_eq!(new.columns[2], Some(build::int4(300))); } #[test] @@ -2227,7 +1888,7 @@ mod tests { let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); assert_eq!(out.op, HeapOp::Delete); let old = out.old.unwrap(); - assert_eq!(old.columns[0], Some(ColumnValue::Int4(7777))); + assert_eq!(old.columns[0], Some(build::int4(7777))); } #[test] @@ -2256,7 +1917,7 @@ mod tests { let new = out.new.unwrap(); assert!(new.partial, "prefix-compressed UPDATE flagged partial"); assert_eq!(new.columns[0], None); - assert_eq!(new.columns[1], Some(ColumnValue::Int4(999))); + assert_eq!(new.columns[1], Some(build::int4(999))); } #[test] @@ -2274,7 +1935,7 @@ mod tests { assert_eq!(out.op, HeapOp::HotUpdate); assert!(out.old.is_none()); let new = out.new.unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(55))); + assert_eq!(new.columns[0], Some(build::int4(55))); } #[test] @@ -2290,8 +1951,8 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Null)); - assert_eq!(new.columns[1], Some(ColumnValue::Int4(8888))); + assert_eq!(new.columns[0], Some(Datum::Null)); + assert_eq!(new.columns[1], Some(build::int4(8888))); } #[test] @@ -2325,7 +1986,7 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Uuid(uuid_bytes))); + assert_eq!(new.columns[0], Some(build::uuid(uuid_bytes))); } #[test] @@ -2346,7 +2007,7 @@ mod tests { let new = out.new.unwrap(); assert_eq!( new.columns[0], - Some(ColumnValue::ExternalToast(ToastPointer { + Some(Datum::External(ToastPointer { va_rawsize: 12345, va_extinfo: 678, va_valueid: 12, @@ -2357,8 +2018,8 @@ mod tests { #[test] fn decode_unsupported_type_emits_pending() { - // tsvector is outside the local matrix; varlena fall-through yields - // PgPending preserving raw bytes (shadow extension resolves at emit) + // tsvector is outside the local matrix; its stored body survives for + // the shadow extension to convert at emit let rel = descriptor(16397, vec![rel_attr(1, "t", TSVECTOROID, -1, 'i')]); let body = b"\x01opaque"; let total = 4 + body.len(); @@ -2371,13 +2032,7 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!( - new.columns[0], - Some(ColumnValue::PgPending { - type_oid: TSVECTOROID, - raw: body.to_vec(), - }) - ); + assert_eq!(new.columns[0], Some(build::bytes(body))); } #[test] @@ -2403,9 +2058,9 @@ mod tests { let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); assert_eq!(new.columns.len(), 3); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(100))); - assert_eq!(new.columns[1], Some(ColumnValue::Int4(200))); - assert_eq!(new.columns[2], Some(ColumnValue::Int4(7))); + assert_eq!(new.columns[0], Some(build::int4(100))); + assert_eq!(new.columns[1], Some(build::int4(200))); + assert_eq!(new.columns[2], Some(build::int4(7))); } #[test] @@ -2427,7 +2082,7 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!(new.columns[2], Some(ColumnValue::Null)); + assert_eq!(new.columns[2], Some(Datum::Null)); } #[test] @@ -2448,8 +2103,8 @@ mod tests { let rec = record_with(RmId::Heap, XLOG_HEAP_INSERT, main_data, payload); let out = decode_heap_record(&rec, 0, &rel).unwrap().remove(0); let new = out.new.unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(55))); - assert_eq!(new.columns[1], Some(ColumnValue::Null)); + assert_eq!(new.columns[0], Some(build::int4(55))); + assert_eq!(new.columns[1], Some(Datum::Null)); } #[test] @@ -2516,7 +2171,7 @@ mod tests { assert_eq!(out[i].source_lsn, 0x2000); let new = out[i].new.as_ref().unwrap(); assert_eq!(new.columns.len(), 1); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(*expected))); + assert_eq!(new.columns[0], Some(build::int4(*expected))); } } @@ -2565,14 +2220,11 @@ mod tests { let cols: Vec<_> = out.iter().map(|h| h.new.clone().unwrap()).collect(); assert_eq!( cols[0].columns, - vec![ - Some(ColumnValue::Int4(7)), - Some(ColumnValue::Text("ab".into())) - ] + vec![Some(build::int4(7)), Some(build::text("ab"))] ); assert_eq!( cols[1].columns, - vec![Some(ColumnValue::Int4(8)), Some(ColumnValue::Null)] + vec![Some(build::int4(8)), Some(Datum::Null)] ); assert!(!cols[0].partial && !cols[1].partial); } @@ -2605,14 +2257,11 @@ mod tests { let out = decode_heap_record(&rec, 0, &rel).unwrap(); let first = out[0].new.as_ref().unwrap(); assert!(first.partial); - assert_eq!(first.columns, vec![Some(ColumnValue::Int4(7)), None]); + assert_eq!(first.columns, vec![Some(build::int4(7)), None]); let second = out[1].new.as_ref().unwrap(); assert_eq!( second.columns, - vec![ - Some(ColumnValue::Int4(8)), - Some(ColumnValue::Text("z".into())) - ] + vec![Some(build::int4(8)), Some(build::text("z"))] ); } @@ -2667,24 +2316,19 @@ mod tests { #[test] fn fixed_system_types_decode_values() { - let cases: [(u32, Vec, char, ColumnValue); 7] = [ - ( - XIDOID, - 7u32.to_le_bytes().to_vec(), - 'i', - ColumnValue::Oid(7), - ), + let cases: [(u32, Vec, char, Datum); 7] = [ + (XIDOID, 7u32.to_le_bytes().to_vec(), 'i', build::oid(7)), ( PG_LSNOID, 0x16_B374_D848u64.to_le_bytes().to_vec(), 'd', - ColumnValue::Oid8(0x16_B374_D848), + build::oid8(0x16_B374_D848), ), ( MONEYOID, (-123_456i64).to_le_bytes().to_vec(), 'd', - ColumnValue::Int8(-123_456), + build::int8(-123_456), ), ( NAMEOID, @@ -2694,28 +2338,25 @@ mod tests { b }, 'c', - ColumnValue::Name("pg_class".into()), + build::name("pg_class"), ), ( MACADDROID, vec![0x08, 0x00, 0x2b, 0x01, 0x02, 0xab], 'i', - ColumnValue::MacAddr([0x08, 0x00, 0x2b, 0x01, 0x02, 0xab]), + Datum::fixed(&[0x08, 0x00, 0x2b, 0x01, 0x02, 0xab]), ), ( MACADDR8OID, vec![0x08, 0x00, 0x2b, 0xff, 0xfe, 0x01, 0x02, 0x03], 'i', - ColumnValue::MacAddr8([0x08, 0x00, 0x2b, 0xff, 0xfe, 0x01, 0x02, 0x03]), + Datum::fixed(&[0x08, 0x00, 0x2b, 0xff, 0xfe, 0x01, 0x02, 0x03]), ), ( TIDOID, vec![0, 0, 5, 0, 3, 0], 's', - ColumnValue::Tid { - block: 5, - offset: 3, - }, + Datum::fixed(&crate::decode::codecs::encode_tid(5, 3)), ), ]; for (oid, body, align, want) in cases { @@ -2729,15 +2370,15 @@ mod tests { fn missing_value_text_parses_system_types() { let mut lsn = rel_attr(1, "l", PG_LSNOID, 8, 'd'); lsn.missing_default = Some("16/B374D848".into()); - assert_eq!(missing_value_for(&lsn), ColumnValue::Oid8(0x16_B374_D848)); + assert_eq!(missing_value_for(&lsn), build::oid8(0x16_B374_D848)); let mut x8 = rel_attr(1, "x", XID8OID, 8, 'd'); x8.missing_default = Some("18446744073709551615".into()); - assert_eq!(missing_value_for(&x8), ColumnValue::Oid8(u64::MAX)); + assert_eq!(missing_value_for(&x8), build::oid8(u64::MAX)); let mut mac = rel_attr(1, "m", MACADDROID, 6, 'i'); mac.missing_default = Some("08:00:2b:01:02:03".into()); assert_eq!( missing_value_for(&mac), - ColumnValue::MacAddr([0x08, 0x00, 0x2b, 0x01, 0x02, 0x03]) + Datum::fixed(&[0x08, 0x00, 0x2b, 0x01, 0x02, 0x03]) ); } @@ -2745,34 +2386,33 @@ mod tests { fn missing_value_for_type_matrix() { let mut bool_att = rel_attr(1, "b", BOOLOID, 1, 'c'); bool_att.missing_default = Some("t".into()); - assert_eq!(missing_value_for(&bool_att), ColumnValue::Bool(true)); + assert_eq!(missing_value_for(&bool_att), build::bool_(true)); bool_att.missing_default = Some("false".into()); - assert_eq!(missing_value_for(&bool_att), ColumnValue::Bool(false)); + assert_eq!(missing_value_for(&bool_att), build::bool_(false)); let mut i4 = rel_attr(1, "x", INT4OID, 4, 'i'); i4.missing_default = Some("42".into()); - assert_eq!(missing_value_for(&i4), ColumnValue::Int4(42)); + assert_eq!(missing_value_for(&i4), build::int4(42)); let mut i8 = rel_attr(1, "x", INT8OID, 8, 'd'); i8.missing_default = Some("-9223372036854775808".into()); - assert_eq!(missing_value_for(&i8), ColumnValue::Int8(i64::MIN)); + assert_eq!(missing_value_for(&i8), build::int8(i64::MIN)); let mut txt = rel_attr(1, "n", TEXTOID, -1, 'i'); txt.missing_default = Some("hello".into()); - assert_eq!(missing_value_for(&txt), ColumnValue::Text("hello".into())); + assert_eq!(missing_value_for(&txt), build::out_text("hello")); let mut num = rel_attr(1, "n", NUMERICOID, -1, 'i'); num.missing_default = Some("3.14".into()); - assert_eq!( - missing_value_for(&num), - ColumnValue::PgPendingText { - type_oid: NUMERICOID, - text: "3.14".into(), - } - ); + assert_eq!(missing_value_for(&num), build::numeric("3.14")); + num.missing_default = Some("pi".into()); + assert_eq!(missing_value_for(&num), Datum::Null); + let mut ts = rel_attr(1, "t", TSVECTOROID, -1, 'i'); + ts.missing_default = Some("'a'".into()); + assert_eq!(missing_value_for(&ts), build::out_text("'a'")); let none = rel_attr(1, "x", INT4OID, 4, 'i'); - assert_eq!(missing_value_for(&none), ColumnValue::Null); + assert_eq!(missing_value_for(&none), Datum::Null); } #[test] @@ -2782,7 +2422,7 @@ mod tests { INT4OID, &7i32.to_le_bytes(), ))); - assert_eq!(missing_value_for(&a), ColumnValue::Int4(7)); + assert_eq!(missing_value_for(&a), build::int4(7)); } #[test] @@ -2792,7 +2432,7 @@ mod tests { TEXTOID, &short_varlena(b"hi"), ))); - assert_eq!(missing_value_for(&a), ColumnValue::Text("hi".into())); + assert_eq!(missing_value_for(&a), build::text("hi")); } #[test] @@ -2805,31 +2445,28 @@ mod tests { &short_varlena(&body), ))); // Tier-3 fast default stays pending - assert_eq!( - missing_value_for(&a), - ColumnValue::PgPending { - type_oid: TSVECTOROID, - raw: body.to_vec(), - } - ); + assert_eq!(missing_value_for(&a), build::bytes(body)); } #[test] - fn fast_default_raw_jsonb_renders_its_document() { + fn fast_default_raw_jsonb_stays_stored() { // `{}`: bare container header, JB_FOBJECT with no pairs let mut a = rel_attr(1, "j", JSONBOID, -1, 'i'); a.missing_default = Some(MissingDefault::Raw(raw_missing_array( JSONBOID, &short_varlena(&0x2000_0000u32.to_le_bytes()), ))); - assert_eq!(missing_value_for(&a), ColumnValue::Json("{}".into())); + assert_eq!( + missing_value_for(&a), + build::bytes(0x2000_0000u32.to_le_bytes()) + ); } #[test] fn fast_default_malformed_raw_is_null_not_panic() { let mut a = rel_attr(1, "x", INT4OID, 4, 'i'); a.missing_default = Some(MissingDefault::Raw(vec![0, 1, 2])); - assert_eq!(missing_value_for(&a), ColumnValue::Null); + assert_eq!(missing_value_for(&a), Datum::Null); } #[test] @@ -2875,7 +2512,8 @@ mod tests { } #[test] - fn local_matrix_agrees_with_decoder() { + fn local_matrix_agrees_with_codecs() { + use crate::emit::ch_emitter::Codec; let fixed: [(u32, i16); 26] = [ (BOOLOID, 1), (CHAROID, 1), @@ -2906,20 +2544,14 @@ mod tests { ]; for (oid, len) in fixed { let att = rel_attr(1, "c", oid, len, 'i'); - let buf = vec![0u8; len as usize]; - let (v, _) = decode_one_value(&att, &buf, 0).expect("decodes"); - let raw = matches!( - v, - ColumnValue::PgPending { .. } | ColumnValue::Unsupported { .. } - ); assert_eq!( - !raw, + Codec::choose(&att) != Codec::Pending, local_matrix_covers(oid, len), "oid {oid} typlen {len}" ); } - let bodies: [(u32, Vec); 9] = [ + let varlena: [(u32, Vec); 9] = [ (BYTEAOID, vec![1, 2]), (TEXTOID, b"x".to_vec()), (VARCHAROID, b"x".to_vec()), @@ -2940,13 +2572,13 @@ mod tests { (JSONOID, b"{}".to_vec()), (3802, vec![0, 0, 0, 0]), ]; - for (oid, body) in bodies { - let v = varlena_to_value(oid, Cow::Owned(body)); - let raw = matches!( - v, - ColumnValue::PgPending { .. } | ColumnValue::Unsupported { .. } + for (oid, _) in varlena { + let att = rel_attr(1, "c", oid, -1, 'i'); + assert_eq!( + Codec::choose(&att) != Codec::Pending, + local_matrix_covers(oid, -1), + "varlena oid {oid}" ); - assert_eq!(!raw, local_matrix_covers(oid, -1), "varlena oid {oid}"); } } @@ -3141,10 +2773,7 @@ mod tests { .new .unwrap(); assert!(new.partial); - assert_eq!( - new.columns, - vec![Some(ColumnValue::Int4(5)), None, None, None] - ); + assert_eq!(new.columns, vec![Some(build::int4(5)), None, None, None]); // Truncate suffixlen let rec = record_with(RmId::Heap, XLOG_HEAP_UPDATE, main_data, vec![12]); let r = decode_heap_record(&rec, 0, &rel); @@ -3192,18 +2821,18 @@ mod tests { .new .unwrap(); assert!(new.partial); - assert_eq!(new.columns, vec![Some(ColumnValue::Int4(9)), None, None]); + assert_eq!(new.columns, vec![Some(build::int4(9)), None, None]); } #[test] fn varlena_alignment_skips_zero_pad_per_typalign() { let mut long_body = ((64u32) << 2).to_le_bytes().to_vec(); long_body.extend_from_slice(&[b'x'; 60]); - let cases: [(char, Vec, ColumnValue); 3] = [ - ('s', short_varlena(b"hi"), ColumnValue::Text("hi".into())), - ('d', short_varlena(b"hi"), ColumnValue::Text("hi".into())), + let cases: [(char, Vec, Datum); 3] = [ + ('s', short_varlena(b"hi"), build::text("hi")), + ('d', short_varlena(b"hi"), build::text("hi")), // Character alignment needs no padding; leading zero belongs to header - ('c', long_body, ColumnValue::Text("x".repeat(60))), + ('c', long_body, build::text("x".repeat(60))), ]; for (align, value, expect) in cases { let rel = descriptor( @@ -3231,7 +2860,7 @@ mod tests { assert!(!new.partial, "align {align}"); assert_eq!( new.columns, - vec![Some(ColumnValue::Bool(true)), Some(expect)], + vec![Some(build::bool_(true)), Some(expect)], "align {align}" ); } @@ -3264,13 +2893,7 @@ mod tests { let indirect = vec![0x01, 1, 0xAA, 0xBB]; assert_eq!( decode_one_value(&text, &indirect, 0).unwrap(), - ( - ColumnValue::Unsupported { - type_oid: TEXTOID, - raw: indirect.clone(), - }, - 4 - ) + (Datum::BadTag(indirect.clone()), 4) ); } @@ -3279,14 +2902,14 @@ mod tests { let cstring = rel_attr(1, "c", 2275, -2, 'c'); assert_eq!( decode_one_value(&cstring, b"ab\0", 0).unwrap(), - (ColumnValue::Text("ab".into()), 3) + (Datum::Cstring(b"ab".to_vec()), 3) ); let dropped = |type_len, align| { let mut att = rel_attr(1, "gone", TEXTOID, type_len, align); att.dropped = true; att }; - let null3 = (ColumnValue::Null, 3); + let null3 = (Datum::Null, 3); assert_eq!( decode_one_value(&dropped(-1, 'i'), &short_varlena(b"zz"), 0).unwrap(), null3 @@ -3319,27 +2942,6 @@ mod tests { ), "{r:?}" ); - - let unsupported = |att: &RelAttr, buf: &[u8]| { - let (v, _) = decode_one_value(att, buf, 0).unwrap(); - assert!( - matches!(v, ColumnValue::Unsupported { type_oid, .. } if type_oid == att.type_oid), - "{}: {v:?}", - att.type_oid - ); - }; - unsupported(&rel_attr(1, "n", NAMEOID, 64, 'c'), &[0xFF; 64]); - // PostgreSQL intervals require 16 bytes - unsupported(&rel_attr(1, "iv", INTERVALOID, 8, 'd'), &[0; 8]); - unsupported( - &rel_attr(1, "num", NUMERICOID, -1, 'i'), - &short_varlena(&[1]), - ); - unsupported(&rel_attr(1, "ip", INETOID, -1, 'i'), &short_varlena(&[9])); - unsupported( - &rel_attr(1, "j", JSONBOID, -1, 'i'), - &short_varlena(&[0xFF; 4]), - ); } #[test] @@ -3353,44 +2955,44 @@ mod tests { let mut arr = raw_missing_array(INT4OID, &77i32.to_le_bytes()); arr[8..12].copy_from_slice(&24i32.to_le_bytes()); att.missing_default = Some(MissingDefault::Raw(arr.clone())); - assert_eq!(missing_value_for(&att), ColumnValue::Int4(77)); + assert_eq!(missing_value_for(&att), build::int4(77)); arr[4..8].copy_from_slice(&2i32.to_le_bytes()); att.missing_default = Some(MissingDefault::Raw(arr)); - assert_eq!(missing_value_for(&att), ColumnValue::Null); + assert_eq!(missing_value_for(&att), Datum::Null); } #[test] - fn missing_value_text_scalar_matrix() { + fn text_datum_scalar_matrix() { + let d = TextDatum::Datum; let cases = [ - (CHAROID, "a", ColumnValue::Char(b'a' as i8)), - (CHAROID, "", ColumnValue::Null), - (INT2OID, "-7", ColumnValue::Int2(-7)), - (INT2OID, "70000", ColumnValue::Null), - (FLOAT4OID, "1.5", ColumnValue::Float4(1.5)), - (FLOAT4OID, "x", ColumnValue::Null), - (FLOAT8OID, "-2.25", ColumnValue::Float8(-2.25)), - (FLOAT8OID, "x", ColumnValue::Null), - (OIDOID, "4294967295", ColumnValue::Oid(u32::MAX)), - (OIDOID, "-1", ColumnValue::Null), + (CHAROID, "a", d(build::char_(b'a' as i8))), + (CHAROID, "", TextDatum::Null), + (INT2OID, "-7", d(build::int2(-7))), + (INT2OID, "70000", TextDatum::Null), + (FLOAT4OID, "1.5", d(build::float4(1.5))), + (FLOAT4OID, "x", TextDatum::Null), + (FLOAT8OID, "-2.25", d(build::float8(-2.25))), + (FLOAT8OID, "x", TextDatum::Null), + (OIDOID, "4294967295", d(build::oid(u32::MAX))), + (OIDOID, "-1", TextDatum::Null), ( MACADDR8OID, "08:00:2b:ff:fe:01:02:03", - ColumnValue::MacAddr8([0x08, 0x00, 0x2b, 0xff, 0xfe, 0x01, 0x02, 0x03]), + d(Datum::fixed(&[ + 0x08, 0x00, 0x2b, 0xff, 0xfe, 0x01, 0x02, 0x03, + ])), ), - (MACADDR8OID, "08:00:2b:01:02:03", ColumnValue::Null), + (MACADDR8OID, "08:00:2b:01:02:03", TextDatum::Null), ( TIDOID, "(3,4)", - ColumnValue::Tid { - block: 3, - offset: 4, - }, + d(Datum::fixed(&crate::decode::codecs::encode_tid(3, 4))), ), - (TIDOID, "3,4", ColumnValue::Null), + (TIDOID, "3,4", TextDatum::Null), ]; for (type_oid, text, expect) in cases { assert_eq!( - missing_value_from_text(type_oid, text), + text_datum(type_oid, text.as_bytes()), expect, "{type_oid} {text:?}" ); diff --git a/src/emit/ch_ddl.rs b/src/emit/ch_ddl.rs index e5ac9846..151b5ed4 100644 --- a/src/emit/ch_ddl.rs +++ b/src/emit/ch_ddl.rs @@ -40,7 +40,8 @@ use crate::column_rules::ColumnRules; use crate::config::{ConfigResolver, DestEmitter, ResolvedConfig}; #[cfg(feature = "test-support")] use crate::crash_test; -use crate::decode::heap_decoder::{self, ColumnValue}; +use crate::decode::datum::DatumRef; +use crate::decode::heap_decoder; use crate::emit::ch_emitter::{DATETIME64_PG_EPOCH_US, EmitterConfig}; use crate::emit::route::{Routing, routing}; use crate::mapping::{ @@ -1483,14 +1484,24 @@ async fn resolve_disk_default<'a>( db: Oid, att: &'a RelAttr, ) -> Cow<'a, RelAttr> { - let ColumnValue::PgPending { raw, .. } = heap_decoder::missing_value_for(att) else { + if heap_decoder::local_matrix_covers(att.type_oid, att.type_len) { + return Cow::Borrowed(att); + } + let (DatumRef::Fixed(raw) | DatumRef::Varlena(raw) | DatumRef::Cstring(raw)) = + heap_decoder::missing_datum(att) + else { return Cow::Borrowed(att); }; let Some(oracle) = oracle else { return Cow::Borrowed(att); }; match oracle - .text_value(db, att.type_oid, att.typmod, OracleCell::DiskRaw(raw)) + .text_value( + db, + att.type_oid, + att.typmod, + OracleCell::DiskRaw(raw.to_vec()), + ) .await { Ok(text) => { diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 8a6e0e87..608ac0ee 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -34,13 +34,13 @@ use clickhouse_c::{Allocator, ColumnBuilder, Kind, TypeAst}; use crate::ch::is_retryable; use crate::ch::{CompressionChoice, ConnectionConfig, EmitterError, quote_ident, types}; use crate::column_rules::{ColumnEntry, ColumnRule, ColumnRules, Substitute}; -use crate::decode::datum::{DatumRef, ValueRef}; +use crate::decode::codecs::{Numeric, parse_numeric}; +use crate::decode::datum::{Datum, DatumRef}; #[cfg(test)] use crate::decode::decoder_sink::DecoderSinkError; -use crate::decode::heap_decoder::ColumnValue; #[cfg(any(test, feature = "test-support"))] use crate::decode::heap_decoder::{CommittedTuple, HeapOp}; -use crate::decode::heap_decoder::{datum_value, datum_view}; +use crate::decode::heap_decoder::{TextDatum, inflate, missing_datum, text_datum}; use crate::mapping::{ ColumnMapping, DropTableStrategy, NamespaceMapping, SystemColumnNames, SystemColumns, TableMapping, TableTarget, named_instance, @@ -1444,51 +1444,72 @@ pub(crate) struct ColumnPlan { pub(crate) struct SourceColumn { /// attnum - 1, indexes tuple images pub(crate) index: usize, - pub(crate) direct: Direct, + pub(crate) codec: Codec, + /// Zero for values patched past the descriptor's attributes + pub(crate) type_oid: u32, } +/// Conversion of one source type's stored bytes into its column #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Direct { - /// Stored little-endian bytes already match target storage - Copy, - /// Decode through shared value policy - Value, +pub(crate) enum Codec { + /// Little-endian storage copied as the column holds it + Le, + Date, + Time, + /// `timestamp` and `timestamptz` + Timestamp, + TimeTz, + Uuid, + Name, + /// Body bytes as String: text, bytea, json, cstring + Bytes, + Numeric, + Jsonb, + Inet, + Cidr, + MacAddr, + MacAddr8, + Tid, + Interval, /// PostGIS point rendered as WKT, typoutput yields HEXEWKB Wkt, + /// No local decode: typoutput text lands as String, stored bytes fail + Pending, } -impl Direct { - fn choose(att: &RelAttr, plan: &ColumnPlan) -> Self { +impl Codec { + pub(crate) fn choose(att: &RelAttr) -> Self { use crate::schema::{ - CHAROID, CIDOID, FLOAT4OID, FLOAT8OID, INT2OID, INT4OID, INT8OID, MONEYOID, OID8OID, - OIDOID, PG_LSNOID, XID8OID, XIDOID, + BOOLOID, BPCHAROID, BYTEAOID, CHAROID, CIDOID, CIDROID, DATEOID, FLOAT4OID, FLOAT8OID, + INETOID, INT2OID, INT4OID, INT8OID, INTERVALOID, JSONBOID, JSONOID, MACADDR8OID, + MACADDROID, MONEYOID, NAMEOID, NUMERICOID, OID8OID, OIDOID, PG_LSNOID, TEXTOID, TIDOID, + TIMEOID, TIMESTAMPOID, TIMESTAMPTZOID, TIMETZOID, UUIDOID, VARCHAROID, XID8OID, XIDOID, }; if matches!(att.type_name.as_str(), "geography" | "geometry") { return Self::Wkt; } - if plan.encoding != ColumnEncoding::Local { - return Self::Value; - } - // Equal width alone is no proof: signedness, float layout, and - // epoch all differ between same-sized types - let storage = match att.type_oid { - CHAROID => Kind::Int8, - INT2OID => Kind::Int16, - INT4OID => Kind::Int32, - INT8OID | MONEYOID => Kind::Int64, - FLOAT4OID => Kind::Float32, - FLOAT8OID => Kind::Float64, - OIDOID | XIDOID | CIDOID => Kind::UInt32, - OID8OID | XID8OID | PG_LSNOID => Kind::UInt64, - _ => return Self::Value, - }; - let inner = types::strip_nullable(plan.ast.view()); - if inner.kind() == Some(storage) - && usize::try_from(att.type_len).is_ok_and(|n| n == inner.elem_size()) - { - Self::Copy - } else { - Self::Value + if att.type_len == -2 { + return Self::Bytes; + } + match att.type_oid { + BOOLOID | CHAROID | INT2OID | INT4OID | INT8OID | MONEYOID | FLOAT4OID | FLOAT8OID + | OIDOID | XIDOID | CIDOID | OID8OID | XID8OID | PG_LSNOID => Self::Le, + DATEOID => Self::Date, + TIMEOID => Self::Time, + TIMESTAMPOID | TIMESTAMPTZOID => Self::Timestamp, + TIMETZOID => Self::TimeTz, + UUIDOID => Self::Uuid, + NAMEOID => Self::Name, + BYTEAOID | TEXTOID | VARCHAROID | BPCHAROID | JSONOID => Self::Bytes, + NUMERICOID => Self::Numeric, + JSONBOID => Self::Jsonb, + INETOID => Self::Inet, + CIDROID => Self::Cidr, + MACADDROID => Self::MacAddr, + MACADDR8OID => Self::MacAddr8, + TIDOID => Self::Tid, + INTERVALOID => Self::Interval, + _ => Self::Pending, } } } @@ -1496,15 +1517,21 @@ impl Direct { /// Values replacing non-finite numerics or temporal infinities, `None` rejects #[derive(Clone, Debug, Default, PartialEq)] pub(crate) struct NonFinite { - nan: Option, - pos_inf: Option, - neg_inf: Option, + nan: Option, + pos_inf: Option, + neg_inf: Option, +} + +/// Substitute cell, encoded for its column +#[derive(Clone, Debug, PartialEq)] +enum Fill { + Null, + Le(SmallVec<[u8; 32]>), } impl NonFinite { /// Resolve settings against Decimal or temporal destination once per plan fn compile(rule: &ColumnRule, ast: &TypeAst) -> Result { - use crate::decode::codecs::NumericKind; let nullable = ast.view().kind() == Some(Kind::Nullable); let inner = types::strip_nullable(ast.view()); let decimal = decimal_wire_of(ast); @@ -1518,19 +1545,19 @@ impl NonFinite { let cell = |key: &str, sub: &Option| { let text = match sub { None | Some(Substitute::Reject) => return Ok(None), - Some(Substitute::Null) if nullable => return Ok(Some(ColumnValue::Null)), + Some(Substitute::Null) if nullable => return Ok(Some(Fill::Null)), Some(Substitute::Null) => { return Err(format!("{key} = \"null\" needs a Nullable destination")); } - Some(_) if decimal.is_none() => { - return Err(format!( - "{key}: temporal infinity supports only reject or null" - )); - } Some(Substitute::Min) => format!("-{bound}"), Some(Substitute::Max) => bound.clone(), Some(Substitute::Literal(lit)) => lit.clone(), }; + let Some(decimal) = decimal else { + return Err(format!( + "{key}: temporal infinity supports only reject or null" + )); + }; let (int, frac) = crate::column_rules::decimal_parts(&text).unwrap_or_default(); if frac.len() > scale && frac[scale..].bytes().any(|b| b != b'0') { return Err(format!( @@ -1542,7 +1569,11 @@ impl NonFinite { "{key} = {text:?}: exceeds Decimal({precision}, {scale})" )); } - Ok(Some(ColumnValue::Numeric(NumericKind::Finite(text)))) + let scaled = decimal_text_to_scaled_le(&text, decimal.scale, decimal.width) + .map_err(|e| format!("{key} = {text:?}: {e}"))?; + Ok(Some(Fill::Le(SmallVec::from_slice( + &scaled[..decimal.width.bytes()], + )))) }; Ok(Self { // Temporal values have no NaN @@ -1556,21 +1587,29 @@ impl NonFinite { }) } - fn substitute<'a>(&'a self, v: ValueRef<'a>) -> ValueRef<'a> { - use crate::decode::codecs::NumericKind; - let sub = match v { - ValueRef::Numeric(NumericKind::NaN) => &self.nan, - ValueRef::Numeric(NumericKind::PInf) - | ValueRef::Date(i32::MAX) - | ValueRef::Timestamp(i64::MAX) - | ValueRef::TimestampTz(i64::MAX) => &self.pos_inf, - ValueRef::Numeric(NumericKind::NInf) - | ValueRef::Date(i32::MIN) - | ValueRef::Timestamp(i64::MIN) - | ValueRef::TimestampTz(i64::MIN) => &self.neg_inf, - _ => return v, + /// Substitute for a non-finite `body`, `None` to encode it as is + fn fill(&self, codec: Codec, body: &[u8]) -> Option<&Fill> { + use crate::decode::codecs::Special; + let sub = match codec { + Codec::Numeric => match parse_numeric(body) { + Ok(Numeric::Special(Special::NaN)) => &self.nan, + Ok(Numeric::Special(Special::PInf)) => &self.pos_inf, + Ok(Numeric::Special(Special::NInf)) => &self.neg_inf, + _ => return None, + }, + Codec::Date => match i32::from_le_bytes(body.try_into().ok()?) { + i32::MAX => &self.pos_inf, + i32::MIN => &self.neg_inf, + _ => return None, + }, + Codec::Timestamp => match i64::from_le_bytes(body.try_into().ok()?) { + i64::MAX => &self.pos_inf, + i64::MIN => &self.neg_inf, + _ => return None, + }, + _ => return None, }; - sub.as_ref().map_or(v, ColumnValue::view) + sub.as_ref() } } @@ -1801,11 +1840,13 @@ impl TablePlan { .and_then(|index| match att { Some(a) => Some(SourceColumn { index, - direct: Direct::choose(a, &plan), + codec: Codec::choose(a), + type_oid: a.type_oid, }), None => (index >= rel.attributes.len()).then_some(SourceColumn { index, - direct: Direct::Value, + codec: Codec::Le, + type_oid: 0, }), }); columns.push(plan); @@ -1875,6 +1916,8 @@ pub(crate) struct TableEncoder { oracle_cells: Vec, /// Mirrors `plan.columns` plus the synthetic columns the plan carries pub(crate) buffers: Vec, + /// Rendered text of the cell being encoded + text: String, } /// On-the-wire-shape column buffer. [`BlockBuilder`] borrows these @@ -2312,6 +2355,7 @@ impl TableEncoder { oracle_overhead, oracle_cells: Vec::new(), buffers, + text: String::new(), }) } @@ -2396,13 +2440,13 @@ impl TableEncoder { HeapOp::Delete => decoded.old.as_ref(), _ => decoded.new.as_ref(), }; - let cells: SmallVec<[Cell<'_>; 16]> = mapping + let cells: SmallVec<[DatumRef<'_>; 16]> = mapping .columns .iter() .map(|col| { side.and_then(|t| t.columns.get((col.src_attnum - 1) as usize)) .and_then(Option::as_ref) - .map_or(Cell::Absent, |v| Cell::Value(v.view())) + .map_or(DatumRef::Absent, Datum::view) }) .collect(); let stamp = RowStamp { @@ -2416,81 +2460,60 @@ impl TableEncoder { } /// Append one image of `rel` read in place. Return [`Append::Full`] - /// without mutation when oracle threshold is hit. `patch` - /// replaces attributes by index: resolved TOAST values, and values past - /// `rel`'s attributes. Values whose decode allocates, and PostGIS - /// renders, build in row scratch + /// without mutation when oracle threshold is hit. `patch` replaces + /// attributes by index: resolved TOAST values, and values past `rel`'s + /// attributes pub(crate) fn append_view( &mut self, rel: &RelDescriptor, datums: &[DatumRef<'_>], - patch: &[(usize, ColumnValue)], + patch: &[(usize, Datum)], stamp: &RowStamp, ) -> Result { - let mut row = RowScratch::new(self.plan.columns.len()); - for c in &self.plan.columns { - let Some(src) = c.source else { - row.cell(Cell::Absent); - continue; - }; - let wkt = src.direct == Direct::Wkt; - if let Some((_, v)) = patch.iter().find(|(i, _)| *i == src.index) { - if wkt { - let mut v = v.clone(); - crate::ops::oracle::render_ext_value(&mut v); - row.own(v); - } else { - row.cell(Cell::Value(v.view())); + let cells: SmallVec<[DatumRef<'_>; 16]> = self + .plan + .columns + .iter() + .map(|c| { + let Some(src) = c.source else { + return DatumRef::Absent; + }; + if let Some((_, v)) = patch.iter().find(|(i, _)| *i == src.index) { + return v.view(); } - continue; - } - let (Some(&datum), Some(att)) = (datums.get(src.index), rel.attributes.get(src.index)) - else { - row.cell(Cell::Absent); - continue; - }; - match (src.direct, datum) { - (Direct::Copy, DatumRef::Fixed(le)) => row.cell(Cell::Le(le)), - (Direct::Wkt, _) => match datum_value(att, datum) { - None => row.cell(Cell::Absent), - Some(mut v) => { - crate::ops::oracle::render_ext_value(&mut v); - row.own(v); - } - }, - _ => match datum_view(att, datum) { - Ok(v) => row.cell(v.map_or(Cell::Absent, Cell::Value)), - Err(_) => { - row.own(datum_value(att, datum).expect("allocating datum is present")) - } - }, - } - } - self.append_cells(|i| row.get(i), stamp) + match (datums.get(src.index), rel.attributes.get(src.index)) { + (Some(DatumRef::Missing), Some(att)) => missing_datum(att), + (Some(&datum), Some(_)) => datum, + _ => DatumRef::Absent, + } + }) + .collect(); + self.append_cells(|i| cells[i], stamp) } /// Append one row whose mapped sources `slot` yields; columns without a /// source attribute stay absent pub(crate) fn append_slots<'a>( &mut self, - mut slot: impl FnMut(SourceColumn) -> Result, EmitterError>, + mut slot: impl FnMut(SourceColumn) -> Result, EmitterError>, stamp: &RowStamp, ) -> Result { - let mut row = RowScratch::new(self.plan.columns.len()); + let mut cells: SmallVec<[DatumRef<'a>; 16]> = SmallVec::new(); for c in &self.plan.columns { - match c.source.map(&mut slot).transpose()? { - None => row.cell(Cell::Absent), - Some(Slot::Cell(cell)) => row.cell(cell), - Some(Slot::Owned(v)) => row.own(v), - } + cells.push( + c.source + .map(&mut slot) + .transpose()? + .unwrap_or(DatumRef::Absent), + ); } - self.append_cells(|i| row.get(i), stamp) + self.append_cells(|i| cells[i], stamp) } - /// Append one row, `cell(i)` yielding `plan.columns[i]`'s cell + /// Append one row, `cell(i)` yielding `plan.columns[i]`'s datum fn append_cells<'c>( &mut self, - cell: impl Fn(usize) -> Cell<'c>, + cell: impl Fn(usize) -> DatumRef<'c>, stamp: &RowStamp, ) -> Result { let columns = self.plan.columns.len(); @@ -2503,8 +2526,9 @@ impl TableEncoder { let mut stranded_bytes = 0usize; for i in 0..columns { if let ColumnBuf::Oracle(o) = &self.buffers[i] { - let cell = oracle_cell(cell(i), o.source_type_oid) - .map_err(|e| name_column(e, &self.plan.columns[i].name))?; + let col_plan = &self.plan.columns[i]; + let cell = oracle_cell(cell(i), col_plan.source, o.source_type_oid) + .map_err(|e| name_column(e, &col_plan.name))?; if !o.resolves_locally(&cell) { row_oracle_bytes += cell.wire_bytes(); stranded_bytes += o.stranded_bytes(); @@ -2534,22 +2558,8 @@ impl TableEncoder { o.push(std::mem::replace(cell, OracleCell::Default)); continue; } - match cell(i) { - // Absent coerces like NULL, covers key-only delete - // tombstones under non-FULL replica identity - Cell::Absent => { - buf.append_default(); - Ok(()) - } - Cell::Le(le) => buf.append_fixed_bytes(le), - Cell::Value(v) => encode_value( - buf, - col_plan.non_finite.substitute(v), - col_plan.decimal, - col_plan.datetime64_scale, - ), - } - .map_err(|e| name_column(e, &col_plan.name))?; + encode_datum(buf, col_plan, cell(i), &mut self.text) + .map_err(|e| name_column(e, &col_plan.name))?; } oracle.clear(); self.oracle_cells = oracle; @@ -2578,60 +2588,6 @@ impl TableEncoder { } } -/// One row's cells in `plan.columns` order, values whose decode allocates -/// held aside so cells can borrow them -struct RowScratch<'a> { - marks: SmallVec<[Mark<'a>; 16]>, - owned: SmallVec<[ColumnValue; 2]>, -} - -/// Cell, or index into [`RowScratch::owned`] -enum Mark<'a> { - Cell(Cell<'a>), - Owned(usize), -} - -impl<'a> RowScratch<'a> { - fn new(columns: usize) -> Self { - Self { - marks: SmallVec::with_capacity(columns), - owned: SmallVec::new(), - } - } - - fn cell(&mut self, cell: Cell<'a>) { - self.marks.push(Mark::Cell(cell)); - } - - fn own(&mut self, v: ColumnValue) { - self.owned.push(v); - self.marks.push(Mark::Owned(self.owned.len() - 1)); - } - - fn get(&self, i: usize) -> Cell<'_> { - match &self.marks[i] { - Mark::Cell(c) => *c, - Mark::Owned(j) => Cell::Value(self.owned[*j].view()), - } - } -} - -/// One mapped source value, borrowed or built in row scratch -pub(crate) enum Slot<'a> { - Cell(Cell<'a>), - Owned(ColumnValue), -} - -/// One mapped cell ready to encode -#[derive(Clone, Copy, Debug)] -pub(crate) enum Cell<'a> { - /// Absent from image, coerces like NULL - Absent, - Value(ValueRef<'a>), - /// Little-endian bytes [`Direct::Copy`] proved match column storage - Le(&'a [u8]), -} - /// Rows of one plan table encoded at planning. Relation and route travel /// once per chunk so the batcher keys and plans its destination pub struct EncodedChunk { @@ -2925,6 +2881,12 @@ impl U256 { rem as u32 } + fn from_le_bytes(b: [u8; 32]) -> Self { + Self(std::array::from_fn(|i| { + u64::from_le_bytes(b[i * 8..][..8].try_into().expect("8 bytes")) + })) + } + fn to_le_bytes(self) -> [u8; 32] { let mut out = [0u8; 32]; for (i, limb) in self.0.iter().enumerate() { @@ -2952,6 +2914,44 @@ impl PartialOrd for U256 { } } +/// Decimal text of a CH `Decimal` cell, a two's-complement little-endian +/// integer scaled by `10^scale` +pub(crate) fn decimal_le_to_text(le: &[u8], scale: u8) -> String { + let neg = le.last().is_some_and(|b| b & 0x80 != 0); + let mut wide = [if neg { 0xFF } else { 0 }; 32]; + wide[..le.len()].copy_from_slice(le); + if neg { + let mut carry = 1u16; + for b in &mut wide { + let v = u16::from(!*b) + carry; + *b = v as u8; + carry = v >> 8; + } + } + let mut mag = U256::from_le_bytes(wide); + let mut digits = Vec::new(); + loop { + digits.push(b'0' + mag.div_small(10) as u8); + if mag.is_zero() { + break; + } + } + let scale = usize::from(scale); + digits.resize(digits.len().max(scale + 1), b'0'); + digits.reverse(); + let (int, frac) = digits.split_at(digits.len() - scale); + let mut out = String::with_capacity(digits.len() + 2); + if neg { + out.push('-'); + } + out.extend(int.iter().map(|&b| char::from(b))); + if scale > 0 { + out.push('.'); + out.extend(frac.iter().map(|&b| char::from(b))); + } + out +} + fn decimal_oob() -> EmitterError { unsupported("numeric out of range for Decimal column") } @@ -3013,6 +3013,42 @@ fn decimal_text_to_scaled_le( } } + signed_le(mag, neg, width) +} + +/// Stored `numeric` body to the scaled integer a CH `Decimal(_, scale)` +/// stores, read from its base-10000 digits. Digits past `scale` drop +fn numeric_to_scaled_le( + body: &[u8], + scale: u8, + width: DecimalWidth, +) -> Result<[u8; 32], EmitterError> { + let scale = u32::from(scale); + let (neg, weight, digits) = match parse_numeric(body) { + Ok(Numeric::Finite { + neg, + weight, + digits, + .. + }) => (neg, weight, digits), + Ok(Numeric::Special(_)) => { + return Err(unsupported( + "non-finite numeric (NaN/Inf) into Decimal column, set nan/pos_inf/neg_inf or map to String", + )); + } + Err(_) => return Err(unsupported("malformed numeric")), + }; + let mut mag = U256::default(); + if !crate::decode::codecs::scaled_groups(weight, digits, scale, |m, a| { + mag.checked_mul_small(m) && mag.checked_add_small(a) + }) { + return Err(decimal_oob()); + } + signed_le(mag, neg, width.bytes()) +} + +/// Two's-complement little-endian `mag`, bounded by the Decimal wire width +fn signed_le(mag: U256, neg: bool, width: usize) -> Result<[u8; 32], EmitterError> { // Bound by physical wire width (signed Int{32,64,128,256} range), // not logical Decimal(p,s) precision 10^p. Backstop turning a // too-wide value (eg operator override onto a narrower Decimal) into @@ -3068,135 +3104,247 @@ fn micros_to_ticks(us: i128, scale: u32) -> Result { i64::try_from(ticks).map_err(|_| unsupported("finite temporal value overflows storage")) } -fn encode_value( +fn fixed(body: &[u8]) -> Result<[u8; N], EmitterError> { + body.try_into() + .map_err(|_| unsupported("stored width differs from source type")) +} + +/// Encode one source datum into its column per the plan's codec +fn encode_datum( buf: &mut ColumnBuf, - v: ValueRef<'_>, - decimal: Option, - timestamp_scale: u32, + plan: &ColumnPlan, + datum: DatumRef<'_>, + text: &mut String, ) -> Result<(), EmitterError> { - match v { - // Non-Nullable target takes type default - ValueRef::Null => { + let (codec, type_oid) = plan + .source + .map_or((Codec::Le, 0), |s| (s.codec, s.type_oid)); + let body = match datum { + // Absent coerces like NULL, covers key-only delete tombstones under + // non-FULL replica identity. Non-Nullable target takes type default + DatumRef::Absent | DatumRef::Null | DatumRef::Dropped => { buf.append_default(); - Ok(()) - } - ValueRef::Bool(b) => buf.append_fixed_bytes(&[b as u8]), - ValueRef::Char(c) => buf.append_fixed_bytes(&c.to_le_bytes()), - ValueRef::Int2(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ValueRef::Int4(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ValueRef::Int8(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ValueRef::Float4(f) => buf.append_fixed_bytes(&f.to_le_bytes()), - ValueRef::Float8(f) => buf.append_fixed_bytes(&f.to_le_bytes()), - ValueRef::Oid(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ValueRef::Oid8(n) => buf.append_fixed_bytes(&n.to_le_bytes()), - ValueRef::Date(i32::MIN | i32::MAX) - | ValueRef::Timestamp(i64::MIN | i64::MAX) - | ValueRef::TimestampTz(i64::MIN | i64::MAX) => Err(unsupported( - "temporal infinity, set pos_inf/neg_inf to null on a Nullable destination", - )), - ValueRef::Date(n) => { - let days = n - .checked_add(DATE32_PG_EPOCH_DAYS) - .ok_or_else(|| unsupported("finite date overflows Date32 epoch conversion"))?; - buf.append_fixed_bytes(&days.to_le_bytes()) + return Ok(()); } - // PG time counts microseconds since midnight, without an epoch offset - ValueRef::Time(n) => { - let ticks = micros_to_ticks(i128::from(n), timestamp_scale)?; - buf.append_fixed_bytes(&ticks.to_le_bytes()) + DatumRef::Missing => return Err(unsupported("fast default left unresolved")), + DatumRef::External(_) => { + return Err(unsupported( + "unresolved TOAST pointer (xact buffer should have reassembled)", + )); } - ValueRef::Timestamp(n) | ValueRef::TimestampTz(n) => { - let ticks = timestamp_ticks(n, timestamp_scale)?; - buf.append_fixed_bytes(&ticks.to_le_bytes()) + DatumRef::BadTag(_) => { + return Err(unsupported( + "unresolved source value routed to a local column", + )); } - // `timetz` → text: CH has no zone-aware time type, text keeps - // the offset the old fixed encoding dropped - ValueRef::TimeTz { micros, tz_seconds } => buf.append_string_bytes( - crate::decode::codecs::timetz_to_text(micros, tz_seconds).as_bytes(), - ), - ValueRef::Uuid(b) => buf.append_fixed_bytes(&crate::decode::codecs::uuid_to_ch_wire(&b)), - ValueRef::Name(s) | ValueRef::Text(s) | ValueRef::Json(s) => { - buf.append_string_bytes(s.as_bytes()) - } - ValueRef::Numeric(n) => { - use crate::decode::codecs::NumericKind; - match decimal { - // Decimal column: non-finite (NaN/±Inf) is unrepresentable, - // error rather than silently corrupt unless column sets a - // substitute or maps to String - Some(decimal) => match n { - NumericKind::Finite(s) => { - let scaled = decimal_text_to_scaled_le(s, decimal.scale, decimal.width)?; - buf.append_fixed_bytes(&scaled[..decimal.width.bytes()]) - } - NumericKind::NaN | NumericKind::PInf | NumericKind::NInf => Err(unsupported( - "non-finite numeric (NaN/Inf) into Decimal column, set nan/pos_inf/neg_inf or map to String", - )), - }, - // String column: lossless text, including NaN/±Inf - None => buf.append_string_bytes(n.as_text().as_bytes()), - } + DatumRef::Compressed(d) => { + let raw = inflate(d).ok_or_else(|| unsupported("corrupt compressed varlena"))?; + return encode_datum(buf, plan, DatumRef::Varlena(&raw), text); } - ValueRef::Inet(v) => buf.append_string_bytes(v.to_text().as_bytes()), - ValueRef::MacAddr(b) => { - buf.append_string_bytes(crate::decode::codecs::macaddr_to_text(&b).as_bytes()) + DatumRef::Text(t) => { + return match text_datum(type_oid, t) { + TextDatum::Datum(d) => encode_datum(buf, plan, d.view(), text), + TextDatum::Null => { + buf.append_default(); + Ok(()) + } + TextDatum::Opaque => buf.append_string_bytes(t), + }; } - ValueRef::MacAddr8(b) => { - buf.append_string_bytes(crate::decode::codecs::macaddr8_to_text(&b).as_bytes()) + DatumRef::Fixed(b) | DatumRef::Varlena(b) | DatumRef::Cstring(b) => b, + }; + if let Some(fill) = plan.non_finite.fill(codec, body) { + return match fill { + Fill::Null => { + buf.append_default(); + Ok(()) + } + Fill::Le(le) => buf.append_fixed_bytes(le), + }; + } + let rendered = |buf: &mut ColumnBuf, s: &str| buf.append_string_bytes(s.as_bytes()); + match codec { + Codec::Le => buf.append_fixed_bytes(body), + Codec::Date => match i32::from_le_bytes(fixed(body)?) { + i32::MIN | i32::MAX => Err(temporal_infinity()), + n => { + let days = n + .checked_add(DATE32_PG_EPOCH_DAYS) + .ok_or_else(|| unsupported("finite date overflows Date32 epoch conversion"))?; + buf.append_fixed_bytes(&days.to_le_bytes()) + } + }, + // PG time counts microseconds since midnight, without an epoch offset + Codec::Time => { + let us = i64::from_le_bytes(fixed(body)?); + let ticks = micros_to_ticks(i128::from(us), plan.datetime64_scale)?; + buf.append_fixed_bytes(&ticks.to_le_bytes()) } - ValueRef::Tid { block, offset } => { - buf.append_string_bytes(crate::decode::codecs::tid_to_text(block, offset).as_bytes()) + Codec::Timestamp => match i64::from_le_bytes(fixed(body)?) { + i64::MIN | i64::MAX => Err(temporal_infinity()), + us => { + let ticks = timestamp_ticks(us, plan.datetime64_scale)?; + buf.append_fixed_bytes(&ticks.to_le_bytes()) + } + }, + // CH has no zone-aware time type, text keeps the offset + Codec::TimeTz => { + let b: [u8; 12] = fixed(body)?; + let micros = i64::from_le_bytes(b[..8].try_into().expect("8 bytes")); + let tz_seconds = i32::from_le_bytes(b[8..].try_into().expect("4 bytes")); + rendered( + buf, + &crate::decode::codecs::timetz_to_text(micros, tz_seconds), + ) } - ValueRef::Interval(v) => buf.append_string_bytes(v.to_text().as_bytes()), - ValueRef::Bytea(b) => buf.append_string_bytes(b), - ValueRef::ExternalToast(_) => Err(unsupported( - "unresolved TOAST pointer (xact buffer should have reassembled)", - )), - ValueRef::PgPendingText { text, .. } => buf.append_string_bytes(text.as_bytes()), + Codec::Uuid => { + buf.append_fixed_bytes(&crate::decode::codecs::uuid_to_ch_wire(&fixed(body)?)) + } + // NUL-padded to NAMEDATALEN + Codec::Name => { + let end = body.iter().position(|&b| b == 0).unwrap_or(body.len()); + simdutf8::basic::from_utf8(&body[..end]) + .map_err(|_| unsupported("unresolved source value routed to a local column"))?; + buf.append_string_bytes(&body[..end]) + } + // PG validates UTF-8 on input, so bytes pass as they are + Codec::Bytes => buf.append_string_bytes(body), + Codec::Numeric => match plan.decimal { + // Non-finite is unrepresentable in Decimal: error rather than + // corrupt unless the column sets a substitute or maps to String + Some(decimal) => { + let scaled = numeric_to_scaled_le(body, decimal.scale, decimal.width)?; + buf.append_fixed_bytes(&scaled[..decimal.width.bytes()]) + } + None => { + text.clear(); + crate::decode::codecs::render_numeric_text(body, text) + .map_err(|_| unsupported("malformed numeric"))?; + rendered(buf, text) + } + }, + Codec::Jsonb => { + text.clear(); + crate::decode::codecs::render_jsonb_text(body, text) + .map_err(|_| unsupported("malformed jsonb"))?; + rendered(buf, text) + } + Codec::Inet | Codec::Cidr => { + let v = crate::decode::codecs::decode_inet(body, codec == Codec::Cidr) + .map_err(|_| unsupported("malformed inet"))?; + rendered(buf, &v.to_text()) + } + Codec::MacAddr => rendered(buf, &crate::decode::codecs::macaddr_to_text(&fixed(body)?)), + Codec::MacAddr8 => rendered(buf, &crate::decode::codecs::macaddr8_to_text(&fixed(body)?)), + Codec::Tid => { + let (block, offset) = crate::decode::codecs::decode_tid(&fixed(body)?); + rendered(buf, &crate::decode::codecs::tid_to_text(block, offset)) + } + Codec::Interval => { + let v = crate::decode::codecs::decode_interval(body) + .map_err(|_| unsupported("malformed interval"))?; + rendered(buf, &v.to_text()) + } + Codec::Wkt => match crate::ops::oracle::point_wkt(body) { + Some(wkt) => rendered(buf, &wkt), + None => Err(unsupported( + "unresolved source value routed to a local column", + )), + }, // Never interpret raw Datum bytes as local String data - ValueRef::PgPending { .. } | ValueRef::Unsupported { .. } => Err(unsupported( + Codec::Pending => Err(unsupported( "unresolved source value routed to a local column", )), } } -/// Map NULL, absent, and unlogged fields to default cells -fn oracle_cell(cell: Cell<'_>, source_type_oid: u32) -> Result { - let mismatch = |got: u32| EmitterError::UnsupportedValue { +fn temporal_infinity() -> EmitterError { + unsupported("temporal infinity, set pos_inf/neg_inf to null on a Nullable destination") +} + +/// Map NULL, absent, and unlogged fields to default cells. Stored bodies go +/// to typinput's inverse, text bypasses typoutput +fn oracle_cell( + datum: DatumRef<'_>, + source: Option, + source_type_oid: u32, +) -> Result { + use crate::schema::{BPCHAROID, TEXTOID, VARCHAROID}; + let unresolved = |kind| EmitterError::UnsupportedValue { target_column: String::new(), - kind: if got == 0 { - "decoded value on an oracle column" - } else { - "source type oid differs from the planned one" - }, - }; - let v = match cell { - Cell::Absent => return Ok(OracleCell::Default), - Cell::Value(v) => v, - Cell::Le(_) => return Err(mismatch(0)), + kind, }; - Ok(match v { - ValueRef::Null => OracleCell::Default, - ValueRef::PgPending { type_oid, raw } | ValueRef::Unsupported { type_oid, raw } => { - if type_oid != source_type_oid { - return Err(mismatch(type_oid)); - } - OracleCell::DiskRaw(raw.to_vec()) + let (codec, type_oid) = source.map_or((Codec::Le, 0), |s| (s.codec, s.type_oid)); + let body = match datum { + DatumRef::Absent | DatumRef::Null | DatumRef::Dropped => return Ok(OracleCell::Default), + _ if type_oid != source_type_oid || type_oid == 0 => { + return Err(unresolved(if type_oid == 0 { + "decoded value on an oracle column" + } else { + "source type oid differs from the planned one" + })); } - ValueRef::PgPendingText { type_oid, text } => { - if type_oid != source_type_oid { - return Err(mismatch(type_oid)); - } - OracleCell::TextInput(text.as_bytes().to_vec()) + DatumRef::Text(t) => return Ok(OracleCell::TextInput(t.to_vec())), + DatumRef::Missing => return Err(unresolved("fast default left unresolved")), + DatumRef::External(_) | DatumRef::BadTag(_) => { + return Err(unresolved("unresolved source value on an oracle column")); } - // jsonb needs typinput to reconstruct binary storage from document text - ValueRef::Json(s) => OracleCell::TextInput(s.as_bytes().to_vec()), + DatumRef::Compressed(d) => { + let raw = inflate(d).ok_or_else(|| unresolved("corrupt compressed varlena"))?; + return oracle_cell(DatumRef::Varlena(&raw), source, source_type_oid); + } + DatumRef::Fixed(b) | DatumRef::Varlena(b) | DatumRef::Cstring(b) => b, + }; + Ok(match codec { // PostGIS WKT must bypass HEXEWKB typoutput - ValueRef::Text(s) | ValueRef::Name(s) => OracleCell::Literal(s.as_bytes().to_vec()), - _ => return Err(mismatch(0)), + Codec::Wkt => crate::ops::oracle::point_wkt(body).map_or_else( + || OracleCell::DiskRaw(body.to_vec()), + |wkt| OracleCell::Literal(wkt.into_bytes()), + ), + Codec::Name => { + let end = body.iter().position(|&b| b == 0).unwrap_or(body.len()); + OracleCell::Literal(body[..end].to_vec()) + } + Codec::Bytes if matches!(datum, DatumRef::Cstring(_)) => OracleCell::Literal(body.to_vec()), + Codec::Bytes if matches!(type_oid, TEXTOID | VARCHAROID | BPCHAROID) => { + OracleCell::Literal(body.to_vec()) + } + _ => OracleCell::DiskRaw(body.to_vec()), }) } +/// Default of `att` encoded for a one-row `ch_inner` column; `None` without +/// a fast default or when the column can't hold it locally +pub(crate) fn encode_default(att: &RelAttr, ch_inner: &str) -> Option { + let datum = missing_datum(att); + if datum == DatumRef::Null { + return None; + } + let ast = TypeAst::parse(ch_inner, Allocator::global(&mimalloc::MiMalloc)).ok()?; + if composite_target(&ast) { + return None; + } + let mut buf = ColumnBuf::new_for_ast(&ast).ok()?; + let plan = ColumnPlan { + name: att.name.clone(), + type_repr: ch_inner.into(), + decimal: decimal_wire_of(&ast), + datetime64_scale: types::strip_nullable(ast.view()) + .datetime64_scale() + .unwrap_or(0), + ast, + non_finite: NonFinite::default(), + encoding: ColumnEncoding::Local, + source: Some(SourceColumn { + index: 0, + codec: Codec::choose(att), + type_oid: att.type_oid, + }), + }; + encode_datum(&mut buf, &plan, datum, &mut String::new()).ok()?; + Some(buf) +} + crate::atomic_stats! { /// CH emitter counters. `fetch_add(_, Relaxed)`; status loop reads /// via `.load(Relaxed)`. @@ -3384,18 +3532,12 @@ pub(crate) enum Wire<'a> { #[cfg(test)] impl EncodedChunk { /// Mapped cells per row read back from wire buffers: fixed widths as - /// signed ints, strings as bytes - pub(crate) fn cells(&self) -> Vec> { - let fixed = |b: &[u8]| match b.len() { - 1 => ColumnValue::Char(b[0] as i8), - 2 => ColumnValue::Int2(i16::from_le_bytes(b.try_into().unwrap())), - 4 => ColumnValue::Int4(i32::from_le_bytes(b.try_into().unwrap())), - 8 => ColumnValue::Int8(i64::from_le_bytes(b.try_into().unwrap())), - _ => ColumnValue::Bytea(b.to_vec()), - }; + /// little-endian bytes, strings as varlena bodies, so builders state both + pub(crate) fn cells(&self) -> Vec> { + let fixed = Datum::fixed; let string = |offsets: &[u64], data: &[u8], r: usize| { let start = r.checked_sub(1).map_or(0, |p| offsets[p] as usize); - ColumnValue::Bytea(data[start..offsets[r] as usize].to_vec()) + Datum::Varlena(data[start..offsets[r] as usize].to_vec()) }; let mapped = self.route.mapping.columns.len(); let mut rows = vec![Vec::with_capacity(mapped); self.rows]; @@ -3407,14 +3549,14 @@ impl EncodedChunk { | Wire::NullableString { null_map, .. } if null_map[r] != 0 => { - ColumnValue::Null + Datum::Null } Wire::NullableFixed { width, inner, .. } => { fixed(&inner[r * width..(r + 1) * width]) } Wire::String { offsets, data, .. } | Wire::NullableString { offsets, data, .. } => string(offsets, data, r), - Wire::Oracle(cells) => ColumnValue::Text(format!("{:?}", cells[r])), + Wire::Oracle(cells) => Datum::Text(format!("{:?}", cells[r]).into_bytes()), }); } } @@ -3602,14 +3744,32 @@ pub async fn load_effective( #[cfg(test)] mod tests { use super::*; + use crate::decode::datum::{Datum, build}; + /// Encode `v` alone under `codec`, as a column of `decimal` or + /// `timestamp_scale` would fn encode_value( buf: &mut ColumnBuf, - v: &ColumnValue, + codec: Codec, + v: &Datum, decimal: Option, timestamp_scale: u32, ) -> Result<(), EmitterError> { - super::encode_value(buf, v.view(), decimal, timestamp_scale) + let plan = ColumnPlan { + name: String::new(), + type_repr: String::new(), + ast: TypeAst::parse("String", Allocator::global(&mimalloc::MiMalloc)).unwrap(), + decimal, + datetime64_scale: timestamp_scale, + non_finite: NonFinite::default(), + encoding: ColumnEncoding::Local, + source: Some(SourceColumn { + index: 0, + codec, + type_oid: 0, + }), + }; + encode_datum(buf, &plan, v.view(), &mut String::new()) } use crate::backfill::copy_backfill::COPY_CHUNK_BLOCKS; use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple}; @@ -3849,10 +4009,7 @@ mod tests { } fn committed(id: i32, name: Option<&str>) -> CommittedTuple { - let name_col = Some( - name.map(|s| ColumnValue::Text(s.to_string())) - .unwrap_or(ColumnValue::Null), - ); + let name_col = Some(name.map(build::text).unwrap_or(Datum::Null)); CommittedTuple { decoded: DecodedHeap { rfn: RelFileNode { @@ -3864,7 +4021,7 @@ mod tests { source_lsn: 0xCAFE, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(id)), name_col], + columns: vec![Some(build::int4(id)), name_col], partial: false, }), old: None, @@ -4009,8 +4166,9 @@ mod tests { let alloc = Allocator::global(&mimalloc::MiMalloc); let mut rel = mk_rel(); // Source type the local matrix misses, so `name` plans as an oracle - // column while its String target still renders literals in-daemon + // column while its String target still renders points in-daemon rel.attributes[1].type_oid = 17_000; + rel.attributes[1].type_name = "geometry".into(); let m = mk_mapping(); let plan = TablePlan::build( alloc, @@ -4026,36 +4184,37 @@ mod tests { )); let mut enc = TableEncoder::new(plan).unwrap(); let overhead = enc.oracle_bytes; - let big = "x".repeat(ORACLE_BATCH_SEAL_BYTES * 3 / 5); + let row = |id: i32, body: Vec| { + let mut row = committed(id, None); + row.decoded.new.as_mut().expect("insert tuple").columns[1] = Some(build::bytes(body)); + row + }; for id in 1..=2 { + let point = crate::ops::oracle::point_body(f64::from(id), 2.0); assert_eq!( - enc.append_row(&committed(id, Some(&big)), &m, OP_INSERT) - .unwrap(), + enc.append_row(&row(id, point), &m, OP_INSERT).unwrap(), Append::Done, - "literal rows do not seal", ); } assert_eq!( enc.oracle_bytes, overhead, "literals owe the request nothing" ); - // A cell needing conversion sends the column whole, so the literals - // it strands seal the batch rather than overrun the frame cap - let mut json_row = committed(3, None); - json_row.decoded.new.as_mut().expect("insert tuple").columns[1] = - Some(ColumnValue::Json("{}".into())); + let oracle = |enc: &TableEncoder| match &enc.buffers[1] { + ColumnBuf::Oracle(o) => literal_column(o, enc.rows).is_some(), + _ => unreachable!("oracle column"), + }; + assert!(oracle(&enc)); + // A cell needing conversion sends the column whole, stranding the + // literals onto the request + let raw = OracleCell::DiskRaw(vec![1, 2, 3]).wire_bytes(); assert_eq!( - enc.append_row(&json_row, &m, OP_INSERT).unwrap(), - Append::Full + enc.append_row(&row(3, vec![1, 2, 3]), &m, OP_INSERT) + .unwrap(), + Append::Done ); - let (buffers, rows) = enc.take_block().unwrap(); - assert_eq!(rows, 2); - assert!(matches!(buffers[1].wire(), Wire::Oracle(cells) if cells.len() == 2)); - let oracle = buffers.iter().find_map(|b| match b { - ColumnBuf::Oracle(o) => Some(o), - _ => None, - }); - assert!(literal_column(oracle.expect("oracle column"), rows).is_some()); + assert!(enc.oracle_bytes > overhead + raw, "literals stranded"); + assert!(!oracle(&enc)); } #[test] @@ -4132,7 +4291,6 @@ mod tests { #[test] fn encode_numeric_into_decimal_and_string() { - use crate::decode::codecs::NumericKind; let alloc = Allocator::global(&mimalloc::MiMalloc); let ast = TypeAst::parse("Decimal(10, 2)", alloc).unwrap(); let decimal = decimal_wire_of(&ast); @@ -4146,7 +4304,8 @@ mod tests { let mut buf = ColumnBuf::new_for_ast(&ast).unwrap(); encode_value( &mut buf, - &ColumnValue::Numeric(NumericKind::Finite("1.50".into())), + Codec::Numeric, + &build::numeric("1.50"), decimal, 6, ) @@ -4162,7 +4321,8 @@ mod tests { assert!( encode_value( &mut buf_nan, - &ColumnValue::Numeric(NumericKind::NaN), + Codec::Numeric, + &build::numeric("NaN"), decimal, 6, ) @@ -4182,7 +4342,8 @@ mod tests { let wide_text = "123456789012345678901234567890123456789012345678.12"; encode_value( &mut wide_buf, - &ColumnValue::Numeric(NumericKind::Finite(wide_text.into())), + Codec::Numeric, + &build::numeric(wide_text), wide_decimal, 6, ) @@ -4199,7 +4360,7 @@ mod tests { let sast = TypeAst::parse("String", alloc).unwrap(); let mut sbuf = ColumnBuf::new_for_ast(&sast).unwrap(); - encode_value(&mut sbuf, &ColumnValue::Numeric(NumericKind::NaN), None, 6).unwrap(); + encode_value(&mut sbuf, Codec::Numeric, &build::numeric("NaN"), None, 6).unwrap(); assert_eq!( sbuf.wire(), Wire::String { @@ -4216,7 +4377,7 @@ mod tests { let micros = 45_296_000_000i64; // 12:34:56 let ast = TypeAst::parse("Time64(6)", alloc).unwrap(); let mut buf = ColumnBuf::new_for_ast(&ast).unwrap(); - encode_value(&mut buf, &ColumnValue::Time(micros), None, 6).unwrap(); + encode_value(&mut buf, Codec::Time, &build::micros(micros), None, 6).unwrap(); assert_eq!( buf.wire(), Wire::Fixed { @@ -4226,7 +4387,14 @@ mod tests { ); let ast = TypeAst::parse("Time64(3)", alloc).unwrap(); let mut buf = ColumnBuf::new_for_ast(&ast).unwrap(); - encode_value(&mut buf, &ColumnValue::Time(micros + 7_999), None, 3).unwrap(); + encode_value( + &mut buf, + Codec::Time, + &build::micros(micros + 7_999), + None, + 3, + ) + .unwrap(); assert_eq!( buf.wire(), Wire::Fixed { @@ -4238,10 +4406,8 @@ mod tests { let mut sbuf = ColumnBuf::new_for_ast(&sast).unwrap(); encode_value( &mut sbuf, - &ColumnValue::TimeTz { - micros, - tz_seconds: -7200, - }, + Codec::TimeTz, + &build::timetz(micros, -7200), None, 6, ) @@ -4284,7 +4450,7 @@ mod tests { #[test] fn column_buffers_reject_mismatched_shapes() { let mut fixed = buf("Int8"); - encode_value(&mut fixed, &ColumnValue::Char(-3), None, 0).unwrap(); + encode_value(&mut fixed, Codec::Le, &build::char_(-3), None, 0).unwrap(); fixed.append_default(); assert_eq!(format!("{fixed:?}"), "Fixed { width: 1, bytes_len: 2 }"); assert!(matches!( @@ -4300,20 +4466,23 @@ mod tests { assert!(rejects(string.append_fixed_bytes(&[0; 4]), "string-shaped")); assert!(rejects(fixed.append_string_bytes(b"x"), "fixed-shaped")); let mut date = buf("Date32"); - let overflow = encode_value(&mut date, &ColumnValue::Date(i32::MAX - 1), None, 0); + let overflow = encode_value(&mut date, Codec::Date, &build::date(i32::MAX - 1), None, 0); assert!(rejects(overflow, "Date32")); - let toast = ColumnValue::ExternalToast(crate::decode::heap_decoder::ToastPointer { + let toast = Datum::External(crate::decode::heap_decoder::ToastPointer { va_rawsize: 0, va_extinfo: 0, va_valueid: 0, va_toastrelid: 0, }); - assert!(rejects(encode_value(&mut string, &toast, None, 0), "TOAST")); - let pending = ColumnValue::PgPending { - type_oid: 600, - raw: vec![1], - }; - let r = encode_value(&mut string, &pending, None, 0); + let toast = encode_value(&mut string, Codec::Bytes, &toast, None, 0); + assert!(rejects(toast, "TOAST")); + let r = encode_value( + &mut string, + Codec::Pending, + &Datum::BadTag(vec![1]), + None, + 0, + ); assert!(rejects(r, "unresolved source value")); } @@ -4331,25 +4500,24 @@ mod tests { #[test] fn oracle_cell_rejects_values_off_the_planned_type() { - let rejects = |v: ColumnValue, want: &str| matches!(oracle_cell(Cell::Value(v.view()), 1007), Err(EmitterError::UnsupportedValue { kind, .. }) if kind == want); - let raw = vec![0u8; 4]; + let source = |type_oid| { + Some(SourceColumn { + index: 0, + codec: Codec::Pending, + type_oid, + }) + }; + let rejects = |v: DatumRef<'_>, type_oid, want: &str| matches!(oracle_cell(v, source(type_oid), 1007), Err(EmitterError::UnsupportedValue { kind, .. }) if kind == want); let moved = "source type oid differs from the planned one"; - assert!(rejects(ColumnValue::PgPending { type_oid: 23, raw }, moved)); - let text = "{1}".into(); - assert!(rejects( - ColumnValue::PgPendingText { type_oid: 23, text }, - moved - )); + assert!(rejects(DatumRef::Varlena(&[0; 4]), 23, moved)); + assert!(rejects(DatumRef::Text(b"{1}"), 23, moved)); assert!(rejects( - ColumnValue::Int4(1), + DatumRef::Fixed(&1i32.to_le_bytes()), + 0, "decoded value on an oracle column" )); - let text = ColumnValue::PgPendingText { - type_oid: 1007, - text: "{1}".into(), - }; assert!( - matches!(oracle_cell(Cell::Value(text.view()), 1007), Ok(OracleCell::TextInput(t)) if t == b"{1}") + matches!(oracle_cell(DatumRef::Text(b"{1}"), source(1007), 1007), Ok(OracleCell::TextInput(t)) if t == b"{1}") ); } @@ -4587,10 +4755,12 @@ mod tests { NonFinite::compile(&rule, &ast) } - fn finite(text: &str) -> Option { - Some(ColumnValue::Numeric( - crate::decode::codecs::NumericKind::Finite(text.into()), - )) + /// Substitute `text` encoded for Decimal type `ty` + fn finite(ty: &str, text: &str) -> Option { + let ast = TypeAst::parse(ty, Allocator::global(&mimalloc::MiMalloc)).unwrap(); + let wire = decimal_wire_of(&ast).unwrap(); + let le = decimal_text_to_scaled_le(text, wire.scale, wire.width).unwrap(); + Some(Fill::Le(SmallVec::from_slice(&le[..wire.width.bytes()]))) } #[test] @@ -4615,16 +4785,20 @@ mod tests { &nullable, ) .unwrap(); - assert_eq!(nf.pos_inf, Some(ColumnValue::Null)); + assert_eq!(nf.pos_inf, Some(Fill::Null)); assert_eq!(nf.neg_inf, None); - for value in [ - ColumnValue::Date(i32::MIN + 1), - ColumnValue::Date(i32::MAX - 1), - ColumnValue::Timestamp(i64::MIN + 1), - ColumnValue::TimestampTz(i64::MAX - 1), + for (codec, body) in [ + (Codec::Date, (i32::MIN + 1).to_le_bytes().to_vec()), + (Codec::Date, (i32::MAX - 1).to_le_bytes().to_vec()), + (Codec::Timestamp, (i64::MIN + 1).to_le_bytes().to_vec()), + (Codec::Timestamp, (i64::MAX - 1).to_le_bytes().to_vec()), ] { - assert_eq!(nf.substitute(value.view()), value.view()); + assert_eq!(nf.fill(codec, &body), None); } + assert_eq!( + nf.fill(Codec::Date, &i32::MAX.to_le_bytes()), + Some(&Fill::Null) + ); } } @@ -4664,23 +4838,17 @@ mod tests { ( "Date32", crate::schema::DATEOID, - vec![ColumnValue::Date(i32::MIN), ColumnValue::Date(i32::MAX)], + vec![build::date(i32::MIN), build::date(i32::MAX)], ), ( "DateTime64(3, 'UTC')", crate::schema::TIMESTAMPOID, - vec![ - ColumnValue::Timestamp(i64::MIN), - ColumnValue::Timestamp(i64::MAX), - ], + vec![build::micros(i64::MIN), build::micros(i64::MAX)], ), ( "DateTime64(6, 'UTC')", crate::schema::TIMESTAMPTZOID, - vec![ - ColumnValue::TimestampTz(i64::MIN), - ColumnValue::TimestampTz(i64::MAX), - ], + vec![build::micros(i64::MIN), build::micros(i64::MAX)], ), ] { let mut rel = mk_rel(); @@ -4760,9 +4928,8 @@ mod tests { let mut encoder = TableEncoder::new(plan).unwrap(); for seconds in [-1i64, 0, 946_684_800] { let mut row = committed(1, None); - row.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Timestamp( - seconds * 1_000_000 - DATETIME64_PG_EPOCH_US, - )); + row.decoded.new.as_mut().unwrap().columns[0] = + Some(build::micros(seconds * 1_000_000 - DATETIME64_PG_EPOCH_US)); encoder.append_row(&row, &mapping, OP_INSERT).unwrap(); } let expected: Vec = [-1i64, 0, 946_684_800] @@ -4793,12 +4960,12 @@ mod tests { #[test] fn non_finite_substitutes_resolve_against_destination() { let nf = non_finite("Decimal(10, 2)", "0", "1.230", "reject").unwrap(); - assert_eq!(nf.nan, finite("0")); - assert_eq!(nf.pos_inf, finite("1.230")); + assert_eq!(nf.nan, finite("Decimal(10, 2)", "0")); + assert_eq!(nf.pos_inf, finite("Decimal(10, 2)", "1.230")); assert_eq!(nf.neg_inf, None); let nf = non_finite("Nullable(Decimal(10, 2))", "null", "reject", "reject").unwrap(); - assert_eq!(nf.nan, Some(ColumnValue::Null)); + assert_eq!(nf.nan, Some(Fill::Null)); for (ty, nan) in [ ("Decimal(10, 2)", "null"), @@ -4824,7 +4991,7 @@ mod tests { ] { for (action, literal) in [("min", format!("-{bound}")), ("max", bound.into())] { let nf = non_finite(ty, action, action, action).unwrap(); - let expected = finite(&literal); + let expected = finite(ty, &literal); assert_eq!(nf.nan, expected); assert_eq!(nf.pos_inf, expected); assert_eq!(nf.neg_inf, expected); @@ -4834,14 +5001,14 @@ mod tests { #[test] fn nan_row_lands_as_configured_substitute() { - use crate::decode::codecs::NumericKind; let alloc = Allocator::global(&mimalloc::MiMalloc); - let rel = mk_rel(); + let mut rel = mk_rel(); + rel.attributes[0].type_oid = crate::schema::NUMERICOID; + rel.attributes[0].type_len = -1; let mut m = mk_mapping(); m.columns[0].target_type = "Decimal(10, 2)".into(); let mut row = committed(1, None); - row.decoded.new.as_mut().expect("insert tuple").columns[0] = - Some(ColumnValue::Numeric(NumericKind::NaN)); + row.decoded.new.as_mut().expect("insert tuple").columns[0] = Some(build::numeric("NaN")); let plan = TablePlan::build( alloc, @@ -4982,8 +5149,7 @@ mod tests { assert_eq!(plan.columns[1].encoding, ColumnEncoding::Local, "{target}"); let mut enc = TableEncoder::new(plan).unwrap(); let mut doc = committed(1, None); - doc.decoded.new.as_mut().unwrap().columns[1] = - Some(ColumnValue::Json(r#"{"a": 1}"#.into())); + doc.decoded.new.as_mut().unwrap().columns[1] = Some(build::out_text(r#"{"a": 1}"#)); enc.append_row(&doc, &m, OP_INSERT).unwrap(); enc.append_row(&committed(2, None), &m, OP_INSERT).unwrap(); let (offsets, data, absent) = (&[8, 10][..], &br#"{"a": 1}{}"#[..], &b"{}"[..]); @@ -5057,20 +5223,27 @@ mod tests { fn append_errors_name_unsupported_columns() { let alloc = Allocator::global(&mimalloc::MiMalloc); let (rules, sys) = (ColumnRules::default(), SystemColumns::default()); - let mut m = mk_mapping(); - m.columns[0].target_type = "Array(Int32)".into(); + let m = mk_mapping(); let plan = TablePlan::build(alloc, &mk_rel(), &m, &rules, &sys).unwrap(); let mut enc = TableEncoder::new(plan).unwrap(); - let r = enc.append_row(&committed(1, None), &m, OP_INSERT); + let mut toast = committed(1, None); + toast.decoded.new.as_mut().unwrap().columns[0] = + Some(Datum::External(crate::decode::heap_decoder::ToastPointer { + va_rawsize: 0, + va_extinfo: 0, + va_valueid: 0, + va_toastrelid: 0, + })); + let r = enc.append_row(&toast, &m, OP_INSERT); assert!( - matches!(r, Err(EmitterError::UnsupportedValue { target_column, .. }) if target_column == "id") + matches!(&r, Err(EmitterError::UnsupportedValue { target_column, .. }) if target_column == "id"), + "{r:?}" ); - let m = mk_mapping(); let plan = TablePlan::build(alloc, &mk_rel(), &m, &rules, &sys).unwrap(); let mut enc = TableEncoder::new(plan).unwrap(); let mut wide = committed(1, None); - wide.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Int8(1)); + wide.decoded.new.as_mut().unwrap().columns[0] = Some(build::int8(1)); let r = enc.append_row(&wide, &m, OP_INSERT); assert!(matches!(r, Err(EmitterError::Type(_))), "{r:?}"); } @@ -5088,7 +5261,7 @@ mod tests { op: HeapOp::Delete, new: None, old: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(id)), None], + columns: vec![Some(build::int4(id)), None], partial: false, }), }, diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs index 69a436ba..8cf0d9b7 100644 --- a/src/emit/direct_parity.rs +++ b/src/emit/direct_parity.rs @@ -1,15 +1,16 @@ -//! Direct tuple-view encoding must write the same blocks, and fail with the -//! same errors, as decoding owned cells first +//! Borrowed tuple views must encode the same blocks, and fail with the same +//! errors, as owned datums resolved from them, and COPY rows as heap views use clickhouse_c::Allocator; use walrus::pg::walparser::RelFileNode; use crate::column_rules::{ColumnRule, ColumnRules, ColumnRulesBuilder, Substitute}; +use crate::decode::datum::{Datum, build}; use crate::decode::heap_decoder::{ - ColumnValue, CommittedTuple, DecodedHeap, HeapOp, decode_block_data, view_block_data, + CommittedTuple, DecodedHeap, HeapOp, decode_block_data, view_block_data, }; use crate::emit::ch_emitter::{ - ColumnBuf, Direct, OP_DELETE, OP_INSERT, RowStamp, TableEncoder, TablePlan, + Codec, ColumnBuf, OP_DELETE, OP_INSERT, RowStamp, TableEncoder, TablePlan, }; use crate::mapping::{ColumnMapping, SystemColumns, TableMapping, TableTarget}; use crate::schema::*; @@ -192,9 +193,7 @@ fn assert_parity( op_code: op, leaf: None, }; - let mut decoded = decode_block_data(&data, rel).unwrap(); - // Placement renders PostGIS cells before batching - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut decoded.columns); + let decoded = decode_block_data(&data, rel).unwrap(); let committed = CommittedTuple { decoded: DecodedHeap { rfn: rel.rfn, @@ -472,30 +471,29 @@ fn fixed_width_types_match() { } #[test] -fn copy_requires_matching_storage() { +fn codec_follows_source_type() { let cases = [ - (INT4OID, 4, 'i', "Int32", Direct::Copy), - (INT4OID, 4, 'i', "UInt32", Direct::Value), - (INT4OID, 4, 'i', "Float32", Direct::Value), - (OIDOID, 4, 'i', "Int32", Direct::Value), - (FLOAT8OID, 8, 'd', "Int64", Direct::Value), - (INT8OID, 8, 'd', "Nullable(Int64)", Direct::Copy), - (MONEYOID, 8, 'd', "Int64", Direct::Copy), - (DATEOID, 4, 'i', "Date32", Direct::Value), - (BOOLOID, 1, 'c', "Bool", Direct::Value), + (INT4OID, 4, 'i', "Int32", Codec::Le), + (OIDOID, 4, 'i', "Int32", Codec::Le), + (MONEYOID, 8, 'd', "Int64", Codec::Le), + (BOOLOID, 1, 'c', "Bool", Codec::Le), + (DATEOID, 4, 'i', "Date32", Codec::Date), + (NUMERICOID, -1, 'i', "Decimal(10, 2)", Codec::Numeric), + (JSONBOID, -1, 'i', "String", Codec::Jsonb), + (999_999, -2, 'c', "String", Codec::Bytes), ]; for (oid, len, align, target, want) in cases { let rel = rel(vec![attr(1, oid, len, align, "")]); let plan = plan(&rel, &[target], &ColumnRules::default()); assert_eq!( - plan.columns[0].source.unwrap().direct, + plan.columns[0].source.unwrap().codec, want, "{oid} {target}" ); } let rel = rel(vec![attr(1, 999_999, -1, 'd', "geometry")]); let plan = plan(&rel, &["String"], &ColumnRules::default()); - assert_eq!(plan.columns[0].source.unwrap().direct, Direct::Wkt); + assert_eq!(plan.columns[0].source.unwrap().codec, Codec::Wkt); } #[test] @@ -830,10 +828,7 @@ fn view_matches_owned_decode_for_records() { let data = tuple(&rel.attributes, 2, &[Some(&one[..]), Some(&text[..])]); let view = view_block_data(&data, &rel).unwrap(); assert_eq!(view.to_tuple(&rel), decode_block_data(&data, &rel).unwrap()); - assert_eq!( - view.to_tuple(&rel).columns[1], - Some(ColumnValue::Text("value".into())) - ); + assert_eq!(view.to_tuple(&rel).columns[1], Some(build::text("value"))); } /// Operation with new and old tuple bytes @@ -841,12 +836,7 @@ type Change = (HeapOp, Option>, Option>); /// Plan row changes both ways; segments must carry identical rows. /// External values resolve through `values` by value id, as detoast would -fn plan_both( - rel: &RelDescriptor, - targets: &[&str], - changes: &[Change], - values: &[(u32, ColumnValue)], -) { +fn plan_both(rel: &RelDescriptor, targets: &[&str], changes: &[Change], values: &[(u32, Datum)]) { let resolved = |p: crate::decode::heap_decoder::ToastPointer| { values .iter() @@ -873,7 +863,7 @@ fn plan_both( if views { let new = new.as_ref().map(|d| view_block_data(d, &rel).unwrap()); let old = old.as_ref().map(|d| view_block_data(d, &rel).unwrap()); - let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); + let mut patches: [Vec<(usize, Datum)>; 2] = Default::default(); for site in crate::xact::xact_buffer::external_sites([new.as_ref(), old.as_ref()]) { patches[site.image].push((site.idx, resolved(site.p))); } @@ -891,7 +881,7 @@ fn plan_both( let decode = |d: &Vec| { let mut t = decode_block_data(d, &rel).unwrap(); for c in t.columns.iter_mut() { - if let Some(ColumnValue::ExternalToast(p)) = *c { + if let Some(Datum::External(p)) = *c { *c = Some(resolved(p)); } } @@ -998,7 +988,7 @@ fn plan_views_match_cells_with_resolved_externals() { let ext = external_id(value_id); tuple(&attrs, 2, &[Some(&id.to_le_bytes()[..]), Some(&ext[..])]) }; - let text = |s: &str| ColumnValue::Text(s.into()); + let text = build::text; plan_both( &r, &["Int32", "String"], @@ -1044,9 +1034,9 @@ fn plan_views_match_cells_across_value_forms() { for target in ["String", "Nullable(String)"] { for filled in [false, true] { let eight = if filled { - ColumnValue::Null + Datum::Null } else { - ColumnValue::Text("eight".into()) + build::text("eight") }; plan_both( &r, @@ -1065,7 +1055,7 @@ fn plan_views_match_cells_across_value_forms() { ), (HeapOp::Delete, None, Some(row(packed(), 8))), ], - &[(7, ColumnValue::Text("seven".into())), (8, eight)], + &[(7, build::text("seven")), (8, eight)], ); } } @@ -1074,6 +1064,16 @@ fn plan_views_match_cells_across_value_forms() { /// Heap datum and COPY field of one value type HeapAndCopy = (Vec, Vec); +/// `numeric_send` words of a finite value, big-endian +fn numeric_send(neg: bool, weight: i16, dscale: u16, digits: &[i16]) -> Vec { + let sign = if neg { 0x4000 } else { 0 }; + [digits.len() as u16, weight as u16, sign, dscale] + .into_iter() + .chain(digits.iter().map(|&d| d as u16)) + .flat_map(u16::to_be_bytes) + .collect() +} + /// COPY binary and text fields encode as page-walk views of same values do #[test] fn copy_fields_match_heap_views() { @@ -1109,8 +1109,14 @@ fn copy_fields_match_heap_views() { Some((vec![1], vec![1])), Some((8000i32.to_le_bytes().into(), 8000i32.to_be_bytes().into())), Some((lz4(&b"y".repeat(80)), b"y".repeat(80))), - Some((short(&numeric_short(false, 2, &[1, 5000])), b"1.50".into())), - Some((short(&numeric_short(true, 0, &[42])), b"-42".into())), + Some(( + short(&numeric_short(false, 2, &[1, 5000])), + numeric_send(false, 0, 2, &[1, 5000]), + )), + Some(( + short(&numeric_short(true, 0, &[42])), + numeric_send(true, 0, 0, &[42]), + )), ], vec![ Some((0i32.to_le_bytes().into(), 0i32.to_be_bytes().into())), @@ -1122,8 +1128,17 @@ fn copy_fields_match_heap_views() { Some((vec![0], vec![0])), Some(((-1i32).to_le_bytes().into(), (-1i32).to_be_bytes().into())), None, - Some((short(&0xC000u16.to_le_bytes()), b"NaN".into())), - Some((short(&numeric_short(false, 0, &[])), b"0".into())), + Some(( + short(&0xC000u16.to_le_bytes()), + [0u16, 0, 0xC000, 0] + .iter() + .flat_map(|w| w.to_be_bytes()) + .collect(), + )), + Some(( + short(&numeric_short(false, 0, &[])), + numeric_send(false, 0, 0, &[]), + )), ], ]; let mut views = TableEncoder::new(plan(&rel, &targets, &ColumnRules::default())).unwrap(); @@ -1147,12 +1162,9 @@ fn copy_fields_match_heap_views() { .iter() .map(|c| c.as_ref().map(|(_, f)| &f[..])) .collect(); - let fields = CopyRow::new(&fields); - copy.append_slots( - |src| copy_slot(&fields, &rel, src.index).map_err(crate::ch::EmitterError::Type), - &stamp, - ) - .unwrap(); + let fields = CopyRow::from_send(&rel, &fields).unwrap(); + copy.append_slots(|src| Ok(copy_slot(&fields, &rel, src.index)), &stamp) + .unwrap(); } let wire = |enc: &mut TableEncoder| { let (bufs, rows) = enc.take_block().unwrap(); diff --git a/src/emit/pipeline/batcher.rs b/src/emit/pipeline/batcher.rs index 4ebc641a..906218d5 100644 --- a/src/emit/pipeline/batcher.rs +++ b/src/emit/pipeline/batcher.rs @@ -88,7 +88,7 @@ impl PlacedSegment { #[cfg(test)] pub(crate) fn segment_rows( segments: &[PlacedSegment], -) -> Vec<(u64, Vec)> { +) -> Vec<(u64, Vec)> { segments .iter() .flat_map(|s| { @@ -616,9 +616,8 @@ async fn flush_all( #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{ - ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp, - }; + use crate::decode::datum::build; + use crate::decode::heap_decoder::{CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; use tokio::sync::oneshot; @@ -705,7 +704,7 @@ mod tests { source_lsn: 0x1000 + id as u64, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(id))], + columns: vec![Some(build::int4(id))], partial: false, }), old: None, @@ -1169,7 +1168,7 @@ mod tests { let (rel, route) = text_table("String"); let mut r = tuple(i, 1); r.1.decoded.new.as_mut().unwrap().columns = - vec![Some(ColumnValue::Text("x".repeat(64 << 10)))]; + vec![Some(build::text("x".repeat(64 << 10)))]; tx.send(one(rel, route, r)).await.unwrap(); } let first = rx.recv().await.unwrap(); @@ -1207,7 +1206,7 @@ mod tests { fn text_row(seq: u64, id: i32, text: &str) -> (u64, CommittedTuple) { let mut r = tuple(seq, id); - r.1.decoded.new.as_mut().unwrap().columns = vec![Some(ColumnValue::Text(text.into()))]; + r.1.decoded.new.as_mut().unwrap().columns = vec![Some(build::text(text))]; r } @@ -1421,7 +1420,8 @@ mod tests { let (rel, route) = date_table("Date32"); let (_, mut overflow) = tuple(0, 1); - overflow.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Date(i32::MAX - 1)); + overflow.decoded.new.as_mut().unwrap().columns[0] = + Some(crate::decode::datum::build::date(i32::MAX - 1)); let msg = fault(rel, route, overflow); assert!(msg.contains("Date32 epoch"), "{msg}"); let (rel, route) = date_table("Int32"); diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index 8b67ea23..60858b3c 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -17,10 +17,9 @@ use crate::backfill::visibility_pending::metadata_patch; use crate::backfill::wal_replay::HeapTid; use crate::backfill::walk_barrier::{WALK_CHECKPOINT_PERIOD, WalkBarrier}; use crate::budget::MemoryPermit; -use crate::ch::EmitterError; use crate::config::ResolvedConfig; -use crate::decode::datum::DatumRef; -use crate::decode::heap_decoder::{ColumnValue, ToastPointer, TupleView, view_block_data_into}; +use crate::decode::datum::{Datum, DatumRef}; +use crate::decode::heap_decoder::{ToastPointer, TupleView, view_block_data_into}; use crate::emit::ch_emitter::{EmitterStats, OP_INSERT, RowStamp}; use crate::emit::pipeline::ack::{AckHandle, OpenSeq, Publish, SeqAlloc}; use crate::emit::pipeline::batcher::{ @@ -525,7 +524,7 @@ struct LoadRow { table: u32, tuple: BackfillTuple, /// Resolved external values - patch: Vec<(usize, ColumnValue)>, + patch: Vec<(usize, Datum)>, } impl Slab { @@ -535,7 +534,7 @@ impl Slab { rel: &Arc, route: &Arc, tuple: BackfillTuple, - patch: Vec<(usize, ColumnValue)>, + patch: Vec<(usize, Datum)>, ) { let table = match self .tables @@ -570,7 +569,7 @@ fn encode_row<'r>( tuple: &'r BackfillTuple, rel: &Arc, route: &Arc, - mut patch: Vec<(usize, ColumnValue)>, + mut patch: Vec<(usize, Datum)>, ) -> Result<(), String> { let key = Arc::as_ptr(rel) as usize; let stamp = RowStamp { @@ -589,10 +588,7 @@ fn encode_row<'r>( }) } TupleData::Copy(row) => chunks.append(key, rel, route, |enc| { - enc.append_slots( - |src| copy_slot(row, rel, src.index).map_err(EmitterError::Type), - &stamp, - ) + enc.append_slots(|src| Ok(copy_slot(row, rel, src.index)), &stamp) }), }; appended.map_err(|e| format!("bootstrap: encode {}: {e}", rel.rel_name)) @@ -649,7 +645,7 @@ impl LoadOut { rel: &Arc, route: &Arc, tuple: BackfillTuple, - patch: Vec<(usize, ColumnValue)>, + patch: Vec<(usize, Datum)>, ) -> Result<(), String> { self.slab.push(seq, rel, route, tuple, patch); if self.slab.due() { @@ -925,7 +921,7 @@ struct ReplayRow { table: u32, pointers: Vec, /// Values settled for external attributes - patch: Vec<(usize, ColumnValue)>, + patch: Vec<(usize, Datum)>, replay_covered: bool, } @@ -956,13 +952,13 @@ fn mapped_pointers(view: &TupleView<'_>, mapping: &TableMapping) -> Vec, - patch: &mut Vec<(usize, ColumnValue)>, + patch: &mut Vec<(usize, Datum)>, resolver: &ToastResolver, ) { sites.retain(|site| { let filled = resolver.fills_oversize(&site.p); if filled { - patch.push((site.idx, ColumnValue::Null)); + patch.push((site.idx, Datum::Null)); } !filled }); @@ -1163,9 +1159,8 @@ async fn resolve_batch( } let RowTable { rel, route } = &batch.tables[*table as usize]; for (site, fetched) in pointers.iter().zip(fetched) { - let type_oid = rel.attributes.get(site.idx).map_or(0, |a| a.type_oid); let target = &route.mapping.columns[site.col].target_name; - let (column, bytes) = apply_fetched(fetched, &site.p, type_oid, rel, target, resolver)?; + let (column, bytes) = apply_fetched(fetched, &site.p, rel, target, resolver)?; retained += bytes; patch.push((site.idx, column)); } @@ -1261,15 +1256,14 @@ async fn fetch_values_at<'a>( fn apply_fetched( fetched: Option, p: &ToastPointer, - type_oid: u32, rel: &RelDescriptor, target: &str, resolver: &ToastResolver, -) -> Result<(ColumnValue, usize), String> { +) -> Result<(Datum, usize), String> { let miss = match StoreMiss::split(fetched) { Ok(stored) => { let raw = finish_value(p, stored).map_err(|e| e.to_string())?; - return Ok(detoasted_sized(raw, type_oid)); + return Ok(detoasted_sized(raw)); } Err(miss) => miss, }; @@ -1289,7 +1283,7 @@ fn apply_fetched( fn fill_toast( mut sites: Vec, resolver: &ToastResolver, -) -> Result, String> { +) -> Result, String> { check_value_caps( sites.iter().map(|site| site.p), resolver.inline_value_max(), @@ -1300,7 +1294,7 @@ fn fill_toast( fill_oversize(&mut sites, &mut patch, resolver); for site in &sites { resolver.note_filled_default(); - patch.push((site.idx, ColumnValue::Null)); + patch.push((site.idx, Datum::Null)); } Ok(patch) } @@ -1367,7 +1361,7 @@ mod tests { } /// Flatten the drain's segments back to `(seq, mapped cells)` rows - async fn collect_rows(rx: &mut mpsc::Receiver) -> Vec<(u64, Vec)> { + async fn collect_rows(rx: &mut mpsc::Receiver) -> Vec<(u64, Vec)> { let mut rows = Vec::new(); // Any other message ends collection short of the expected rows while let Some(BatcherMsg::Segments(segments)) = rx.recv().await { @@ -1391,6 +1385,7 @@ mod tests { } use super::*; + use crate::decode::datum::{Datum, build}; use crate::emit::pipeline::batcher::segment_rows; use crate::toast::ToastRow; @@ -1431,7 +1426,7 @@ mod tests { Arc::new(EmitterStats::default()), ); assert!(seeded.stores_chunks()); - let err = apply_fetched(short.clone(), &ptr, 25, &rel, "body", &seeded) + let err = apply_fetched(short.clone(), &ptr, &rel, "body", &seeded) .expect_err("a seeded mirror losing its own chunk must stop the load"); assert!( err.contains("chunks sum to 7984 bytes, pointer says 9100"), @@ -1454,9 +1449,9 @@ mod tests { .await .unwrap(); assert_eq!(fetched, short); - let (column, retained) = apply_fetched(fetched, &ptr, 25, &rel, "body", &read_only) + let (column, retained) = apply_fetched(fetched, &ptr, &rel, "body", &read_only) .expect("a read-only backend fills instead of failing the load"); - assert_eq!(column, ColumnValue::Null); + assert_eq!(column, Datum::Null); assert_eq!(retained, 0); assert_eq!( stats.toast_values_filled_superseded.load(Ordering::Relaxed), @@ -1478,7 +1473,7 @@ mod tests { DEFERRED_SPOOL_MEM_MAX, ) } - use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; + use crate::decode::heap_decoder::ToastPointer; use crate::emit::pipeline::batcher::BatcherMsg; use crate::emit::pipeline::{Fatal, ack}; use crate::schema::{RelAttr, RelDescriptor, RelName, ReplIdent}; @@ -1546,7 +1541,7 @@ mod tests { page_lsn: 0, blkno: 0, offnum: 0, - data: TupleData::from_cells(&[Some(ColumnValue::Int4(id))]), + data: TupleData::from_cells(&[Some(build::int4(id))]), pending: None, } } @@ -1632,10 +1627,7 @@ mod tests { /// Main-rel tuple whose columns are on-disk TOAST pointers fn referrer_tuple(rel_node: u32, pointers: &[ToastPointer]) -> BackfillTuple { - let cells: Vec<_> = pointers - .iter() - .map(|p| Some(ColumnValue::ExternalToast(*p))) - .collect(); + let cells: Vec<_> = pointers.iter().map(|p| Some(Datum::External(*p))).collect(); BackfillTuple { rfn: RelFileNode { spc_node: 1663, @@ -1671,9 +1663,9 @@ mod tests { blkno: 1, offnum: 1, data: TupleData::from_cells(&[ - Some(ColumnValue::Oid(value_id)), - Some(ColumnValue::Int4(seq)), - Some(ColumnValue::Bytea(body.to_vec())), + Some(build::oid(value_id)), + Some(build::int4(seq)), + Some(build::bytes(body)), ]), pending: None, } @@ -1825,7 +1817,7 @@ mod tests { batcher_open: bool, ) -> ( Result, - Vec<(u64, Vec)>, + Vec<(u64, Vec)>, ) { let mut catalog = CatalogMap::new(); catalog.insert(rel(16400)); @@ -2011,7 +2003,7 @@ mod tests { assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); let cols = &rows[0].1; - assert_eq!(cols[0], ColumnValue::Null, "unresolved toast NULL-filled"); + assert_eq!(cols[0], Datum::Null, "unresolved toast NULL-filled"); assert_eq!(stats.toast_values_filled_default.load(Ordering::Relaxed), 1); drop(ack); collector.await.unwrap(); @@ -2060,7 +2052,7 @@ mod tests { assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); let cols = &rows[0].1; - assert_eq!(cols[0], ColumnValue::Null); + assert_eq!(cols[0], Datum::Null); assert_eq!( stats.toast_values_filled_oversize.load(Ordering::Relaxed), 1 @@ -2127,7 +2119,7 @@ mod tests { assert_eq!(rows.len(), 1); assert_eq!(rows[0].0, 2, "deferred row routed under the trailing seq"); let cols = &rows[0].1; - assert_eq!(cols[0], ColumnValue::Bytea(b"hello".to_vec())); + assert_eq!(cols[0], build::bytes(b"hello")); assert_eq!(stats.toast_chunks_stored.load(Ordering::Relaxed), 1); assert_eq!(stats.toast_values_fetched.load(Ordering::Relaxed), 1); drop(ack); @@ -2343,7 +2335,7 @@ mod tests { let rows = collect_rows(&mut msg_rx).await; assert_eq!(rows.len(), 1); let cols = &rows[0].1; - assert_eq!(cols[0], ColumnValue::Bytea(b"hello".to_vec())); + assert_eq!(cols[0], build::bytes(b"hello")); drop(ack); collector.await.unwrap(); } @@ -2513,10 +2505,7 @@ mod tests { ); assert_eq!( chunk.cells(), - [[ - ColumnValue::Bytea(b.to_vec()), - ColumnValue::Bytea(c.to_vec()) - ]], + [[build::bytes(b), build::bytes(c)]], "{label}" ); assert_eq!(covered, 0, "{label}"); @@ -2667,7 +2656,7 @@ mod tests { assert_eq!(outcome.rows_routed, 1); assert_eq!(rows.len(), 1); let cols = &rows[0].1; - assert_eq!(cols[0], ColumnValue::Bytea(b"hello".to_vec())); + assert_eq!(cols[0], build::bytes(b"hello")); drop(ack); collector.await.unwrap(); } @@ -2781,7 +2770,7 @@ mod tests { let outcome = drain_task.await.unwrap().unwrap(); assert_eq!(outcome.rows_routed as usize, rows); let got: Vec<_> = routed.iter().map(|r| r.1[0].clone()).collect(); - let want: Vec<_> = bodies.into_iter().map(ColumnValue::Bytea).collect(); + let want: Vec<_> = bodies.into_iter().map(build::bytes).collect(); assert_eq!(got, want, "replay order kept across the batch boundary"); assert_eq!( stats.toast_value_fetch_batches.load(Ordering::Relaxed), @@ -3039,7 +3028,7 @@ mod tests { .await .unwrap(); for (row, expected) in batch.rows.iter().zip([b"first", b"other"]) { - assert_eq!(row.patch, [(0, ColumnValue::Bytea(expected.to_vec()))]); + assert_eq!(row.patch, [(0, build::bytes(expected))]); } } @@ -3121,7 +3110,7 @@ mod tests { assert_eq!( bodies, [b"hello", b"hello", b"world", b"there"] - .map(|b| ColumnValue::Bytea(b.to_vec())) + .map(build::bytes) .to_vec(), "replay order kept, shared value resolved for both referrers", ); diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index 270153d0..962aec85 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -26,8 +26,8 @@ use clickhouse_c::Allocator; use thiserror::Error; use crate::ch::EmitterError; -use crate::decode::datum::DatumRef; -use crate::decode::heap_decoder::{ColumnValue, HeapOp, TupleView, datum_value}; +use crate::decode::datum::{Datum, DatumRef}; +use crate::decode::heap_decoder::{HeapOp, TupleView, inflate, missing_datum}; use crate::emit::ch_emitter::{ Append, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, RowStamp, TableEncoder, TablePlan, }; @@ -723,16 +723,16 @@ pub struct ViewChange<'v, 'a> { pub source_lsn: u64, pub new: Option<&'v TupleView<'a>>, pub old: Option<&'v TupleView<'a>>, - pub new_patch: &'v [(usize, ColumnValue)], - pub old_patch: &'v [(usize, ColumnValue)], + pub new_patch: &'v [(usize, Datum)], + pub old_patch: &'v [(usize, Datum)], } /// Whether an UPDATE changes primary/index key, or any column under /// `REPLICA IDENTITY FULL` without a primary key. PostgreSQL default/index /// identity logs old values only when key changes or contains TOASTed values -fn key_moved( +fn key_moved<'v>( rel: &RelDescriptor, - change: &ViewChange<'_, '_>, + change: &ViewChange<'v, '_>, old: &TupleView<'_>, new: &TupleView<'_>, ) -> bool { @@ -741,25 +741,23 @@ fn key_moved( HeapOp::HotUpdate => true, _ => return false, }; - let patched = |patch: &[(usize, ColumnValue)], a: usize| { - patch.iter().find(|(i, _)| *i == a).map(|(_, v)| v.clone()) - }; let differs = |a: usize| { let (Some(&o), Some(&n), Some(att)) = (old.datums.get(a), new.datums.get(a), rel.attributes.get(a)) else { return false; }; - match (patched(change.old_patch, a), patched(change.new_patch, a)) { - (None, None) => datums_differ(att, o, n), - (po, pn) => values_differ( - att, - [ - po.or_else(|| datum_value(att, o)), - pn.or_else(|| datum_value(att, n)), - ], - ), - } + let patched = |patch: &'v [(usize, Datum)], d| { + patch + .iter() + .find(|(i, _)| *i == a) + .map_or(d, |(_, v)| v.view()) + }; + datums_differ( + att, + patched(change.old_patch, o), + patched(change.new_patch, n), + ) }; let keys = crate::schema::replident_key_attnums(rel); if keys.is_empty() { @@ -773,34 +771,39 @@ fn key_moved( } } -/// Cell inequality as owned decode compares it. Equal stored bytes decode -/// equal except float NaN, which never equals itself +/// Stored-byte inequality, as PostgreSQL's `datumIsEqual` judges index +/// keys; absent never differs. Floats compare by value, so NaN differs from +/// itself and signed zeros match fn datums_differ(att: &crate::schema::RelAttr, o: DatumRef<'_>, n: DatumRef<'_>) -> bool { use crate::schema::{FLOAT4OID, FLOAT8OID}; match (o, n) { (DatumRef::Absent, _) | (_, DatumRef::Absent) => false, - (DatumRef::Fixed(a), DatumRef::Fixed(b)) - if a == b && !matches!(att.type_oid, FLOAT4OID | FLOAT8OID) => - { - false + (DatumRef::Missing, d) | (d, DatumRef::Missing) => { + datums_differ(att, missing_datum(att), d) + } + (DatumRef::Compressed(c), d) | (d, DatumRef::Compressed(c)) => { + inflate(c).is_none_or(|raw| datums_differ(att, DatumRef::Varlena(&raw), d)) } - (DatumRef::Varlena(a), DatumRef::Varlena(b)) if a == b => false, (DatumRef::Null | DatumRef::Dropped, DatumRef::Null | DatumRef::Dropped) => false, - _ => values_differ(att, [o, n].map(|d| datum_value(att, d))), + (DatumRef::Fixed(a), DatumRef::Fixed(b)) => match att.type_oid { + FLOAT4OID => float_bits_differ::<4>(a, b, |w| f64::from(f32::from_le_bytes(w))), + FLOAT8OID => float_bits_differ::<8>(a, b, f64::from_le_bytes), + _ => a != b, + }, + ( + DatumRef::Varlena(a) | DatumRef::Cstring(a) | DatumRef::Text(a), + DatumRef::Varlena(b) | DatumRef::Cstring(b) | DatumRef::Text(b), + ) => a != b, + (DatumRef::External(a), DatumRef::External(b)) => a != b, + _ => true, } } -/// Owned cells compared after PostGIS render; absent never differs -fn values_differ(att: &crate::schema::RelAttr, mut values: [Option; 2]) -> bool { - if values.iter().any(Option::is_none) { - return false; +fn float_bits_differ(a: &[u8], b: &[u8], value: fn([u8; N]) -> f64) -> bool { + match (a.try_into(), b.try_into()) { + (Ok(a), Ok(b)) => value(a) != value(b), + _ => a != b, } - if !att.dropped && matches!(att.type_name.as_str(), "geography" | "geometry") { - for v in values.iter_mut().flatten() { - crate::ops::oracle::render_ext_value(v); - } - } - values[0] != values[1] } pub(crate) fn op_code(op: HeapOp) -> std::result::Result { @@ -999,7 +1002,8 @@ fn read_frame(f: &mut FileInput, tables: &[PlanTable]) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple}; + use crate::decode::datum::{Datum, build}; + use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple}; use crate::emit::ch_emitter::Wire; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{INT4OID, RelAttr, RelName, SchemaEvent}; @@ -1048,12 +1052,8 @@ mod tests { (v.attnum, v.name, v.type_oid) = (2, "v".into(), crate::schema::TEXTOID); (v.type_name, v.type_byval, v.type_len) = ("text".into(), false, -1); rel.attributes.push(v); - let row = |id: i32, v: &str| { - form_block_data(&[ - Some(ColumnValue::Int4(id)), - Some(ColumnValue::Text(v.into())), - ]) - }; + let row = + |id: i32, v: &str| form_block_data(&[], &[Some(build::int4(id)), Some(build::text(v))]); let pk = || ReplIdent::Default { pk_attnums: Some(vec![1]), }; @@ -1105,7 +1105,7 @@ mod tests { source_lsn: lsn, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(v))], + columns: vec![Some(build::int4(v))], partial: false, }), old: None, @@ -1280,7 +1280,7 @@ mod tests { let d = descriptor(16500); let image = |v| { Some(DecodedTuple { - columns: vec![Some(ColumnValue::Int4(v))], + columns: vec![Some(build::int4(v))], partial: false, }) }; @@ -1416,14 +1416,8 @@ mod tests { let mut w = PlanWriter::create(path, 1 << 20, mem_max); let table = w.table(&d, 0x40, || Some(route_to("String"))); for (lsn, v) in [ - ( - 100, - Some(ColumnValue::PgPending { - type_oid: 3614, - raw: vec![1, 2, 3], - }), - ), - (110, Some(ColumnValue::Null)), + (100, Some(build::bytes(vec![1, 2, 3]))), + (110, Some(Datum::Null)), (120, None), ] { let mut h = heap(&d, lsn, 0); diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index d79e96cf..f8240104 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -39,7 +39,8 @@ use std::sync::Arc; -use crate::decode::heap_decoder::{ColumnValue, view_heap_record}; +use crate::decode::datum::Datum; +use crate::decode::heap_decoder::view_heap_record; use crate::emit::pipeline::plan_spool::{ DEFAULT_PLAN_MEM_MAX, EncoderCache, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, @@ -241,13 +242,13 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { if h.new.as_ref().is_some_and(|t| t.partial) { return Err(PlanError::PartialUpdate { lsn }); } - let mut patches: [Vec<(usize, ColumnValue)>; 2] = Default::default(); + let mut patches: [Vec<(usize, Datum)>; 2] = Default::default(); let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); if !sites.is_empty() { let db = rel.rfn.db_node; // Value permit drops once bytes land in the plan let (values, _permit) = - detoast_sites(&sites, rel, db, lsn, spool, ref_maps, self.resolver).await?; + detoast_sites(&sites, db, lsn, spool, ref_maps, self.resolver).await?; for (site, v) in sites.iter().zip(values) { patches[site.image].push((site.idx, v)); } @@ -282,8 +283,9 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { #[cfg(test)] mod tests { use super::*; + use crate::decode::datum::{Datum, build}; use crate::decode::heap_decoder::{ - ColumnValue, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, ToastPointer, + DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, ToastPointer, }; use crate::emit::pipeline::plan_spool::PlanItem; use crate::mapping::{TableMapping, TableTarget}; @@ -393,7 +395,7 @@ mod tests { }) } - fn heap(rel: &Arc, lsn: u64, col: ColumnValue) -> DescribedHeap { + fn heap(rel: &Arc, lsn: u64, col: Datum) -> DescribedHeap { DescribedHeap { decoded: DecodedHeap { rfn: rel.rfn, @@ -510,12 +512,8 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = descriptor(16600, INT4OID); - b.on_heap(heap(&rel, 100, ColumnValue::Int4(1))) - .await - .unwrap(); - b.on_heap(heap(&rel, 120, ColumnValue::Int4(2))) - .await - .unwrap(); + b.on_heap(heap(&rel, 100, build::int4(1))).await.unwrap(); + b.on_heap(heap(&rel, 120, build::int4(2))).await.unwrap(); b.on_schema_event( 1, 150, @@ -525,9 +523,7 @@ mod tests { partition: None, }, ); - b.on_heap(heap(&rel, 200, ColumnValue::Int4(3))) - .await - .unwrap(); + b.on_heap(heap(&rel, 200, build::int4(3))).await.unwrap(); let mut view = MapView { routes: HashMap::from_iter([(rel.rel_name.clone(), route())]), @@ -596,13 +592,11 @@ mod tests { }) .unwrap(); let rel = descriptor(16601, TEXTOID); - b.on_heap(heap(&rel, 90, ColumnValue::Text("ok".into()))) - .await - .unwrap(); + b.on_heap(heap(&rel, 90, build::text("ok"))).await.unwrap(); b.on_heap(heap( &rel, 100, - ColumnValue::ExternalToast(ToastPointer { + Datum::External(ToastPointer { va_rawsize: 64, va_extinfo: 60, va_valueid: 9999, @@ -709,7 +703,7 @@ mod tests { // Encoded rows of empty mappings cost about 21 bytes each const ROWS: usize = 64 * 1024; for i in 0..ROWS { - b.on_heap(heap(&rel, 100 + i as u64, ColumnValue::Int4(i as i32))) + b.on_heap(heap(&rel, 100 + i as u64, build::int4(i as i32))) .await .unwrap(); } @@ -787,7 +781,7 @@ mod tests { ) .await .unwrap(); - b.on_heap(heap(&rel, lsn + 1, ColumnValue::Int4(seq as i32))) + b.on_heap(heap(&rel, lsn + 1, build::int4(seq as i32))) .await .unwrap(); } @@ -800,7 +794,7 @@ mod tests { partition: None, }, ); - let mut truncate = heap(&rel, 105, ColumnValue::Int4(0)); + let mut truncate = heap(&rel, 105, build::int4(0)); truncate.decoded.op = HeapOp::Truncate; truncate.decoded.new = None; b.on_heap(truncate).await.unwrap(); @@ -860,7 +854,7 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = descriptor(16607, INT4OID); - let mut partial = heap(&rel, 100, ColumnValue::Int4(1)); + let mut partial = heap(&rel, 100, build::int4(1)); partial.decoded.op = HeapOp::Update; partial.decoded.new.as_mut().unwrap().partial = true; b.on_heap(partial).await.unwrap(); @@ -915,7 +909,7 @@ mod tests { b.on_heap(heap( &rel, 100, - ColumnValue::ExternalToast(ToastPointer { + Datum::External(ToastPointer { va_rawsize: 64, va_extinfo: 60, va_valueid: 9999, @@ -991,7 +985,7 @@ mod tests { rel: &Arc, resolver: &ToastResolver, dir: &std::path::Path, - ) -> Vec { + ) -> Vec { let mut view = MapView { routes: HashMap::from_iter([(rel.rel_name.clone(), text_route())]), dropped: HashSet::new(), @@ -1032,9 +1026,9 @@ mod tests { let body = &data[start..*end as usize]; start = *end as usize; out.push(if *null == 1 { - ColumnValue::Null + Datum::Null } else { - ColumnValue::Text(String::from_utf8(body.to_vec()).unwrap()) + build::text(String::from_utf8(body.to_vec()).unwrap()) }); } } @@ -1055,10 +1049,10 @@ mod tests { const OIDS: u32 = 5; let tmp = tempfile::tempdir().unwrap(); let rel = descriptor(16699, TEXTOID); - let text = |body: &[u8]| ColumnValue::Text(String::from_utf8(body.to_vec()).unwrap()); + let text = |body: &[u8]| build::text(String::from_utf8(body.to_vec()).unwrap()); let row = |xid, lsn, op, vid| { let t = Some(DecodedTuple { - columns: vec![Some(ColumnValue::ExternalToast(ToastPointer { + columns: vec![Some(Datum::External(ToastPointer { va_rawsize: 12, va_extinfo: 8, va_valueid: vid, @@ -1206,9 +1200,9 @@ mod tests { // Prescan reads ahead the values rows keep if born.is_none() && let Some(t) = h.decoded.new.as_ref().or(h.decoded.old.as_ref()) - && let Some(ColumnValue::ExternalToast(p)) = t.columns[0] + && let Some(Datum::External(p)) = t.columns[0] { - ahead.read_ahead(rel.rfn.db_node, p, TEXTOID, h.decoded.source_lsn); + ahead.read_ahead(rel.rfn.db_node, p, h.decoded.source_lsn); tokio::task::yield_now().await; } if h.decoded.xid >= 1_000_000 { @@ -1241,7 +1235,7 @@ mod tests { assert_eq!(served.len(), want.len()); for ((w, r), s) in want.iter().zip(&read).zip(&served) { assert!( - s == w || r != w && *s == ColumnValue::Null, + s == w || r != w && *s == Datum::Null, "seed {seed} xid {xid}: {s:?} {r:?} {w:?}" ); } diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index 6a5433c5..f753070f 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -368,9 +368,8 @@ async fn resolve_oracle( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::decode::heap_decoder::{ - ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp, - }; + use crate::decode::datum::build; + use crate::decode::heap_decoder::{CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use crate::emit::ch_emitter::Wire; use crate::emit::pipeline::batcher::{BatcherConfig, BatcherMsg, PlacedSegment, RowTable}; use crate::emit::route::RouteSnapshot; @@ -431,7 +430,7 @@ pub(crate) mod tests { RowTable { rel, route } } - pub(crate) fn wkt_row(seq: u64, text: Option<&str>) -> (u64, CommittedTuple) { + pub(crate) fn wkt_row(seq: u64, point: Option<(f64, f64)>) -> (u64, CommittedTuple) { ( seq, CommittedTuple { @@ -445,7 +444,9 @@ pub(crate) mod tests { source_lsn: 0x1000 + seq, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![text.map(|t| ColumnValue::Text(t.into()))], + columns: vec![ + point.map(|(x, y)| build::bytes(crate::ops::oracle::point_body(x, y))), + ], partial: false, }), old: None, @@ -505,10 +506,9 @@ pub(crate) mod tests { #[tokio::test] async fn rendered_cells_against_a_string_target_stay_local() { let mut batch = seal(vec![ - wkt_row(0, Some("POINT(1 2)")), + wkt_row(0, Some((1.0, 2.0))), wkt_row(0, None), - wkt_row(0, Some("")), - wkt_row(0, Some("POINT(3 4)")), + wkt_row(0, Some((3.0, 4.0))), ]) .await; assert!( @@ -527,9 +527,9 @@ pub(crate) mod tests { assert_eq!( batch.blocks[0].buffers[0].wire(), Wire::NullableString { - offsets: &[10, 10, 10, 20], + offsets: &[10, 10, 20], data: b"POINT(1 2)POINT(3 4)", - null_map: &[0, 1, 0, 0], + null_map: &[0, 1, 0], absent: b"", } ); @@ -551,7 +551,7 @@ pub(crate) mod tests { let gauge = Arc::new(std::sync::atomic::AtomicU64::new(0)); let budget = output_pool(4 << 10, 1, false, gauge.clone()); let queued = budget.admit(budget.admission_max()).await; - let batch = seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await; + let batch = seal(vec![wkt_row(0, Some((1.0, 2.0)))]).await; let (batch_tx, batch_rx) = async_channel::bounded(1); let (resolved_tx, resolved_rx) = async_channel::bounded(1); let fatal = Fatal::new(); @@ -591,12 +591,9 @@ pub(crate) mod tests { /// when its neighbours are rendered #[tokio::test] async fn one_unrendered_cell_keeps_the_column_remote() { - let mut rows = vec![wkt_row(0, Some("POINT(1 2)"))]; + let mut rows = vec![wkt_row(0, Some((1.0, 2.0)))]; let mut raw = wkt_row(0, None); - raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { - type_oid: GEOM_OID, - raw: vec![1, 2, 3], - }); + raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(build::bytes(vec![1, 2, 3])); rows.push(raw); let mut batch = seal(rows).await; @@ -625,6 +622,7 @@ pub(crate) mod tests { stats: Arc::default(), fatal: fatal.clone(), config_rx: None, + budget: MemoryBudget::new(1 << 30), }; let (in_tx, in_rx) = async_channel::bounded(1); let (out_tx, out_rx) = async_channel::bounded(1); @@ -640,14 +638,11 @@ pub(crate) mod tests { #[tokio::test] async fn resolver_exits_on_oracle_failure_or_closed_output() { let mut raw = wkt_row(0, None); - raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { - type_oid: GEOM_OID, - raw: vec![1], - }); + raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(build::bytes(vec![1])); let fatal = run_one(seal(vec![raw]).await, true).await; let msg = fatal.message().expect("missing oracle trips fatal"); assert!(msg.starts_with("oracle resolve: "), "{msg}"); - let fatal = run_one(seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await, false).await; + let fatal = run_one(seal(vec![wkt_row(0, Some((1.0, 2.0)))]).await, false).await; assert!(fatal.message().is_none()); } @@ -656,17 +651,14 @@ pub(crate) mod tests { async fn unanswered_oracle_column_and_bad_type_fail_seal() { let mut sealer = Sealer::new(CompressionChoice::None).unwrap(); let mut raw = wkt_row(0, None); - raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(ColumnValue::Unsupported { - type_oid: GEOM_OID, - raw: vec![1], - }); + raw.1.decoded.new.as_mut().unwrap().columns[0] = Some(build::bytes(vec![1])); let batch = seal(vec![raw]).await; let Err(e) = sealer.seal(&batch.meta, &batch.blocks[0], None) else { panic!("unanswered oracle column seals"); }; assert!(e.to_string().contains("oracle answered no column"), "{e}"); - let batch = seal(vec![wkt_row(0, Some("POINT(1 2)"))]).await; + let batch = seal(vec![wkt_row(0, Some((1.0, 2.0)))]).await; let mut columns = batch.meta.columns.clone(); columns[0].type_repr = "Nope(((".into(); let meta = BatchMeta { diff --git a/src/lib.rs b/src/lib.rs index 097ffd89..7b3e8930 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,7 +67,7 @@ pub use backfill::{ #[doc(hidden)] pub use catalog::{desc_log, pending, shadow, shadow_catalog, type_bridge}; #[doc(hidden)] -pub use decode::{codecs, decoder_sink, fpi, heap_decoder, visibility, wal_xact}; +pub use decode::{codecs, datum, decoder_sink, fpi, heap_decoder, visibility, wal_xact}; #[doc(hidden)] pub use emit::{ch_ddl, ch_emitter, pipeline}; #[doc(hidden)] diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index e814c2b9..9e87e284 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -6,7 +6,6 @@ use std::sync::atomic::{AtomicU64, Ordering}; use clickhouse_c::{Allocator, Block, BlockOpts, BlockReader, Column, Kind, SliceIo, TypeRef}; use crate::ch::types; -use crate::decode::heap_decoder::ColumnValue; use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES, request_frame}; use tokio_postgres::types::Oid; @@ -562,38 +561,19 @@ fn decode_block( Ok(block) } -/// Render PostGIS 2-D points as WKT, typoutput yields HEXEWKB +/// GSERIALIZED 2-D point, as [`point_wkt`] reads it #[cfg(test)] -pub fn render_ext_columns(attrs: &[crate::schema::RelAttr], columns: &mut [Option]) { - // Scan cells first: attribute type names sit behind a pointer per column - for (idx, cell) in columns.iter_mut().enumerate() { - let Some(ColumnValue::PgPending { .. }) = cell.as_ref() else { - continue; - }; - let is_geo = attrs.iter().any(|att| { - !att.dropped - && usize::try_from(att.attnum - 1) == Ok(idx) - && matches!(att.type_name.as_str(), "geography" | "geometry") - }); - if is_geo && let Some(v) = cell.as_mut() { - render_ext_value(v); - } - } -} - -/// Render one PostGIS cell as WKT, typoutput yields HEXEWKB -pub(crate) fn render_ext_value(cell: &mut ColumnValue) { - if let ColumnValue::PgPending { raw, .. } = cell - && let Some(text) = gserialized_point_to_wkt(raw) - { - *cell = ColumnValue::Text(text); - } +pub(crate) fn point_body(x: f64, y: f64) -> Vec { + let mut raw = vec![0u8, 0, 0, 0, 1, 0, 0, 0]; + raw.extend_from_slice(&x.to_le_bytes()); + raw.extend_from_slice(&y.to_le_bytes()); + raw } /// PostGIS on-disk GSERIALIZED → `POINT(x y)` for 2-D points. Layout: /// `[srid(3) + gflags(1)][geomtype u32-le][…]`; POINT has geomtype 1 and its /// two `f64`-LE coordinates are the trailing 16 bytes. `None` for non-points. -fn gserialized_point_to_wkt(raw: &[u8]) -> Option { +pub(crate) fn point_wkt(raw: &[u8]) -> Option { if raw.len() < 16 { return None; } @@ -741,13 +721,10 @@ mod tests { let mut raw = vec![0u8, 0, 0, 0, 1, 0, 0, 0]; raw.extend_from_slice(&30.5_f64.to_le_bytes()); raw.extend_from_slice(&81.25_f64.to_le_bytes()); - assert_eq!( - gserialized_point_to_wkt(&raw).as_deref(), - Some("POINT(30.5 81.25)") - ); + assert_eq!(point_wkt(&raw).as_deref(), Some("POINT(30.5 81.25)")); // non-point geomtype → None raw[4] = 2; - assert_eq!(gserialized_point_to_wkt(&raw), None); + assert_eq!(point_wkt(&raw), None); } #[test] @@ -880,34 +857,14 @@ mod tests { type_storage: 'm', missing_default: None, }; - let mut point = vec![0u8, 0, 0, 0, 1, 0, 0, 0]; - point.extend_from_slice(&1.5_f64.to_le_bytes()); - point.extend_from_slice(&(-2.0_f64).to_le_bytes()); - let pending = |raw: &[u8]| { - Some(ColumnValue::PgPending { - type_oid: 0, - raw: raw.to_vec(), - }) - }; - let attrs = [ - attr(1, "geometry", false), - attr(2, "geography", false), - attr(3, "bytea", false), - attr(4, "geometry", true), - ]; - let mut columns = vec![ - pending(&point), - pending(&point[..8]), - pending(&point), - pending(&point), - None, - ]; - render_ext_columns(&attrs, &mut columns); - assert!(matches!(&columns[0], Some(ColumnValue::Text(t)) if t == "POINT(1.5 -2)")); - for (i, c) in columns.iter().enumerate().take(4).skip(1) { - assert!(matches!(c, Some(ColumnValue::PgPending { .. })), "{i}"); + use crate::emit::ch_emitter::Codec; + for (type_name, wkt) in [("geometry", true), ("geography", true), ("bytea", false)] { + let codec = Codec::choose(&attr(1, type_name, false)); + assert_eq!(matches!(codec, Codec::Wkt), wkt, "{type_name}"); } - assert!(columns[4].is_none()); + let point = point_body(1.5, -2.0); + assert_eq!(point_wkt(&point).as_deref(), Some("POINT(1.5 -2)")); + assert_eq!(point_wkt(&point[..8]), None); } /// Answer that fails to decode counts as an oracle error diff --git a/src/runtime_config.rs b/src/runtime_config.rs index 5e8a5649..a6ba2d51 100644 --- a/src/runtime_config.rs +++ b/src/runtime_config.rs @@ -19,7 +19,9 @@ //! typed rows and [`ConfigOverlay::apply`] just replaces the entry. Values are //! validated late, at resolver merge time, not here. -use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, HeapOp}; +use crate::decode::datum::Datum; +use crate::decode::heap_decoder::{DecodedHeap, HeapOp}; +use crate::schema::RelAttr; use crate::schema::{RelDescriptor, RelName}; use crate::table_rules::MatchKind; use foldhash::HashMap; @@ -249,7 +251,7 @@ impl ConfigOverlay { /// the old image, for the row key (`REPLICA IDENTITY FULL` keeps it complete). /// `None` for ops with no usable image (TRUNCATE, or an UPDATE lacking a new /// image). -fn full_image(decoded: &DecodedHeap) -> Option>> { +fn full_image(decoded: &DecodedHeap) -> Option>> { match decoded.op { HeapOp::Insert => decoded.new.as_ref().map(|t| t.columns.clone()), HeapOp::Update | HeapOp::HotUpdate => { @@ -271,50 +273,60 @@ fn full_image(decoded: &DecodedHeap) -> Option>> { } } -/// `Some(&value)` when the named column is present in the image (including an -/// explicit SQL NULL as `ColumnValue::Null`); `None` when absent. +/// Attribute and datum when the named column is present in the image +/// (including an explicit SQL NULL); `None` when absent fn column<'a>( - rel: &RelDescriptor, - cols: &'a [Option], + rel: &'a RelDescriptor, + cols: &'a [Option], name: &str, -) -> Option<&'a ColumnValue> { +) -> Option<(&'a RelAttr, &'a Datum)> { let att = rel .attributes .iter() .find(|a| a.name == name && !a.dropped)?; - cols.get((att.attnum - 1).max(0) as usize)?.as_ref() + Some((att, cols.get((att.attnum - 1).max(0) as usize)?.as_ref()?)) } -fn field_i64(rel: &RelDescriptor, cols: &[Option], name: &str) -> Option { +fn field_i64(rel: &RelDescriptor, cols: &[Option], name: &str) -> Option { + use crate::schema::{INT2OID, INT4OID, INT8OID}; match column(rel, cols, name)? { - ColumnValue::Int8(v) => Some(*v), - ColumnValue::Int4(v) => Some(*v as i64), - ColumnValue::Int2(v) => Some(*v as i64), + (_, Datum::Text(t)) => std::str::from_utf8(t).ok()?.parse().ok(), + (att, Datum::Fixed(b)) => match att.type_oid { + INT8OID => Some(i64::from_le_bytes(b.as_slice().try_into().ok()?)), + INT4OID => Some(i32::from_le_bytes(b.as_slice().try_into().ok()?).into()), + INT2OID => Some(i16::from_le_bytes(b.as_slice().try_into().ok()?).into()), + _ => None, + }, _ => None, } } -fn field_bool(rel: &RelDescriptor, cols: &[Option], name: &str) -> Option { +fn field_bool(rel: &RelDescriptor, cols: &[Option], name: &str) -> Option { match column(rel, cols, name)? { - ColumnValue::Bool(v) => Some(*v), + (att, Datum::Fixed(b)) if att.type_oid == crate::schema::BOOLOID => Some(*b.first()? != 0), + (_, Datum::Text(t)) => Some(matches!(&t[..], b"t" | b"true" | b"yes" | b"on" | b"1")), _ => None, } } -fn field_string(rel: &RelDescriptor, cols: &[Option], name: &str) -> Option { - match column(rel, cols, name)? { - ColumnValue::Text(v) | ColumnValue::Name(v) | ColumnValue::Json(v) => Some(v.clone()), - _ => None, - } +fn field_string(rel: &RelDescriptor, cols: &[Option], name: &str) -> Option { + let bytes = match column(rel, cols, name)? { + (att, Datum::Fixed(b)) if att.type_oid == crate::schema::NAMEOID => { + &b[..b.iter().position(|&c| c == 0).unwrap_or(b.len())] + } + (_, Datum::Varlena(b) | Datum::Text(b)) => b.as_slice(), + _ => return None, + }; + String::from_utf8(bytes.to_vec()).ok() } fn field_string_array( rel: &RelDescriptor, - cols: &[Option], + cols: &[Option], name: &str, ) -> Option> { match column(rel, cols, name)? { - ColumnValue::PgPending { type_oid, raw } if *type_oid == crate::schema::TEXTARRAYOID => { + (att, Datum::Varlena(raw)) if att.type_oid == crate::schema::TEXTARRAYOID => { crate::decode::codecs::decode_text_array(raw) } _ => None, @@ -426,7 +438,8 @@ pub fn interpret( #[cfg(test)] mod tests { use super::*; - use crate::decode::heap_decoder::{ColumnValue, DecodedHeap, DecodedTuple, HeapOp}; + use crate::decode::datum::{Datum, build}; + use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple, HeapOp}; use crate::schema::{RelAttr, RelDescriptor, ReplIdent}; use walrus::pg::walparser::RelFileNode; @@ -466,17 +479,14 @@ mod tests { } } - fn text_array(values: &[&str]) -> ColumnValue { - ColumnValue::PgPending { - type_oid: crate::schema::TEXTARRAYOID, - raw: crate::decode::codecs::text_array_body(values), - } + fn text_array(values: &[&str]) -> Datum { + build::bytes(crate::decode::codecs::text_array_body(values)) } fn heap( op: HeapOp, - new: Option>>, - old: Option>>, + new: Option>>, + old: Option>>, ) -> DecodedHeap { DecodedHeap { rfn: RelFileNode { @@ -518,13 +528,13 @@ mod tests { let d = heap( HeapOp::Insert, Some(vec![ - Some(ColumnValue::Int2(1)), - Some(ColumnValue::Int8(1000)), - Some(ColumnValue::Null), // byte_budget NULL → daemon default - Some(ColumnValue::Int8(250)), - Some(ColumnValue::Text("zstd".into())), - Some(ColumnValue::Int4(9)), - Some(ColumnValue::Text("drop".into())), + Some(build::int2(1)), + Some(build::int8(1000)), + Some(Datum::Null), // byte_budget NULL → daemon default + Some(build::int8(250)), + Some(build::text("zstd")), + Some(build::int4(9)), + Some(build::text("drop")), ]), None, ); @@ -547,23 +557,15 @@ mod tests { /// image is whole for these tables, so this exercises it with synthetic gaps. #[test] fn global_update_backfills_absent_columns_from_old() { - let new = vec![ - None, - None, - Some(ColumnValue::Int8(2048)), - None, - None, - None, - None, - ]; + let new = vec![None, None, Some(build::int8(2048)), None, None, None, None]; let old = vec![ - Some(ColumnValue::Int2(1)), - Some(ColumnValue::Int8(1000)), - Some(ColumnValue::Int8(999)), - Some(ColumnValue::Int8(250)), - Some(ColumnValue::Text("lz4".into())), - Some(ColumnValue::Int4(5)), - Some(ColumnValue::Text("retain".into())), + Some(build::int2(1)), + Some(build::int8(1000)), + Some(build::int8(999)), + Some(build::int8(250)), + Some(build::text("lz4")), + Some(build::int4(5)), + Some(build::text("retain")), ]; // byte_budget changed in new image, rest filled from old assert_eq!( @@ -587,13 +589,13 @@ mod tests { #[test] fn global_delete_clears() { let old = vec![ - Some(ColumnValue::Int2(1)), - Some(ColumnValue::Int8(1)), - Some(ColumnValue::Int8(1)), - Some(ColumnValue::Int8(1)), - Some(ColumnValue::Text("lz4".into())), - Some(ColumnValue::Int4(1)), - Some(ColumnValue::Text("retain".into())), + Some(build::int2(1)), + Some(build::int8(1)), + Some(build::int8(1)), + Some(build::int8(1)), + Some(build::text("lz4")), + Some(build::int4(1)), + Some(build::text("retain")), ]; assert_eq!( interpret( @@ -617,10 +619,10 @@ mod tests { ], ); let old = vec![ - Some(ColumnValue::Text("public".into())), - Some(ColumnValue::Null), - Some(ColumnValue::Bool(true)), - Some(ColumnValue::Null), + Some(build::text("public")), + Some(Datum::Null), + Some(build::bool_(true)), + Some(Datum::Null), ]; assert_eq!( interpret( @@ -650,12 +652,12 @@ mod tests { ], ); let new = vec![ - Some(ColumnValue::Text("public".into())), - Some(ColumnValue::Text("events".into())), - Some(ColumnValue::Text("default".into())), - Some(ColumnValue::Text("events".into())), - Some(ColumnValue::Bool(true)), - Some(ColumnValue::Text("copy".into())), + Some(build::text("public")), + Some(build::text("events")), + Some(build::text("default")), + Some(build::text("events")), + Some(build::bool_(true)), + Some(build::text("copy")), Some(text_array(&["tenant", "id"])), Some(text_array(&["tenant"])), ]; @@ -693,9 +695,9 @@ mod tests { ], ); let new = vec![ - Some(ColumnValue::Text("public".into())), - Some(ColumnValue::Text("events".into())), - Some(ColumnValue::Text("events".into())), + Some(build::text("public")), + Some(build::text("events")), + Some(build::text("events")), ]; assert_eq!( interpret( @@ -727,12 +729,12 @@ mod tests { ], ); let new = vec![ - Some(ColumnValue::Text("app".into())), - Some(ColumnValue::Text("events_.*".into())), - Some(ColumnValue::Text("regex".into())), - Some(ColumnValue::Bool(true)), - Some(ColumnValue::Text("_peerdb_version".into())), - Some(ColumnValue::Text(String::new())), + Some(build::text("app")), + Some(build::text("events_.*")), + Some(build::text("regex")), + Some(build::bool_(true)), + Some(build::text("_peerdb_version")), + Some(build::text(String::new())), ]; // Absent system columns (xid, commit_ts) inherit assert_eq!( @@ -781,11 +783,11 @@ mod tests { ], ); let new = vec![ - Some(ColumnValue::Text("app".into())), - Some(ColumnValue::Text("m".into())), - Some(ColumnValue::Text("self".into())), - Some(ColumnValue::Text("toYYYYMM(ts)".into())), - Some(ColumnValue::Text("_part".into())), + Some(build::text("app")), + Some(build::text("m")), + Some(build::text("self")), + Some(build::text("toYYYYMM(ts)")), + Some(build::text("_part")), ]; assert_eq!( interpret( @@ -810,10 +812,7 @@ mod tests { CONFIG_NAMESPACE, vec![attr(1, "namespace", 25), attr(2, "detach_strategy", 25)], ); - let new = vec![ - Some(ColumnValue::Text("app".into())), - Some(ColumnValue::Text("drop".into())), - ]; + let new = vec![Some(build::text("app")), Some(build::text("drop"))]; assert_eq!( interpret( ConfigTableKind::Namespace, @@ -846,14 +845,14 @@ mod tests { ], ); let new = vec![ - Some(ColumnValue::Text("app".into())), - Some(ColumnValue::Text("*".into())), - Some(ColumnValue::Text("*_amount".into())), - Some(ColumnValue::Text("glob".into())), - Some(ColumnValue::Text("Decimal(38, 9)".into())), - Some(ColumnValue::Text("0".into())), - Some(ColumnValue::Text("max".into())), - Some(ColumnValue::Null), + Some(build::text("app")), + Some(build::text("*")), + Some(build::text("*_amount")), + Some(build::text("glob")), + Some(build::text("Decimal(38, 9)")), + Some(build::text("0")), + Some(build::text("max")), + Some(Datum::Null), ]; // NULL neg_inf inherits assert_eq!( @@ -891,7 +890,7 @@ mod tests { #[test] fn global_smallint_budget_widens() { let r = rel(CONFIG_GLOBAL, vec![attr(1, "row_budget", 21)]); - let d = heap(HeapOp::Insert, Some(vec![Some(ColumnValue::Int2(5))]), None); + let d = heap(HeapOp::Insert, Some(vec![Some(build::int2(5))]), None); assert_eq!( interpret(ConfigTableKind::Global, &d, &r), Some(ConfigEvent::GlobalUpserted(GlobalRow { diff --git a/src/schema.rs b/src/schema.rs index defa867b..145f5947 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -211,7 +211,7 @@ pub struct RelAttr { pub missing_default: Option, } -/// Fast default (`attmissingval`), decoded by `missing_value_for`. Shadow +/// Fast default (`attmissingval`), read by `missing_datum`. Shadow /// catalog reads yield `Raw` on-disk bytes; `Text` (element's `anyarray_out` /// form) comes from source-PG SQL, which has no way to the bytes, oracle /// rendered DDL defaults, and descriptor logs older than raw shadow reads diff --git a/src/toast/read_ahead.rs b/src/toast/read_ahead.rs index a1c14def..7b156e18 100644 --- a/src/toast/read_ahead.rs +++ b/src/toast/read_ahead.rs @@ -10,7 +10,8 @@ use tokio::sync::Semaphore; use tokio_util::task::AbortOnDropHandle; use crate::budget::{MemoryBudget, MemoryPermit}; -use crate::decode::heap_decoder::{ColumnValue, ToastPointer}; +use crate::decode::datum::Datum; +use crate::decode::heap_decoder::ToastPointer; use crate::toast::{pointer_extsize, pointer_footprint}; pub const AHEAD_READS: usize = crate::ops::bridge::MAX_BRIDGE_WORKERS; @@ -21,7 +22,7 @@ const ENTRY_OVERHEAD: usize = 512; type Key = (u32, u32); pub struct ReadValue { - pub value: ColumnValue, + pub value: Datum, pub permit: MemoryPermit, } @@ -45,7 +46,6 @@ impl Pending { struct Entry { db: u32, p: ToastPointer, - type_oid: u32, read: Pending, } @@ -87,7 +87,6 @@ impl PendingReads { &mut self, db: u32, p: &ToastPointer, - type_oid: u32, lsn: u64, read: impl FnOnce(MemoryPermit) -> F, ) where @@ -122,7 +121,6 @@ impl PendingReads { Entry { db, p: *p, - type_oid, read: Pending { lsn, started, @@ -132,10 +130,10 @@ impl PendingReads { ); } - pub fn take(&mut self, db: u32, p: &ToastPointer, type_oid: u32, lsn: u64) -> Option { + pub fn take(&mut self, db: u32, p: &ToastPointer, lsn: u64) -> Option { let key = (p.va_toastrelid, p.va_valueid); let e = self.entries.get(&key)?; - if (e.db, e.p, e.type_oid) != (db, *p, type_oid) || e.read.lsn > lsn { + if (e.db, e.p) != (db, *p) || e.read.lsn > lsn { return None; } Some(self.entries.remove(&key)?.read) @@ -163,7 +161,7 @@ impl crate::budget::Reclaim for std::sync::Mutex { #[cfg(test)] mod tests { use super::*; - use crate::schema::{JSONBOID, TEXTOID}; + use crate::decode::datum::build; fn pointer(id: u32, size: u32) -> ToastPointer { ToastPointer { @@ -175,7 +173,7 @@ mod tests { } fn ready(value: &str) -> impl FnOnce(MemoryPermit) -> std::future::Ready> { - let value = ColumnValue::Text(value.into()); + let value = build::text(value); |permit| std::future::ready(Some(ReadValue { value, permit })) } @@ -190,19 +188,13 @@ mod tests { let p = pointer(1, 4104); let mut pending = PendingReads::new(MemoryBudget::new(128 << 10)); let budget = pending.budget().clone(); - pending.reserve(1, &p, JSONBOID, 100, ready("v")); + pending.reserve(1, &p, 100, ready("v")); settle().await; - assert!(pending.take(2, &p, JSONBOID, 100).is_none()); - assert!(pending.take(1, &p, TEXTOID, 100).is_none()); - assert!(pending.take(1, &p, JSONBOID, 99).is_none()); - let value = pending - .take(1, &p, JSONBOID, 100) - .unwrap() - .take() - .await - .unwrap(); - assert_eq!(value.value, ColumnValue::Text("v".into())); - assert!(pending.take(1, &p, JSONBOID, 100).is_none()); + assert!(pending.take(2, &p, 100).is_none()); + assert!(pending.take(1, &p, 99).is_none()); + let value = pending.take(1, &p, 100).unwrap().take().await.unwrap(); + assert_eq!(value.value, build::text("v")); + assert!(pending.take(1, &p, 100).is_none()); assert!(budget.resident_bytes() > 0); drop(value); assert_eq!(budget.resident_bytes(), 0); @@ -213,26 +205,26 @@ mod tests { let mut pending = PendingReads::new(MemoryBudget::new(1024)); let budget = pending.budget().clone(); let p = pointer(1, 1); - pending.reserve(1, &p, TEXTOID, 100, |permit| async move { + pending.reserve(1, &p, 100, |permit| async move { let _permit = permit; std::future::pending().await }); settle().await; - pending.reserve(1, &pointer(2, 1), TEXTOID, 100, ready("v")); - assert!(pending.take(1, &pointer(2, 1), TEXTOID, 100).is_none()); + pending.reserve(1, &pointer(2, 1), 100, ready("v")); + assert!(pending.take(1, &pointer(2, 1), 100).is_none()); pending.forget((p.va_toastrelid, p.va_valueid)); settle().await; assert_eq!(budget.resident_bytes(), 0); - pending.reserve(1, &pointer(2, 1), TEXTOID, 100, ready("v")); - assert!(pending.take(1, &pointer(2, 1), TEXTOID, 100).is_some()); + pending.reserve(1, &pointer(2, 1), 100, ready("v")); + assert!(pending.take(1, &pointer(2, 1), 100).is_some()); } #[tokio::test(flavor = "current_thread")] async fn planning_does_not_wait_for_queued_reads() { let mut pending = PendingReads::new(MemoryBudget::new(4096)); let p = pointer(1, 1); - pending.reserve(1, &p, TEXTOID, 100, ready("v")); - let slot = pending.take(1, &p, TEXTOID, 100).unwrap(); + pending.reserve(1, &p, 100, ready("v")); + let slot = pending.take(1, &p, 100).unwrap(); assert!(slot.take().await.is_none()); } } diff --git a/src/toast/resolver.rs b/src/toast/resolver.rs index 0263a8c9..78813087 100644 --- a/src/toast/resolver.rs +++ b/src/toast/resolver.rs @@ -19,8 +19,9 @@ use bytes::Bytes; use crate::ch::{ ChConn, EmitterError, drain_to_end_of_stream, is_retryable, quote_ident, with_timeout, }; +use crate::decode::datum::Datum; use crate::decode::heap_decoder::{ - ColumnValue, ToastPointer, VARLENA_EXTSIZE_BITS, VARLENA_EXTSIZE_MASK, decompress_varlena, + ToastPointer, VARLENA_EXTSIZE_BITS, VARLENA_EXTSIZE_MASK, decompress_varlena, }; use crate::emit::ch_emitter::{EmitterConfig, EmitterStats, InlineValueOverflow, ToastMode}; use crate::toast::read_ahead::{PendingReads, ReadValue, retained_bytes}; @@ -123,43 +124,20 @@ pub(crate) fn check_value_caps( Ok(retained + transient) } -/// Convert raw TOAST bytes through inline varlena decoder -pub(crate) fn detoasted_value(raw: Vec, type_oid: u32) -> ColumnValue { - crate::decode::heap_decoder::varlena_to_value(type_oid, std::borrow::Cow::Owned(raw)) -} - -/// [`detoasted_value`] and bytes it holds, `None` unless `fits` admits decode -/// peak. JSONB and numeric text may outgrow raw, so measure before rendering +/// Detoasted body as the datum it stands for and bytes it holds, `None` +/// unless `fits` admits them pub(crate) fn detoasted_within( raw: Vec, - type_oid: u32, fits: impl FnOnce(usize) -> bool, -) -> Option<(ColumnValue, usize)> { - use crate::decode::codecs::{decode_jsonb_with_capacity, jsonb_text_len, numeric_text_len}; - use crate::schema::{JSONBOID, NUMERICOID}; - let text_len = match type_oid { - JSONBOID => jsonb_text_len(&raw).ok(), - NUMERICOID => numeric_text_len(&raw).ok(), - _ => None, - }; - if !fits(raw.len().max(text_len.unwrap_or(0))) { - return None; - } - let value = if type_oid == JSONBOID - && let Some(len) = text_len - && let Ok(text) = decode_jsonb_with_capacity(&raw, len) - { - ColumnValue::Json(text) - } else { - detoasted_value(raw, type_oid) - }; - let held = value.approx_bytes(); - Some((value, held)) +) -> Option<(Datum, usize)> { + let held = raw.len(); + fits(held).then_some((Datum::Varlena(raw), held)) } /// [`detoasted_within`] for leaf holders, whose retained bytes settle after -pub(crate) fn detoasted_sized(raw: Vec, type_oid: u32) -> (ColumnValue, usize) { - detoasted_within(raw, type_oid, |_| true).expect("admits every size") +pub(crate) fn detoasted_sized(raw: Vec) -> (Datum, usize) { + let held = raw.len(); + (Datum::Varlena(raw), held) } /// Convert stored bytes to raw bytes using pointer compression method @@ -1376,7 +1354,7 @@ impl ToastResolver { Some(self.ahead.as_ref()?.lock().expect("pending reads")) } - pub fn read_ahead(&self, db_oid: u32, p: ToastPointer, type_oid: u32, lsn: u64) { + pub fn read_ahead(&self, db_oid: u32, p: ToastPointer, lsn: u64) { if self.value_oversize(&p) { return; } @@ -1389,14 +1367,13 @@ impl ToastResolver { ahead: None, ..self.clone() }; - pending.reserve(db_oid, &p, type_oid, lsn, |mut permit| async move { + pending.reserve(db_oid, &p, lsn, |mut permit| async move { this.stats .toast_values_read_ahead .fetch_add(1, Ordering::Relaxed); let raw = this.read_raw(db_oid, p, lsn).await?; - // Rendering may expand past fetch transients - let fits = |rendered: usize| { - let need = retained_bytes(rendered, &p); + let fits = |held: usize| { + let need = retained_bytes(held, &p); if need > permit.bytes() { let Some(more) = budget.try_speculate((need - permit.bytes()) as usize) else { return false; @@ -1405,7 +1382,7 @@ impl ToastResolver { } true }; - let (value, held) = detoasted_within(raw, type_oid, fits)?; + let (value, held) = detoasted_within(raw, fits)?; permit.shrink(retained_bytes(held, &p)); Some(ReadValue { value, permit }) }); @@ -1434,13 +1411,10 @@ impl ToastResolver { &self, db_oid: u32, p: &ToastPointer, - type_oid: u32, lsn: u64, permit: &mut Option, - ) -> Option { - let slot = self - .pending() - .and_then(|mut c| c.take(db_oid, p, type_oid, lsn))?; + ) -> Option { + let slot = self.pending().and_then(|mut c| c.take(db_oid, p, lsn))?; let started = std::time::Instant::now(); let value = slot.take().await; self.stats @@ -1916,6 +1890,7 @@ mod tests { } use super::*; + use crate::decode::datum::build; use crate::emit::ch_emitter::InstanceConfig; use std::time::Duration; @@ -2817,7 +2792,6 @@ mod tests { va_valueid: 7, va_toastrelid: 16500, }, - crate::schema::TEXTOID, 0x2000, ); // Store read misses within same poll on current_thread runtime @@ -3062,7 +3036,6 @@ mod tests { va_valueid: id, va_toastrelid: 16500, }, - crate::schema::TEXTOID, 0x100, ); tokio::task::yield_now().await; @@ -3117,7 +3090,7 @@ mod tests { .with_budget(budget.clone()) .with_read_ahead(); for id in 1..=2 { - resolver.read_ahead(0, pointer(id), crate::schema::TEXTOID, 1); + resolver.read_ahead(0, pointer(id), 1); } tokio::time::timeout(std::time::Duration::from_secs(1), async { while stats.toast_values_read_ahead.load(Ordering::Relaxed) != 2 @@ -3130,13 +3103,9 @@ mod tests { .unwrap(); assert!(budget.resident_bytes() > 0); let next = pointer(3); - resolver.read_ahead(0, next, crate::schema::TEXTOID, 2); + resolver.read_ahead(0, next, 2); assert!( - resolver - .pending() - .unwrap() - .take(0, &next, crate::schema::TEXTOID, 2) - .is_none(), + resolver.pending().unwrap().take(0, &next, 2).is_none(), "unconsumed slots exhaust admission" ); let leaf = budget.acquire(2048).await; @@ -3150,7 +3119,7 @@ mod tests { drop(leaf); assert_eq!(budget.resident_bytes(), 0); assert!(budget.peak_bytes() <= budget.total() as u64); - resolver.read_ahead(0, next, crate::schema::TEXTOID, 2); + resolver.read_ahead(0, next, 2); tokio::time::timeout(std::time::Duration::from_secs(1), async { while stats.toast_values_read_ahead.load(Ordering::Relaxed) != 3 { tokio::task::yield_now().await; @@ -3158,20 +3127,13 @@ mod tests { }) .await .expect("read ahead resumes after foreground reclaim"); - assert!( - resolver - .pending() - .unwrap() - .take(0, &next, crate::schema::TEXTOID, 2) - .is_some() - ); + assert!(resolver.pending().unwrap().take(0, &next, 2).is_some()); } } - /// Escaped JSON renders to six bytes per control byte, past fetch transients, - /// into exactly the capacity admitted ahead of rendering + /// Values read ahead hold their stored bytes, whatever their type renders to #[tokio::test(flavor = "current_thread")] - async fn rendered_growth_tops_up_or_discards_read() { + async fn read_ahead_holds_stored_bytes() { struct Store(Vec); #[async_trait] impl ChunkStore for Store { @@ -3194,30 +3156,23 @@ mod tests { va_valueid: 1, va_toastrelid: 16500, }; - for (total, fits) in [(1 << 20, true), (64 << 10, false)] { - let budget = crate::budget::MemoryBudget::with_leaf_reserve(total, 32 << 10); - let r = ToastResolver::with_store( - Arc::new(Store(body.clone())), - Arc::new(EmitterStats::default()), - ) - .with_budget(budget.clone()) - .with_read_ahead(); - r.read_ahead(0, p, crate::schema::JSONBOID, 1); - for _ in 0..8 { - tokio::task::yield_now().await; - } - let slot = r.pending().unwrap().take(0, &p, crate::schema::JSONBOID, 1); - let read = slot.unwrap().take().await; - assert_eq!(read.is_some(), fits); - if let Some(ReadValue { value, permit }) = read { - let ColumnValue::Json(s) = value else { - panic!("jsonb renders as Json, got {value:?}"); - }; - assert_eq!((s.len(), s.capacity()), (24578, 24578)); - assert!(permit.bytes() >= 2 * 24578); - } - assert_eq!(budget.resident_bytes(), 0); + let budget = crate::budget::MemoryBudget::with_leaf_reserve(1 << 20, 32 << 10); + let r = ToastResolver::with_store( + Arc::new(Store(body.clone())), + Arc::new(EmitterStats::default()), + ) + .with_budget(budget.clone()) + .with_read_ahead(); + r.read_ahead(0, p, 1); + for _ in 0..8 { + tokio::task::yield_now().await; } + let slot = r.pending().unwrap().take(0, &p, 1); + let ReadValue { value, permit } = slot.unwrap().take().await.expect("read fits"); + assert_eq!(value, build::bytes(&body)); + assert!(permit.bytes() >= body.len() as u64); + drop(permit); + assert_eq!(budget.resident_bytes(), 0); } #[tokio::test(flavor = "current_thread")] @@ -3237,7 +3192,6 @@ mod tests { va_valueid: 1, va_toastrelid: 16500, }, - crate::schema::TEXTOID, 1, ); while stats.toast_values_read_ahead.load(Ordering::Relaxed) == 0 { diff --git a/src/xact/spill.rs b/src/xact/spill.rs index 1a5c3483..9ff6fc62 100644 --- a/src/xact/spill.rs +++ b/src/xact/spill.rs @@ -221,7 +221,9 @@ impl From for DescribedInput { fn from(h: crate::decode::heap_decoder::DescribedHeap) -> Self { use crate::decode::heap_decoder::*; let d = &h.decoded; - let image = |t: &Option| t.as_ref().map(|t| form_block_data(&t.columns)); + let attrs = &h.descriptor.attributes; + let image = + |t: &Option| t.as_ref().map(|t| form_block_data(attrs, &t.columns)); let (old, new) = (image(&d.old), image(&d.new)); let (info, mut main, flag_at, old_flag) = match d.op { HeapOp::Insert => (XLOG_HEAP_INSERT, vec![0; SIZE_OF_HEAP_INSERT], None, 0), @@ -1306,6 +1308,7 @@ fn decode_raw(c: &mut Cursor) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::decode::datum::build; use tempfile::tempdir; fn sample_descriptor(rel_node: u32) -> Arc { @@ -1352,7 +1355,7 @@ mod tests { source_lsn: lsn, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Bytea(vec![7; body]))], + columns: vec![Some(build::bytes(vec![7; body]))], partial: false, }), old: None, diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index 6d63f86c..78a06fee 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -51,14 +51,13 @@ use crate::catalog::desc_log::{ Ambiguity, DescriptorLog, DescriptorLogs, LogEntry, LogValue, LookupMiss, }; use crate::catalog::pending::{PendingCatalog, PendingSlot}; -use crate::decode::datum::DatumRef; +use crate::decode::datum::{Datum, DatumRef}; use crate::decode::decoder_sink::{DecoderSinkError, DecoderStats, SkipReason}; -use crate::decode::heap_decoder::{ - ColumnValue, HeapImages, HeapOp, ToastPointer, TupleView, decode_heap_record, record_shape, - view_heap_record, -}; #[cfg(any(test, feature = "test-support"))] use crate::decode::heap_decoder::{DecodedHeap, DescribedHeap}; +use crate::decode::heap_decoder::{ + HeapImages, HeapOp, ToastPointer, TupleView, decode_heap_record, record_shape, view_heap_record, +}; #[cfg(test)] use crate::decode::wal_xact::{ XACT_XINFO_HAS_DBINFO, XACT_XINFO_HAS_GID, XACT_XINFO_HAS_INVALS, XACT_XINFO_HAS_SUBXACTS, @@ -2471,22 +2470,13 @@ pub async fn detoast_heap( let mut sites = Vec::new(); for (image, t) in images.iter().enumerate() { for (idx, c) in t.iter().flat_map(|t| t.columns.iter().enumerate()) { - if let Some(ColumnValue::ExternalToast(p)) = *c { + if let Some(Datum::External(p)) = *c { sites.push(ToastSite { image, idx, p }); } } } let (lsn, db) = (heap.decoded.source_lsn, heap.decoded.rfn.db_node); - let (values, permit) = detoast_sites( - &sites, - &heap.descriptor, - db, - lsn, - spool, - chunk_maps, - resolver, - ) - .await?; + let (values, permit) = detoast_sites(&sites, db, lsn, spool, chunk_maps, resolver).await?; for (site, value) in sites.iter().zip(values) { if let Some(t) = images[site.image].as_mut() { t.columns[site.idx] = Some(value); @@ -2502,9 +2492,6 @@ pub async fn detoast_heap( /// without budget or external values pub async fn detoast_sites( sites: &[ToastSite], - // Attached at decode: same descriptor interpretation from decode to - // detoast regardless of captures landing in between - rel: &RelDescriptor, db: u32, lsn: u64, // Xact body spool backing `File` refs; None while memory-resident @@ -2514,17 +2501,12 @@ pub async fn detoast_sites( // parallel path) chunk_maps: &[&ChunkRefMap], resolver: &ToastResolver, -) -> std::result::Result<(Vec, Option), XactBufferError> { - let type_oid = |site: &ToastSite| rel.attributes.get(site.idx).map_or(0, |a| a.type_oid); +) -> std::result::Result<(Vec, Option), XactBufferError> { // Transfer speculative values before admitting foreground resolution let mut ahead = None; - let mut values: Vec> = Vec::with_capacity(sites.len()); + let mut values: Vec> = Vec::with_capacity(sites.len()); for site in sites { - values.push( - resolver - .take_ahead(db, &site.p, type_oid(site), lsn, &mut ahead) - .await, - ); + values.push(resolver.take_ahead(db, &site.p, lsn, &mut ahead).await); } let mut pointers: Vec = sites .iter() @@ -2564,7 +2546,7 @@ pub async fn detoast_sites( .zip(values) .map(|(site, v)| match v { Some(v) => Ok(v), - None => res.resolve(&site.p, type_oid(site)), + None => res.resolve(&site.p), }) .collect::, _>>()?; if let Some(p) = leaf.as_mut() { @@ -2713,7 +2695,7 @@ pub(crate) fn fill_store_miss( p: &ToastPointer, resolver: &ToastResolver, policy: MissPolicy, -) -> std::result::Result { +) -> std::result::Result { match (miss, policy) { (StoreMiss::NoStore, _) => resolver.note_filled_default(), (StoreMiss::Generation, _) => resolver.note_filled_generation(), @@ -2733,7 +2715,7 @@ pub(crate) fn fill_store_miss( )); } } - Ok(ColumnValue::Null) + Ok(Datum::Null) } /// Per-heap value resolution over prefetched store values, decoded bytes @@ -2750,13 +2732,9 @@ struct ValueResolution<'a> { } impl ValueResolution<'_> { - fn resolve( - &mut self, - p: &ToastPointer, - type_oid: u32, - ) -> std::result::Result { + fn resolve(&mut self, p: &ToastPointer) -> std::result::Result { if self.resolver.fills_oversize(p) { - return Ok(ColumnValue::Null); + return Ok(Datum::Null); } let key = (p.va_toastrelid, p.va_valueid); let Some(v) = self @@ -2765,11 +2743,11 @@ impl ValueResolution<'_> { .find_map(|m| m.get(&key)) .filter(|v| !observed_mid_value(v)) else { - return self.resolve_store(p, type_oid); + return self.resolve_store(p); }; match reassemble_value_ref(p, self.spool, v)? { Reassembled::Bytes(raw) => { - let (value, held) = detoasted_sized(raw, type_oid); + let (value, held) = detoasted_sized(raw); self.retained += held; Ok(value) } @@ -2778,7 +2756,7 @@ impl ValueResolution<'_> { // never an error. Reassembled::Missing if self.resolver.fill_on_miss() => { self.resolver.note_filled_default(); - Ok(ColumnValue::Null) + Ok(Datum::Null) } // In-xact chunks gapped or short: decode bug, surfaced loud outcome => { @@ -2801,11 +2779,7 @@ impl ValueResolution<'_> { /// Pre-window / bootstrap value whose chunks aren't in this xact, /// assembled by [`prefetch_store_values`] against the pointer's /// stored size - fn resolve_store( - &mut self, - p: &ToastPointer, - type_oid: u32, - ) -> std::result::Result { + fn resolve_store(&mut self, p: &ToastPointer) -> std::result::Result { let key = (p.va_toastrelid, p.va_valueid); let cached = if self.resolver.fill_on_miss() { Err(StoreMiss::NoStore) @@ -2821,7 +2795,7 @@ impl ValueResolution<'_> { }; match cached { Ok(raw) => { - let (value, held) = detoasted_sized(raw, type_oid); + let (value, held) = detoasted_sized(raw); self.retained += held; Ok(value) } @@ -2964,7 +2938,7 @@ fn read_ahead_kept( continue; } for t in [&d.new, &d.old].into_iter().flatten() { - for (datum, attr) in t.datums.iter().zip(&rel.attributes) { + for datum in &t.datums { if let &DatumRef::External(p) = datum { let recent = born .lock() @@ -2972,7 +2946,7 @@ fn read_ahead_kept( .counts .contains_key(&(p.va_toastrelid, p.va_valueid)); if !recent { - ahead.read_ahead(rel.rfn.db_node, p, attr.type_oid, source_lsn); + ahead.read_ahead(rel.rfn.db_node, p, source_lsn); } } } @@ -3859,6 +3833,7 @@ mod tests { use super::raw_fixtures::*; use super::*; + use crate::decode::datum::{Datum, build}; use crate::decode::heap_decoder::{DecodedTuple, HeapOp, VARLENA_EXTSIZE_BITS}; use crate::emit::ch_emitter::InlineValueOverflow; use tempfile::tempdir; @@ -3920,7 +3895,7 @@ mod tests { source_lsn: lsn, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![Some(ColumnValue::Bytea(vec![0u8; payload_size]))], + columns: vec![Some(build::bytes(vec![0u8; payload_size]))], partial: false, }), old: None, @@ -4401,11 +4376,14 @@ mod tests { ], }; use crate::decode::heap_decoder::{form_block_data, view_block_data}; - let full = form_block_data(&[ - Some(ColumnValue::Oid(55)), - Some(ColumnValue::Int4(2)), - Some(ColumnValue::Bytea(b"hello".to_vec())), - ]); + let full = form_block_data( + &rel.attributes, + &[ + Some(build::oid(55)), + Some(build::int4(2)), + Some(build::bytes(b"hello")), + ], + ); let view = view_block_data(&full, &rel).unwrap(); let chunk = toast_chunk(&view, &rel, 0x1234, Some((7, 3))).expect("recognised toast shape"); assert_eq!(chunk.toast_relid, 99); // pg_class.oid, not rel_node @@ -4413,94 +4391,30 @@ mod tests { assert_eq!(chunk.chunk_seq, 2); assert_eq!(&chunk.chunk_data[..], b"hello"); assert_eq!((chunk.blkno, chunk.offnum), (7, 3)); - let short = form_block_data(&[Some(ColumnValue::Oid(55)), Some(ColumnValue::Int4(2))]); + let short = form_block_data( + &rel.attributes, + &[Some(build::oid(55)), Some(build::int4(2))], + ); let view = view_block_data(&short, &rel).unwrap(); assert!(toast_chunk(&view, &rel, 0, None).is_none()); } + /// Detoasted bodies stay stored bytes, whatever their type, and hold + /// exactly their length #[test] - fn detoasted_sized_routes_tier3_like_inline() { - use crate::schema::{BYTEAOID, TEXTOID}; - /// Fringe varlena type with no local codec - const TSVECTOROID: u32 = 3614; - assert!( - matches!(detoasted_sized(b"raw".to_vec(), BYTEAOID), (ColumnValue::Bytea(b), 3) if b == b"raw") - ); - assert!( - matches!(detoasted_sized(b"hi".to_vec(), TEXTOID), (ColumnValue::Text(s), 2) if s == "hi") - ); - // Tier 3 lands as PgPending carrying the body so the oracle resolves - // it like an inline value, not Unsupported - assert_eq!( - detoasted_sized(b"\x01body".to_vec(), TSVECTOROID), - ( - ColumnValue::PgPending { - type_oid: TSVECTOROID, - raw: b"\x01body".to_vec(), - }, - 5 - ) - ); - } - - /// Numeric text outgrows its base-10000 digits, so admission sees text size - #[test] - fn detoasted_numeric_admits_its_text() { - use crate::schema::NUMERICOID; - // 10^12: long form, weight 3, one digit group - let raw = vec![0, 0, 3, 0, 1, 0]; - let text = "1000000000000"; + fn detoasted_values_stay_stored() { + assert_eq!(detoasted_sized(b"raw".to_vec()), (build::bytes(b"raw"), 3)); + let jsonb = 0x2000_0000u32.to_le_bytes().to_vec(); assert_eq!( - crate::toast::detoasted_within(raw.clone(), NUMERICOID, |peak| { - assert_eq!(peak, text.len()); + crate::toast::detoasted_within(jsonb.clone(), |held| { + assert_eq!(held, 4); false }), None ); assert_eq!( - detoasted_sized(raw, NUMERICOID), - ( - ColumnValue::Numeric(crate::decode::codecs::NumericKind::Finite(text.into())), - text.len() - ) - ); - } - - /// A detoasted jsonb renders through its codec, same as an inline one, - /// holding exactly its text - #[test] - fn detoasted_jsonb_renders_its_document() { - use crate::schema::JSONBOID; - let empty = 0x2000_0000u32.to_le_bytes().to_vec(); - assert_eq!( - detoasted_sized(empty, JSONBOID), - (ColumnValue::Json("{}".into()), 2) - ); - let mut escaped = 0x5000_0001u32.to_le_bytes().to_vec(); - escaped.extend_from_slice(&4u32.to_le_bytes()); - escaped.extend_from_slice(&[1; 4]); - let text = r#""\u0001\u0001\u0001\u0001""#; - assert_eq!( - detoasted_sized(escaped.clone(), JSONBOID), - (ColumnValue::Json(text.into()), text.len()) - ); - assert_eq!( - crate::toast::detoasted_within(escaped, JSONBOID, |peak| { - assert_eq!(peak, text.len(), "text outgrows its 12 raw bytes"); - false - }), - None - ); - let malformed = vec![0u8; 3]; - assert_eq!( - detoasted_sized(malformed.clone(), JSONBOID), - ( - ColumnValue::Unsupported { - type_oid: JSONBOID, - raw: malformed - }, - 3 - ) + crate::toast::detoasted_within(jsonb.clone(), |_| true), + Some((build::bytes(&jsonb), 4)) ); } @@ -4548,7 +4462,7 @@ mod tests { fn toast_ptr_tuple(value_id: u32) -> DecodedTuple { DecodedTuple { - columns: vec![Some(ColumnValue::ExternalToast(toast_ptr(value_id)))], + columns: vec![Some(Datum::External(toast_ptr(value_id)))], partial: false, } } @@ -4576,7 +4490,7 @@ mod tests { .with_budget(budget.clone()) .with_read_ahead(); let p = toast_ptr(55); - resolver.read_ahead(5, p, crate::schema::BYTEAOID, 0x100); + resolver.read_ahead(5, p, 0x100); tokio::time::timeout(std::time::Duration::from_secs(1), async { while stats.toast_values_fetched.load(Ordering::Relaxed) == 0 { tokio::task::yield_now().await; @@ -4595,7 +4509,7 @@ mod tests { let (heap, permit) = resolve.await.unwrap(); assert_eq!( heap.decoded.new.as_ref().unwrap().columns[0], - Some(ColumnValue::Bytea(b"abcd".to_vec())) + Some(build::bytes(b"abcd")) ); assert!(budget.resident_bytes() >= 1024 + 4); assert_eq!(budget.waits_total(), 0); @@ -4641,16 +4555,16 @@ mod tests { .with_inline_value_max(1), ]; for r in &declining { - r.read_ahead(5, p, crate::schema::BYTEAOID, 0x100); + r.read_ahead(5, p, 0x100); } assert_eq!(stats.toast_values_read_ahead.load(Relaxed), 0); let r = stored().with_budget(budget).with_read_ahead(); - r.read_ahead(5, p, crate::schema::BYTEAOID, 0x100); + r.read_ahead(5, p, 0x100); while stats.toast_values_read_ahead.load(Relaxed) == 0 { tokio::task::yield_now().await; } let mut permit = None; - let miss = r.take_ahead(5, &p, crate::schema::BYTEAOID, 0x100, &mut permit); + let miss = r.take_ahead(5, &p, 0x100, &mut permit); assert_eq!(miss.await, None); } @@ -4739,7 +4653,7 @@ mod tests { .await .expect("fills instead of failing"); for t in [&heap.decoded.new, &heap.decoded.old] { - assert_eq!(t.as_ref().unwrap().columns[0], Some(ColumnValue::Null)); + assert_eq!(t.as_ref().unwrap().columns[0], Some(Datum::Null)); } assert_eq!(stats.toast_values_filled_oversize.load(Relaxed), 2); assert_eq!(stats.toast_values_filled_default.load(Relaxed), 0); @@ -4787,12 +4701,10 @@ mod tests { fn resolve_tuple( &mut self, t: &mut crate::decode::heap_decoder::DecodedTuple, - rel: &RelDescriptor, ) -> std::result::Result<(), XactBufferError> { - for (idx, col) in t.columns.iter_mut().enumerate() { - if let Some(ColumnValue::ExternalToast(p)) = *col { - let type_oid = rel.attributes.get(idx).map_or(0, |a| a.type_oid); - *col = Some(self.resolve(&p, type_oid)?); + for col in t.columns.iter_mut() { + if let Some(Datum::External(p)) = *col { + *col = Some(self.resolve(&p)?); } } Ok(()) @@ -4844,7 +4756,7 @@ mod tests { let mut errs = Vec::new(); for miss in misses { match fill_store_miss(miss, &p, &resolver, policy) { - Ok(v) => assert_eq!(v, ColumnValue::Null), + Ok(v) => assert_eq!(v, Datum::Null), Err(detail) => errs.push(detail), } } @@ -4872,7 +4784,6 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn value_observed_mid_write_resolves_from_store() { - let rel = bytea_rel(); let stats = Arc::new(crate::emit::ch_emitter::EmitterStats::default()); let resolver = ToastResolver::with_store(Arc::new(crate::toast::MemChunkStore::new()), stats.clone()); @@ -4883,15 +4794,15 @@ mod tests { let mut t = toast_ptr_tuple(55); let cache = HashMap::from_iter([(key, Ok(b"abcd".to_vec()))]); seeded(&resolver, None, &maps, cache) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Bytea(b"abcd".to_vec()))); + assert_eq!(t.columns[0], Some(build::bytes(b"abcd"))); let gapped_tail = mem_refs(key, &[(1, b"c"), (3, b"d")]); let maps = [&gapped_tail]; let mut t = toast_ptr_tuple(55); let err = seeded(&resolver, None, &maps, HashMap::new()) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .expect_err("a gap after the first observed chunk stays loud"); assert!( matches!(err, XactBufferError::MissingToastChunk { .. }), @@ -4919,7 +4830,6 @@ mod tests { /// disabled mode NULL-fills + counts default. #[tokio::test(flavor = "current_thread")] async fn resolve_tuple_splits_in_xact_gap_from_store_miss() { - let rel = bytea_rel(); let tmp = tempdir().unwrap(); let stats = Arc::new(crate::emit::ch_emitter::EmitterStats::default()); let store_resolver = @@ -4931,8 +4841,8 @@ mod tests { let mut t = toast_ptr_tuple(55); let cache = HashMap::from_iter([(key, Err(StoreMiss::Missing))]); let mut r = seeded(&store_resolver, None, &[], cache); - r.resolve_tuple(&mut t, &rel).unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Null)); + r.resolve_tuple(&mut t).unwrap(); + assert_eq!(t.columns[0], Some(Datum::Null)); assert_eq!(r.retained, 0, "fills retain nothing"); assert_eq!( stats @@ -4946,7 +4856,7 @@ mod tests { let maps = [&gapped]; let mut t = toast_ptr_tuple(55); let err = seeded(&store_resolver, None, &maps, HashMap::new()) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .expect_err("in-xact gap surfaces"); assert!(matches!( err, @@ -4964,8 +4874,8 @@ mod tests { let maps = [&whole]; let mut t = toast_ptr_tuple(55); let mut r = seeded(&store_resolver, None, &maps, HashMap::new()); - r.resolve_tuple(&mut t, &rel).unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Bytea(b"abcd".to_vec()))); + r.resolve_tuple(&mut t).unwrap(); + assert_eq!(t.columns[0], Some(build::bytes(b"abcd"))); assert_eq!(r.retained, 4, "decoded bytes tally for the permit shrink"); // In-xact file refs resolve through the spool @@ -4978,9 +4888,9 @@ mod tests { &maps, HashMap::new(), ) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Bytea(b"abcd".to_vec()))); + assert_eq!(t.columns[0], Some(build::bytes(b"abcd"))); // Out-of-order file refs land in the tail, read one by one let (w, tail) = file_refs(tmp.path(), 0x20, key, &[(1, b"cd"), (0, b"ab")]); @@ -4993,16 +4903,16 @@ mod tests { &maps, HashMap::new(), ) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Bytea(b"abcd".to_vec()))); + assert_eq!(t.columns[0], Some(build::bytes(b"abcd"))); // Store-resolved hit lands assembled bytes let mut t = toast_ptr_tuple(55); let cache = HashMap::from_iter([(key, Ok(b"abcd".to_vec()))]); let mut r = seeded(&store_resolver, None, &[], cache); - r.resolve_tuple(&mut t, &rel).unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Bytea(b"abcd".to_vec()))); + r.resolve_tuple(&mut t).unwrap(); + assert_eq!(t.columns[0], Some(build::bytes(b"abcd"))); assert_eq!(r.retained, 4); assert!(r.cache.is_empty(), "last use moves the buffer out"); @@ -5011,9 +4921,9 @@ mod tests { let mut t = toast_ptr_tuple(55); let cache = HashMap::from_iter([(key, Err(StoreMiss::Mismatch { got: 3 }))]); seeded(&store_resolver, None, &[], cache) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Null)); + assert_eq!(t.columns[0], Some(Datum::Null)); assert_eq!( stats .toast_values_filled_mismatch @@ -5034,7 +4944,7 @@ mod tests { let maps = [&xact_short]; let mut t = toast_ptr_tuple(55); let err = seeded(&store_resolver, None, &maps, HashMap::new()) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .expect_err("in-xact size mismatch surfaces"); assert!(matches!(err, XactBufferError::Detoast(_))); @@ -5044,15 +4954,15 @@ mod tests { let disabled = ToastResolver::disabled().with_stats(disabled_stats.clone()); let mut t = toast_ptr_tuple(55); seeded(&disabled, None, &[], HashMap::new()) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Null)); + assert_eq!(t.columns[0], Some(Datum::Null)); let maps = [&gapped]; let mut t = toast_ptr_tuple(55); seeded(&disabled, None, &maps, HashMap::new()) - .resolve_tuple(&mut t, &rel) + .resolve_tuple(&mut t) .unwrap(); - assert_eq!(t.columns[0], Some(ColumnValue::Null)); + assert_eq!(t.columns[0], Some(Datum::Null)); let filled = &disabled_stats.toast_values_filled_default; assert_eq!(filled.load(std::sync::atomic::Ordering::Relaxed), 2); } @@ -5096,11 +5006,11 @@ mod tests { cache, retained: 0, }; - let first = r.resolve_store(&p, 17).unwrap(); + let first = r.resolve_store(&p).unwrap(); assert!(!r.cache.is_empty(), "pending duplicate use stays cached"); - let second = r.resolve_store(&p, 17).unwrap(); - assert_eq!(first, ColumnValue::Bytea(b"abcd".to_vec())); - assert_eq!(second, ColumnValue::Bytea(b"abcd".to_vec())); + let second = r.resolve_store(&p).unwrap(); + assert_eq!(first, build::bytes(b"abcd")); + assert_eq!(second, build::bytes(b"abcd")); assert!(r.cache.is_empty(), "last use moves the buffer out"); assert_eq!(r.retained, 8, "both uses retain their copy"); assert_eq!( @@ -5411,7 +5321,7 @@ mod tests { assert_eq!(h.decoded.xid, 8, "writer xid survives subxact merge"); assert_eq!(h.descriptor_valid_from, 0x50); let new = h.decoded.new.as_ref().unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(*expected))); + assert_eq!(new.columns[0], Some(build::int4(*expected))); } assert_eq!( batch.cells().unwrap()[3].decoded.source_lsn, @@ -6000,7 +5910,7 @@ mod tests { ); let cells = batch.cells().unwrap(); let new = cells[0].decoded.new.as_ref().unwrap(); - assert_eq!(new.columns[0], Some(ColumnValue::Int4(7))); + assert_eq!(new.columns[0], Some(build::int4(7))); drain.finish().await.unwrap(); } @@ -6116,7 +6026,7 @@ mod tests { .as_ref() .unwrap() .columns[0], - Some(ColumnValue::Int4(9)), + Some(build::int4(9)), ); drain.finish().await.unwrap(); } @@ -6973,7 +6883,7 @@ mod tests { .iter() .map(|h| h.decoded.new.as_ref().unwrap().columns[0].clone()) .collect(); - let int4 = |v| Some(ColumnValue::Int4(v)); + let int4 = |v| Some(build::int4(v)); assert_eq!( values, [int4(1), int4(10), int4(2), int4(20), int4(3), int4(30)] diff --git a/tests/backfill_gap_across_promotion.rs b/tests/backfill_gap_across_promotion.rs index 116cb241..718f4d52 100644 --- a/tests/backfill_gap_across_promotion.rs +++ b/tests/backfill_gap_across_promotion.rs @@ -26,7 +26,8 @@ use walrus::storage::fs::FsStorage; use walshadow::archive; use walshadow::backfill_bootstrap::seed_catalog_from_source; use walshadow::backup_backfill::fetch_segments; -use walshadow::heap_decoder::{ColumnValue, decode_heap_record}; +use walshadow::datum::Datum; +use walshadow::heap_decoder::decode_heap_record; use walshadow::record::{ Record, RecordSink, SinkError, WAL_SEG_SIZE, segments_covering, segments_covering_lineage, }; @@ -140,12 +141,13 @@ impl RecordSink for BranchTally { .expect("decode replayed heap") { let tuple = heap.new.expect("insert tuple"); - let [Some(ColumnValue::Int4(id)), Some(ColumnValue::Text(name))] = - tuple.columns.as_slice() + let [Some(Datum::Fixed(id)), Some(Datum::Varlena(name))] = tuple.columns.as_slice() else { panic!("unexpected tuple: {tuple:?}"); }; - self.rows.push((*id, name.clone())); + let id = i32::from_le_bytes(id.as_slice().try_into().expect("int4")); + self.rows + .push((id, String::from_utf8(name.clone()).expect("utf8"))); } } Box::pin(std::future::ready(Ok(()))) diff --git a/tests/bootstrap_direct_e2e.rs b/tests/bootstrap_direct_e2e.rs index dc3e5f8f..a6cb7091 100644 --- a/tests/bootstrap_direct_e2e.rs +++ b/tests/bootstrap_direct_e2e.rs @@ -37,7 +37,7 @@ use walrus::pg::replication::conn::PgConfig; use walrus::pg::replication::tls::{SslMode, TlsParams}; use walshadow::backfill_bootstrap::{BootstrapConfig, run_greenfield_bootstrap, seed_in_snapshot}; use walshadow::backup_source_direct::DirectSource; -use walshadow::heap_decoder::ColumnValue; +use walshadow::datum::Datum; use walshadow::shadow::{Shadow, ShadowConfig}; /// Same row budget as the object-store sibling: one heap page worth so @@ -259,7 +259,8 @@ async fn direct_source_self_hosted_via_replication_protocol() { let rel = catalog_map .get(t.rfn.db_node, t.rfn.rel_node) .expect("descriptor"); - if let Some(Some(ColumnValue::Int4(v))) = t.cells(&rel).expect("cells").first() { + if let Some(Some(Datum::Fixed(v))) = t.cells(&rel).expect("cells").first() { + let v = &i32::from_le_bytes(v.as_slice().try_into().expect("int4")); int_ids.insert(*v); } } diff --git a/tests/bootstrap_object_store_e2e.rs b/tests/bootstrap_object_store_e2e.rs index ed1394cf..96f08d8e 100644 --- a/tests/bootstrap_object_store_e2e.rs +++ b/tests/bootstrap_object_store_e2e.rs @@ -66,7 +66,7 @@ use walrus::storage::DynStorage; use walrus::storage::fs::FsStorage; use walshadow::backfill_bootstrap::{BootstrapConfig, run_greenfield_bootstrap, seed_in_snapshot}; use walshadow::backup_source_object_store::ObjectStoreSource; -use walshadow::heap_decoder::ColumnValue; +use walshadow::datum::Datum; use walshadow::shadow::{Shadow, ShadowConfig}; /// Row count loaded into the user table before push runs. Must be >0 @@ -360,7 +360,8 @@ async fn object_store_source_self_hosted_via_wal_rs_push() { let rel = catalog_map .get(t.rfn.db_node, t.rfn.rel_node) .expect("descriptor"); - if let Some(Some(ColumnValue::Int4(v))) = t.cells(&rel).expect("cells").first() { + if let Some(Some(Datum::Fixed(v))) = t.cells(&rel).expect("cells").first() { + let v = &i32::from_le_bytes(v.as_slice().try_into().expect("int4")); int_ids.insert(*v); } } diff --git a/tests/bootstrap_pipeline_ch.rs b/tests/bootstrap_pipeline_ch.rs index 07acd619..35252d5f 100644 --- a/tests/bootstrap_pipeline_ch.rs +++ b/tests/bootstrap_pipeline_ch.rs @@ -27,8 +27,8 @@ use walrus::pg::walparser::RelFileNode; use walshadow::backup_page_walk::{BackfillTuple, CatalogMap, TupleData}; use walshadow::ch::CompressionChoice; use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; +use walshadow::datum::build; use walshadow::emit::route::freeze_routes; -use walshadow::heap_decoder::ColumnValue; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::batcher::BatcherMsg; use walshadow::pipeline::{Fatal, bootstrap, tail}; @@ -100,7 +100,7 @@ fn tuple(rel_node: u32, id: i32) -> BackfillTuple { page_lsn: 0, blkno: 0, offnum: 0, - data: TupleData::from_cells(&[Some(ColumnValue::Int4(id))]), + data: TupleData::from_cells(&[Some(build::int4(id))]), pending: None, } } diff --git a/tests/bridge.rs b/tests/bridge.rs index f3f03b4a..bca7a552 100644 --- a/tests/bridge.rs +++ b/tests/bridge.rs @@ -22,6 +22,7 @@ use walshadow::bridge::{ AttributeRow, Bridge, BridgeError, Catalog, ClassRow, IndexRow, NamespaceRow, PROJECTION_VERSION, PROTO_VERSION, }; +use walshadow::datum::{DatumRef, build}; use walshadow::oracle::{Oracle, OracleCell, OracleColumnBuf, OracleRequestColumn}; use walshadow::pg::socket_conninfo; use walshadow::schema::{NUMERICOID, ReplIdent}; @@ -510,8 +511,8 @@ async fn bridge_scans_uncommitted_ddl() { missing_default: Some(walshadow::schema::MissingDefault::Raw(bytes)), }; assert_eq!( - walshadow::heap_decoder::missing_value_for(&c_attr), - walshadow::heap_decoder::ColumnValue::Int4(7) + walshadow::heap_decoder::resolve(&c_attr, DatumRef::Missing), + Some(build::int4(7)) ); assert_eq!(added.atttypid, 23); assert!(attrs[0].attmissingval.is_none()); @@ -912,8 +913,8 @@ async fn bridge_overlay_descriptors_track_open_ddl() { let cols: Vec<&str> = t.attributes.iter().map(|a| a.name.as_str()).collect(); assert_eq!(cols, ["id", "a", "c"]); assert_eq!( - walshadow::heap_decoder::missing_value_for(&t.attributes[2]), - walshadow::heap_decoder::ColumnValue::Int4(7) + walshadow::heap_decoder::resolve(&t.attributes[2], DatumRef::Missing), + Some(build::int4(7)) ); assert_eq!(t.attributes[2].type_name, "int4"); // Unchanged by the ALTER, so still the committed values diff --git a/tests/copy_chunk_resume_e2e.rs b/tests/copy_chunk_resume_e2e.rs index c794b6f1..8e7989e0 100644 --- a/tests/copy_chunk_resume_e2e.rs +++ b/tests/copy_chunk_resume_e2e.rs @@ -21,7 +21,7 @@ use walrus::pg::walparser::RelFileNode; use walshadow::backup_page_walk::BackfillTuple; use walshadow::ch_emitter::EmitterStats; use walshadow::copy_backfill::{BlockRange, copy_rows_into}; -use walshadow::decode::heap_decoder::ColumnValue; +use walshadow::datum::Datum; use walshadow::schema::{INT8OID, RelAttr, RelDescriptor, RelName, ReplIdent, TEXTOID}; use walshadow::source_feed::open_sql_client; @@ -60,7 +60,9 @@ async fn ids( while let Some(slab) = rx.recv().await { for t in slab { match t.cells(&rel).unwrap().first() { - Some(Some(ColumnValue::Int8(v))) => out.push(*v), + Some(Some(Datum::Fixed(v))) => { + out.push(i64::from_le_bytes(v.as_slice().try_into().expect("int8"))) + } other => panic!("unexpected id column {other:?}"), } } diff --git a/tests/emitter_budget_flush.rs b/tests/emitter_budget_flush.rs index 88512981..8caaf5a9 100644 --- a/tests/emitter_budget_flush.rs +++ b/tests/emitter_budget_flush.rs @@ -24,7 +24,8 @@ use std::sync::atomic::Ordering; use walrus::pg::walparser::RelFileNode; use walshadow::ch::CompressionChoice; use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; -use walshadow::heap_decoder::{ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; +use walshadow::datum::build; +use walshadow::heap_decoder::{CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::ack::Publish; use walshadow::pipeline::batcher::{BatcherMsg, PlacedSegment}; @@ -113,8 +114,8 @@ fn tuple(id: i32, source_lsn: u64, commit_lsn: u64) -> CommittedTuple { op: HeapOp::Insert, new: Some(DecodedTuple { columns: vec![ - Some(ColumnValue::Int4(id)), - Some(ColumnValue::Text(format!("row-{id}"))), + Some(build::int4(id)), + Some(build::text(format!("row-{id}"))), ], partial: false, }), diff --git a/tests/emitter_native_types.rs b/tests/emitter_native_types.rs index 13ccba98..44689ed7 100644 --- a/tests/emitter_native_types.rs +++ b/tests/emitter_native_types.rs @@ -20,14 +20,16 @@ use std::sync::Arc; use walrus::pg::walparser::RelFileNode; use walshadow::ch::CompressionChoice; use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; -use walshadow::codecs::NumericKind; -use walshadow::heap_decoder::{ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; +use walshadow::datum::build; +use walshadow::heap_decoder::{CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::ack::Publish; use walshadow::pipeline::batcher::{BatcherMsg, PlacedSegment}; use walshadow::pipeline::{Fatal, tail}; use walshadow::pos::{EmitterAck, Monotone}; -use walshadow::schema::{RelDescriptor, RelName, ReplIdent}; +use walshadow::schema::{ + INT4OID, NUMERICOID, RelAttr, RelDescriptor, RelName, ReplIdent, TIMEOID, TIMETZOID, +}; const RFN: RelFileNode = RelFileNode { spc_node: 1663, @@ -38,9 +40,29 @@ const RFN: RelFileNode = RelFileNode { // 12:34:56 = 45296 s = 45_296_000_000 µs since midnight. const MICROS: i64 = 45_296_000_000; +/// Source attribute of `type_oid` stored `type_len` wide; encoding follows it +fn attr(attnum: i16, name: &str, type_oid: u32, type_len: i16) -> RelAttr { + RelAttr { + attnum, + name: name.into(), + type_oid, + typmod: -1, + not_null: false, + dropped: false, + type_name: String::new(), + type_byval: type_len > 0 && type_len <= 8, + type_len, + type_align: if type_len == 8 || type_len == 12 { + 'd' + } else { + 'i' + }, + type_storage: if type_len < 0 { 'x' } else { 'p' }, + missing_default: None, + } +} + fn rel_descriptor() -> RelDescriptor { - // Encoding reads the mapping's target types, not these attributes, - // so a minimal descriptor (qualified_name + rfn) suffices. RelDescriptor { rfn: RFN, oid: 16385, @@ -51,7 +73,13 @@ fn rel_descriptor() -> RelDescriptor { persistence: 'p', ancestors: Vec::new(), replident: ReplIdent::Default { pk_attnums: None }, - attributes: vec![], + attributes: vec![ + attr(1, "id", INT4OID, 4), + attr(2, "n", NUMERICOID, -1), + attr(3, "nw", NUMERICOID, -1), + attr(4, "t", TIMEOID, 8), + attr(5, "tz", TIMETZOID, 12), + ], } } @@ -151,17 +179,14 @@ async fn native_numeric_time_timetz_round_trip() { op: HeapOp::Insert, new: Some(DecodedTuple { columns: vec![ - Some(ColumnValue::Int4(1)), - Some(ColumnValue::Numeric(NumericKind::Finite("1.50".into()))), - Some(ColumnValue::Numeric(NumericKind::Finite( - "123456789012345678901234567890123456789012345678.12".into(), - ))), - Some(ColumnValue::Time(MICROS)), + Some(build::int4(1)), + Some(build::numeric("1.50")), + Some(build::numeric( + "123456789012345678901234567890123456789012345678.12", + )), + Some(build::micros(MICROS)), // UTC+2 → PG stores tz_seconds west-positive = -7200. - Some(ColumnValue::TimeTz { - micros: MICROS, - tz_seconds: -7200, - }), + Some(build::timetz(MICROS, -7200)), ], partial: false, }), diff --git a/tests/emitter_tls.rs b/tests/emitter_tls.rs index 2a6066f6..08b7ffd7 100644 --- a/tests/emitter_tls.rs +++ b/tests/emitter_tls.rs @@ -37,13 +37,14 @@ use clickhouse_c::tls::{self, rustls}; use walrus::pg::walparser::RelFileNode; use walshadow::ch::CompressionChoice; use walshadow::ch_emitter::{EmitterConfig, EmitterStats, InstanceConfig}; -use walshadow::heap_decoder::{ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; +use walshadow::datum::build; +use walshadow::heap_decoder::{CommittedTuple, DecodedHeap, DecodedTuple, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::ack::Publish; use walshadow::pipeline::batcher::{BatcherMsg, PlacedSegment}; use walshadow::pipeline::{Fatal, tail}; use walshadow::pos::{EmitterAck, Monotone}; -use walshadow::schema::{RelDescriptor, RelName, ReplIdent}; +use walshadow::schema::{INT4OID, RelAttr, RelDescriptor, RelName, ReplIdent, TEXTOID}; const RFN: RelFileNode = RelFileNode { spc_node: 1663, @@ -269,6 +270,28 @@ impl Drop for TlsChServer { } } +/// Source attribute of `type_oid` stored `type_len` wide; encoding follows it +fn attr(attnum: i16, name: &str, type_oid: u32, type_len: i16) -> RelAttr { + RelAttr { + attnum, + name: name.into(), + type_oid, + typmod: -1, + not_null: false, + dropped: false, + type_name: String::new(), + type_byval: type_len > 0 && type_len <= 8, + type_len, + type_align: if type_len == 8 || type_len == 12 { + 'd' + } else { + 'i' + }, + type_storage: if type_len < 0 { 'x' } else { 'p' }, + missing_default: None, + } +} + fn rel_descriptor() -> RelDescriptor { RelDescriptor { rfn: RFN, @@ -280,7 +303,7 @@ fn rel_descriptor() -> RelDescriptor { persistence: 'p', ancestors: Vec::new(), replident: ReplIdent::Default { pk_attnums: None }, - attributes: vec![], + attributes: vec![attr(1, "id", INT4OID, 4), attr(2, "name", TEXTOID, -1)], } } @@ -363,10 +386,7 @@ async fn emitter_tls_round_trip() { source_lsn: 0x2000, op: HeapOp::Insert, new: Some(DecodedTuple { - columns: vec![ - Some(ColumnValue::Int4(1)), - Some(ColumnValue::Text("over-tls".into())), - ], + columns: vec![Some(build::int4(1)), Some(build::text("over-tls"))], partial: false, }), old: None, diff --git a/tests/xact_buffer.rs b/tests/xact_buffer.rs index 76e662f8..48f5873b 100644 --- a/tests/xact_buffer.rs +++ b/tests/xact_buffer.rs @@ -28,9 +28,10 @@ use std::time::Duration; use tokio::sync::Mutex; use walrus::pg::walparser::RelFileNode; use walshadow::ch_emitter::EmitterStats; +use walshadow::datum::{Datum, build}; use walshadow::desc_log::{BatchRecord, DescLogIdentity, DescriptorLog, LogEntry, LogValue}; use walshadow::heap_decoder::{ - ColumnValue, CommittedTuple, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, ToastPointer, + CommittedTuple, DecodedHeap, DecodedTuple, DescribedHeap, HeapOp, ToastPointer, }; use walshadow::pg::socket_conninfo; use walshadow::pos::Pos; @@ -140,13 +141,7 @@ fn rfn(spc: u32, db: u32, rel: u32) -> RelFileNode { } } -fn heap( - rfn: RelFileNode, - xid: u32, - lsn: u64, - op: HeapOp, - cols: Vec>, -) -> DecodedHeap { +fn heap(rfn: RelFileNode, xid: u32, lsn: u64, op: HeapOp, cols: Vec>) -> DecodedHeap { DecodedHeap { rfn, xid, @@ -301,12 +296,7 @@ async fn commit_drains_in_arrival_order_and_clears_state() { let cat = Arc::new(Mutex::new(cat)); let spill_dir = tmp.path().join("spill"); let mut b = XactBuffer::new(cfg(spill_dir, 1024)).unwrap(); - let one_col = |id: i32| { - vec![ - Some(ColumnValue::Int4(id)), - Some(ColumnValue::Text("x".into())), - ] - }; + let one_col = |id: i32| vec![Some(build::int4(id)), Some(build::text("x"))]; let log = log_from_catalog(&cat, tmp.path()).await; b.on_heap(described( &log, @@ -481,14 +471,8 @@ async fn commit_drains_spilled_then_in_memory_entries() { let cat = Arc::new(Mutex::new(cat)); let spill_dir = tmp.path().join("spill"); let mut b = XactBuffer::new(cfg(spill_dir, 1024)).unwrap(); - let fat_col = vec![ - Some(ColumnValue::Int4(0)), - Some(ColumnValue::Bytea(vec![0u8; 700])), - ]; - let small_col = vec![ - Some(ColumnValue::Int4(0)), - Some(ColumnValue::Text("z".into())), - ]; + let fat_col = vec![Some(build::int4(0)), Some(build::bytes(vec![0u8; 700]))]; + let small_col = vec![Some(build::int4(0)), Some(build::text("z"))]; let log = log_from_catalog(&cat, tmp.path()).await; // Three big tuples first — spill engages after the second. for i in 0..3 { @@ -547,12 +531,7 @@ async fn commit_merges_top_and_subxact_in_source_lsn_order() { let cat = Arc::new(Mutex::new(cat)); let spill_dir = tmp.path().join("spill"); let mut b = XactBuffer::new(cfg(spill_dir, 1024)).unwrap(); - let col = |id: i32| { - vec![ - Some(ColumnValue::Int4(id)), - Some(ColumnValue::Text("m".into())), - ] - }; + let col = |id: i32| vec![Some(build::int4(id)), Some(build::text("m"))]; let log = log_from_catalog(&cat, tmp.path()).await; b.on_heap(described(&log, heap(rfn, 7, 100, HeapOp::Insert, col(1)))) .await @@ -591,8 +570,8 @@ async fn detoast_concatenates_uncompressed_chunks_into_text() { let spill_dir = tmp.path().join("spill"); let mut b = XactBuffer::new(cfg(spill_dir, 1024)).unwrap(); // wc.things schema: (id int4, body text). Column 0 = id, column 1 = body. - let id_col = Some(ColumnValue::Int4(1)); - let body_ptr = Some(ColumnValue::ExternalToast(ToastPointer { + let id_col = Some(build::int4(1)); + let body_ptr = Some(Datum::External(ToastPointer { va_rawsize: (4 + 3 + 3) + 4, // ext_size + VARHDRSZ va_extinfo: 4 + 3 + 3, // ext_size, no compression bits va_valueid: 55, @@ -643,10 +622,7 @@ async fn detoast_concatenates_uncompressed_chunks_into_text() { .unwrap(); assert_eq!(seen.len(), 1); let body_col = &seen[0].decoded.new.as_ref().unwrap().columns[1]; - match body_col { - Some(ColumnValue::Text(s)) => assert_eq!(s, "Hello, wor"), - other => panic!("expected Text after detoast, got {other:?}"), - } + assert_eq!(*body_col, Some(build::text("Hello, wor")), "detoasted body"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -660,8 +636,8 @@ async fn detoast_missing_chunk_seq_errors_clearly() { let cat = Arc::new(Mutex::new(cat)); let spill_dir = tmp.path().join("spill"); let mut b = XactBuffer::new(cfg(spill_dir, 1024)).unwrap(); - let id_col = Some(ColumnValue::Int4(1)); - let body_ptr = Some(ColumnValue::ExternalToast(ToastPointer { + let id_col = Some(build::int4(1)); + let body_ptr = Some(Datum::External(ToastPointer { va_rawsize: 8, va_extinfo: 6, va_valueid: 1, @@ -737,10 +713,7 @@ async fn abort_drops_xact_and_unlinks_spill_against_real_shadow() { let log = log_from_catalog(&cat, tmp.path()).await; let spill_dir = tmp.path().join("spill"); let mut b = XactBuffer::new(cfg(spill_dir.clone(), 1024)).unwrap(); - let fat_col = vec![ - Some(ColumnValue::Int4(0)), - Some(ColumnValue::Bytea(vec![0u8; 256])), - ]; + let fat_col = vec![Some(build::int4(0)), Some(build::bytes(vec![0u8; 256]))]; for i in 0..10 { b.on_heap(described( &log, From 983bd42dd73404e0dbd7fe80687a4f7eafd3fa50 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:59:47 +0000 Subject: [PATCH 16/20] Pass binary COPY values to PostgreSQL for conversion when supported; fall back to text otherwise Avoid text parsing when reading column defaults from PostgreSQL Wait for shadow recovery before connecting COPY tests to conversion worker Clarify TOAST accounting comments to describe retained raw bytes --- architecture/values.md | 6 +- pgext/native.c | 56 +++++- pgext/walshadow.h | 3 +- src/backfill/copy_backfill.rs | 349 ++++++++++++++++++++++++++++------ src/backfill/spool.rs | 23 +++ src/catalog/shadow_catalog.rs | 1 + src/catalog/type_bridge.rs | 5 +- src/decode/datum.rs | 10 +- src/decode/heap_decoder.rs | 9 +- src/emit/ch_emitter.rs | 23 ++- src/emit/direct_parity.rs | 5 +- src/ops/bridge.rs | 2 +- src/ops/oracle.rs | 107 +++++++---- src/pg.rs | 28 ++- src/schema.rs | 9 +- src/toast/read_ahead.rs | 8 +- tests/oracle.rs | 112 +++++++++++ tests/runtime_config_e2e.rs | 94 +++++++-- tests/shadow_catalog.rs | 75 ++++++++ 19 files changed, 795 insertions(+), 130 deletions(-) diff --git a/architecture/values.md b/architecture/values.md index 60cf97a7..d8dba34e 100644 --- a/architecture/values.md +++ b/architecture/values.md @@ -38,8 +38,10 @@ unconditionally wiping history needed by earlier reads Values stay in forms PostgreSQL produced until they become ClickHouse Native column bytes: stored datums read in place from WAL, page images and TOAST chunks, or `typoutput` text from SQL-read fast defaults and COPY casts. COPY's -binary send forms convert to stored datums on arrival. Each destination column -compiles one conversion from its source type, so there is no decoded +binary send forms convert to stored datums on arrival where a local codec reads +the type; types only PostgreSQL converts keep their send bytes for its +`typreceive`, falling back to text casts without binary I/O. Each destination +column compiles one conversion from its source type, so there is no decoded intermediate value. Detoasted values hold their stored bytes, so resident accounting counts what was fetched rather than a rendering diff --git a/pgext/native.c b/pgext/native.c index 799b77e0..af47ea7f 100644 --- a/pgext/native.c +++ b/pgext/native.c @@ -39,6 +39,9 @@ typedef struct WsNativeCol { FmgrInfo infn; Oid ioparam; bool infn_ready; + FmgrInfo recvfn; + Oid recv_ioparam; + bool recvfn_ready; FmgrInfo expander; @@ -166,6 +169,43 @@ ws_input_datum(WsNativeCol* col, const char* body, uint32 len) { return val; } +/* COPY fields carry typsend bytes, typreceive must consume all of them */ +static Datum +ws_receive_datum(WsNativeCol* col, const char* body, uint32 len) { + StringInfoData buf; + Datum val; + + if (!col->recvfn_ready) { + Oid recvfunc; + + getTypeBinaryInputInfo(col->source_oid, &recvfunc, &col->recv_ioparam); + fmgr_info_cxt(recvfunc, &col->recvfn, CurrentMemoryContext); + col->recvfn_ready = true; + } + /* Receive functions expect a writable, NUL-terminated buffer */ + buf.data = (char*)palloc(len + 1); + memcpy(buf.data, body, len); + buf.data[len] = '\0'; + buf.len = (int)len; + buf.maxlen = (int)len + 1; + buf.cursor = 0; + val = + ReceiveFunctionCall(&col->recvfn, &buf, col->recv_ioparam, col->source_typmod); + if (buf.cursor != buf.len) { + ereport( + ERROR, + (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), + errmsg( + "incorrect binary data format for type %u: %d of %u bytes consumed", + col->source_oid, + buf.cursor, + len + )) + ); + } + return val; +} + /* Require one expander owned by source type's extension */ static bool ws_find_hstore_expander(Oid source_oid, Oid* out) { @@ -274,7 +314,8 @@ ws_append_cell(pgch_writer* w, size_t col, WsNativeCol* c, StringInfo req) { ws_append_default(w, col, c); return; } - if (tag != WS_CELL_DISK_RAW && tag != WS_CELL_TEXT && tag != WS_CELL_LITERAL) { + if (tag != WS_CELL_DISK_RAW && tag != WS_CELL_TEXT && tag != WS_CELL_LITERAL && + tag != WS_CELL_BINARY) { ereport( ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -306,8 +347,17 @@ ws_append_cell(pgch_writer* w, size_t col, WsNativeCol* c, StringInfo req) { ); return; } - val = tag == WS_CELL_DISK_RAW ? ws_reconstruct_datum(c, body, len) - : ws_input_datum(c, body, len); + switch (tag) { + case WS_CELL_DISK_RAW: + val = ws_reconstruct_datum(c, body, len); + break; + case WS_CELL_BINARY: + val = ws_receive_datum(c, body, len); + break; + default: + val = ws_input_datum(c, body, len); + break; + } val = ws_expand(c, val, &valtype); pgch_append_datum(w, col, val, valtype, false); } diff --git a/pgext/walshadow.h b/pgext/walshadow.h index bae035e6..b9d003c7 100644 --- a/pgext/walshadow.h +++ b/pgext/walshadow.h @@ -8,7 +8,7 @@ /* Bumped when a request or response layout changes, or when an op's reading * of an unchanged layout changes */ -#define WS_PROTO_VERSION 7 +#define WS_PROTO_VERSION 8 /* Bumped when any catalog projection changes shape */ #define WS_PROJECTION_VERSION 2 @@ -38,6 +38,7 @@ #define WS_CELL_DISK_RAW 0x01 #define WS_CELL_TEXT 0x02 #define WS_CELL_LITERAL 0x03 +#define WS_CELL_BINARY 0x04 /* Match daemon MAX_REQUEST_BYTES, exceed maximum inline TOAST value */ #define WS_MAX_REQUEST_BYTES (256 * 1024 * 1024) diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 2ae6a109..86bd2f06 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -37,9 +37,11 @@ //! the on-disk datum form the WAL heap decoder reads. Each row converts to //! on-disk form as it arrives: fixed-width fields byte-swap, `name` pads to //! `NAMEDATALEN`, `numeric_send` rebuilds the stored `numeric` body, and -//! byte/text strings pass unchanged. `jsonb` and every out-of-matrix type -//! select as `::text` and stay `typoutput` text, which the oracle converts -//! through `typinput` exactly as it does a WAL-path default +//! byte/text strings pass unchanged. Types only the oracle converts keep +//! their `typsend` bytes, which the oracle reads through `typreceive`. +//! `jsonb`, other types the local matrix renders, and types lacking binary +//! I/O select as `::text` and stay `typoutput` text, which the oracle +//! converts through `typinput` exactly as it does a WAL-path default //! (architecture/values.md). //! //! ## Resume ledger @@ -94,8 +96,9 @@ use crate::pos::{Pos, Snapshot}; use crate::runtime_config::InitialLoadMode; use crate::schema::{ BOOLOID, BPCHAROID, BYTEAOID, CHAROID, CIDOID, DATEOID, FLOAT4OID, FLOAT8OID, INT2OID, INT4OID, - INT8OID, JSONOID, MONEYOID, NAMEOID, NUMERICOID, OID8OID, OIDOID, PG_LSNOID, RelDescriptor, - RelName, TEXTOID, TIMEOID, TIMESTAMPOID, TIMESTAMPTZOID, UUIDOID, VARCHAROID, XID8OID, XIDOID, + INT8OID, JSONOID, MONEYOID, NAMEOID, NUMERICOID, OID8OID, OIDOID, PG_LSNOID, RelAttr, + RelDescriptor, RelName, TEXTOID, TIMEOID, TIMESTAMPOID, TIMESTAMPTZOID, UUIDOID, VARCHAROID, + XID8OID, XIDOID, }; use crate::source::source_feed::open_sql_client; use crate::toast::{ToastResolver, ToastRow}; @@ -402,9 +405,10 @@ impl Ledger { // --------------------------------------------------------------------------- /// How one selected column decodes from the wire. Native kinds read `typsend` -/// output; `Cast` columns were cast to `text` in the SELECT +/// output; `Cast` columns were cast to `text` in the SELECT; `Send` columns +/// keep `typsend` bytes for the oracle's `typreceive` #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum WireKind { +pub(crate) enum WireKind { Bool, Char, Int2, @@ -425,6 +429,7 @@ enum WireKind { Json, Numeric, Cast, + Send, } struct ColPlan { @@ -434,7 +439,8 @@ struct ColPlan { struct CopyPlan { select: String, cols: Vec, - natts: usize, + /// Indexed by attnum - 1 + kinds: Vec, } fn wire_kind(type_oid: u32) -> Option { @@ -462,26 +468,73 @@ fn wire_kind(type_oid: u32) -> Option { }) } -/// Wire form a column selects as; out-of-matrix types cast to `text`. -/// jsonb's send form is its text behind a version byte, so cast it too -fn copy_kind(type_oid: u32) -> WireKind { - wire_kind(type_oid).unwrap_or(WireKind::Cast) +/// Types no local reader converts, so their values always reach the oracle +fn oracle_only(att: &RelAttr) -> bool { + wire_kind(att.type_oid).is_none() + && !crate::decode::heap_decoder::local_matrix_covers(att.type_oid, att.type_len) } -fn column_plan(desc: &RelDescriptor) -> CopyPlan { +/// Oracle-only types of `desc` with binary I/O. Shadow replays source's +/// catalog, so one source `pg_type` read covers `typsend` and the oracle's +/// `typreceive` +async fn binary_io_types( + client: &tokio_postgres::Client, + desc: &RelDescriptor, +) -> anyhow::Result> { + let oids: Vec = desc + .attributes + .iter() + .filter(|a| !a.dropped && oracle_only(a)) + .map(|a| a.type_oid) + .collect(); + if oids.is_empty() { + return Ok(HashSet::new()); + } + let rows = client + .query( + "SELECT oid FROM pg_type WHERE oid = ANY($1) \ + AND typsend::oid <> 0 AND typreceive::oid <> 0", + &[&oids], + ) + .await + .context("backfill: binary I/O types")?; + Ok(rows.iter().map(|r| r.get(0)).collect()) +} + +/// Wire form each attribute selects as, attnum - 1 indexed. Oracle-only +/// types in `binary` keep `typsend` bytes, the rest cast to `text`. jsonb's +/// send form is its text behind a version byte, so cast it too +pub(crate) fn wire_kinds(desc: &RelDescriptor, binary: &HashSet) -> Vec { + let natts = desc + .attributes + .iter() + .map(|a| a.attnum.max(0) as usize) + .max() + .unwrap_or(0); + let mut kinds = vec![WireKind::Cast; natts]; + for a in desc.attributes.iter().filter(|a| a.attnum >= 1) { + kinds[a.attnum as usize - 1] = match wire_kind(a.type_oid) { + Some(kind) => kind, + None if oracle_only(a) && binary.contains(&a.type_oid) => WireKind::Send, + None => WireKind::Cast, + }; + } + kinds +} + +fn column_plan(desc: &RelDescriptor, binary: &HashSet) -> CopyPlan { + let kinds = wire_kinds(desc, binary); let mut select = String::new(); let mut cols = Vec::with_capacity(desc.attributes.len()); - let mut natts = 0usize; for a in &desc.attributes { - natts = natts.max(a.attnum.max(0) as usize); if a.dropped { continue; } let ident = quote_ident(&a.name); - let expr = if wire_kind(a.type_oid).is_some() { - ident - } else { + let expr = if kinds[(a.attnum - 1).max(0) as usize] == WireKind::Cast { format!("{ident}::text") + } else { + ident }; if !select.is_empty() { select.push_str(", "); @@ -492,7 +545,7 @@ fn column_plan(desc: &RelDescriptor) -> CopyPlan { CopyPlan { select, cols, - natts, + kinds, } } @@ -536,7 +589,7 @@ pub async fn copy_rows_into( stats: &EmitterStats, range: BlockRange, ) -> anyhow::Result { - let plan = column_plan(desc); + let plan = column_plan(desc, &binary_io_types(client, desc).await?); let sql = format!( "COPY (SELECT {} FROM ONLY {}.{}{}) TO STDOUT (FORMAT binary)", plan.select, @@ -553,14 +606,14 @@ pub async fn copy_rows_into( let mut slab_bytes = 0usize; while let Some(row) = stream.next().await { let row = row.context("backfill: COPY stream")?; - let mut fields: Vec> = vec![None; plan.natts]; + let mut fields: Vec> = vec![None; plan.kinds.len()]; for (i, cp) in plan.cols.iter().enumerate() { fields[(cp.attnum - 1).max(0) as usize] = row.try_get(i).context("backfill: COPY field")?; } let payload_bytes = fields.iter().flatten().map(|f| f.len() as u64).sum::(); - let data = - CopyRow::from_send(desc, &fields).map_err(|e| anyhow::anyhow!("backfill: {e}"))?; + let data = CopyRow::from_send(&plan.kinds, &fields) + .map_err(|e| anyhow::anyhow!("backfill: {e}"))?; slab_bytes += data.bytes().len(); slab.push(BackfillTuple { rfn: desc.rfn, @@ -676,11 +729,14 @@ fn fixed(raw: &[u8], what: &str) -> Result<[u8; N], String> { /// One binary COPY row: per attribute, `typsend` bytes or NULL. Laid out /// `[natts: u16][end: u32 per attribute][field bytes]`, an end's high bit -/// marking NULL +/// marking NULL and the next marking a field kept in `typsend` form #[derive(Clone, Debug, PartialEq)] pub struct CopyRow(Box<[u8]>); const COPY_NULL: u32 = 1 << 31; +const COPY_SEND: u32 = 1 << 30; +/// PostgreSQL builds each COPY row in one palloc, so 1 GiB bounds it +const COPY_END: u32 = COPY_SEND - 1; impl CopyRow { /// `fields` indexed by attnum - 1 @@ -702,20 +758,27 @@ impl CopyRow { } /// Row of `typsend` fields indexed by attnum - 1, each converted to its - /// on-disk form per `rel`'s attribute types - pub fn from_send(rel: &RelDescriptor, fields: &[Option<&[u8]>]) -> Result { + /// on-disk form per `kinds` + pub(crate) fn from_send(kinds: &[WireKind], fields: &[Option<&[u8]>]) -> Result { let head = 2 + 4 * fields.len(); let body = fields.iter().flatten().map(|f| f.len()).sum::(); let mut out = Vec::with_capacity(head + body); out.extend_from_slice(&(fields.len() as u16).to_le_bytes()); out.resize(head, 0); for (i, f) in fields.iter().enumerate() { + let kind = kinds.get(i).copied().unwrap_or(WireKind::Cast); if let Some(raw) = f { - let type_oid = rel.attributes.get(i).map_or(0, |a| a.type_oid); - store_field(copy_kind(type_oid), raw, &mut out)?; + store_field(kind, raw, &mut out)?; } - let end = u32::try_from(out.len() - head).map_err(|_| "COPY row past 4 GiB")?; - let mark = if f.is_none() { COPY_NULL } else { 0 }; + let end = u32::try_from(out.len() - head) + .ok() + .filter(|&end| end <= COPY_END) + .ok_or("COPY row past 1 GiB")?; + let mark = match (f, kind) { + (None, _) => COPY_NULL, + (Some(_), WireKind::Send) => COPY_SEND, + _ => 0, + }; out[2 + 4 * i..][..4].copy_from_slice(&(end | mark).to_le_bytes()); } Ok(Self(out.into())) @@ -741,11 +804,16 @@ impl CopyRow { fn end(&self, i: usize) -> Option { let at = 2 + 4 * i; let word = u32::from_le_bytes(self.0.get(at..at + 4)?.try_into().ok()?); - Some(word & !COPY_NULL) + Some(word & COPY_END) } /// Field of attribute index `i`; `None` past the row or NULL pub fn field(&self, i: usize) -> Option<&[u8]> { + self.slot(i).map(|(raw, _)| raw) + } + + /// Field of attribute index `i` and whether it kept `typsend` form + fn slot(&self, i: usize) -> Option<(&[u8], bool)> { let natts = self.natts()?; if i >= natts { return None; @@ -757,22 +825,27 @@ impl CopyRow { } let start = if i == 0 { 0 } else { self.end(i - 1)? }; let base = 2 + 4 * natts; - self.0 - .get(base + start as usize..base + (word & !COPY_NULL) as usize) + let raw = self + .0 + .get(base + start as usize..base + (word & COPY_END) as usize)?; + Some((raw, word & COPY_SEND != 0)) } } /// Attribute index `i` of a row [`CopyRow::from_send`] stored; NULL past the row pub(crate) fn copy_slot<'a>(row: &'a CopyRow, rel: &RelDescriptor, i: usize) -> DatumRef<'a> { - let (Some(att), Some(raw)) = (rel.attributes.get(i), row.field(i)) else { + let (Some(att), Some((raw, send))) = (rel.attributes.get(i), row.slot(i)) else { return DatumRef::Null; }; - match copy_kind(att.type_oid) { - WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Numeric => { + if send { + return DatumRef::Send(raw); + } + match wire_kind(att.type_oid) { + Some(WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Numeric) => { DatumRef::Varlena(raw) } - WireKind::Cast => DatumRef::Text(raw), - _ => DatumRef::Fixed(raw), + Some(_) => DatumRef::Fixed(raw), + None => DatumRef::Text(raw), } } @@ -826,13 +899,46 @@ fn store_field(kind: WireKind, raw: &[u8], out: &mut Vec) -> Result<(), Stri out.extend_from_slice(&numeric_from_send(raw)?); Ok(()) } - WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Cast => { + WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Cast | WireKind::Send => { out.extend_from_slice(raw); Ok(()) } } } +/// On-disk `attmissingval` from its `anyarray_send` form, `None` for types +/// outside the wire matrix or an array other than one non-null element +pub(crate) fn missing_from_send(type_oid: u32, send: &[u8]) -> Option> { + let kind = wire_kind(type_oid)?; + // array_send: ndim, hasnull, elemtype, (dim, lbound) per dim, (len, bytes) per element + let word = |i: usize| { + Some(i32::from_be_bytes( + send.get(4 * i..4 * i + 4)?.try_into().ok()?, + )) + }; + if word(0)? != 1 || word(3)? != 1 { + return None; + } + let len = usize::try_from(word(5)?).ok()?; + let raw = send.get(24..)?; + if raw.len() != len { + return None; + } + let mut elem = Vec::new(); + store_field(kind, raw, &mut elem).ok()?; + if matches!( + kind, + WireKind::Bytea | WireKind::Text | WireKind::Json | WireKind::Numeric + ) { + // 4-byte uncompressed varlena header, VARSIZE in upper 30 bits + let size = u32::try_from(elem.len() + 4).ok()?; + elem.splice(0..0, (size << 2).to_le_bytes()); + } + Some(crate::decode::heap_decoder::raw_missing_array( + type_oid, &elem, + )) +} + /// PostgreSQL `NAMEDATALEN`, `name`'s stored width const NAMEDATALEN: usize = 64; @@ -2236,28 +2342,163 @@ mod tests { } } - #[test] - fn plan_casts_out_of_matrix_and_skips_dropped() { - let d = desc(vec![ + fn varlena(attnum: i16, name: &str, type_oid: u32) -> RelAttr { + RelAttr { + type_len: -1, + type_byval: false, + type_align: 'i', + type_storage: 'x', + ..attr(attnum, name, type_oid, false) + } + } + + /// text[], aclitem, and a user enum: none has a local reader + const TEXTARRAYOID: u32 = 1009; + const ACLITEMOID: u32 = 1033; + const ENUM_OID: u32 = 16_500; + + fn mixed_desc() -> RelDescriptor { + desc(vec![ attr(1, "id", INT8OID, false), attr(2, "gone", TEXTOID, true), - attr(3, "price", NUMERICOID, false), - attr(4, "tags", 1009, false), // text[] — out of matrix - attr(5, "doc", JSONBOID, false), - ]); - let plan = column_plan(&d); + varlena(3, "price", NUMERICOID), + varlena(4, "tags", TEXTARRAYOID), + varlena(5, "doc", JSONBOID), + RelAttr { + type_len: 12, + ..attr(6, "acl", ACLITEMOID, false) + }, + RelAttr { + type_len: 4, + ..attr(7, "mood", ENUM_OID, false) + }, + ]) + } + + #[test] + fn plan_casts_out_of_matrix_and_skips_dropped() { + let plan = column_plan(&mixed_desc(), &HashSet::new()); assert_eq!( plan.select, - "\"id\", \"price\", \"tags\"::text, \"doc\"::text" + "\"id\", \"price\", \"tags\"::text, \"doc\"::text, \"acl\"::text, \"mood\"::text" ); - assert_eq!(plan.natts, 5); - assert_eq!(plan.cols.len(), 4, "dropped column not selected"); + assert_eq!(plan.kinds.len(), 7); + assert_eq!(plan.cols.len(), 6, "dropped column not selected"); assert_eq!(plan.cols[2].attnum, 4); - assert_eq!(copy_kind(INT8OID), WireKind::Int8); - assert_eq!(copy_kind(NUMERICOID), WireKind::Numeric); - assert_eq!(copy_kind(1009), WireKind::Cast); + assert_eq!(plan.kinds[0], WireKind::Int8); + assert_eq!(plan.kinds[2], WireKind::Numeric); + assert_eq!(plan.kinds[3], WireKind::Cast); // jsonb_out text, the document shape the WAL path yields - assert_eq!(copy_kind(JSONBOID), WireKind::Cast); + assert_eq!(plan.kinds[4], WireKind::Cast); + } + + /// Oracle-only types with binary I/O select bare; local-matrix types and + /// types lacking `typsend`/`typreceive` keep their text cast + #[test] + fn plan_keeps_send_form_of_binary_io_types() { + // jsonb has binary I/O but a local reader, aclitem lacks it + let binary = HashSet::from_iter([TEXTARRAYOID, ENUM_OID, JSONBOID]); + let plan = column_plan(&mixed_desc(), &binary); + assert_eq!( + plan.select, + "\"id\", \"price\", \"tags\", \"doc\"::text, \"acl\"::text, \"mood\"" + ); + assert_eq!( + plan.kinds[3..], + [ + WireKind::Send, + WireKind::Cast, + WireKind::Cast, + WireKind::Send + ] + ); + } + + /// Send fields keep their bytes and mark, NULLs stay NULL, and the row + /// reads back the same from its spooled bytes + #[test] + fn send_fields_round_trip_as_send_datums() { + let d = mixed_desc(); + let plan = column_plan(&d, &HashSet::from_iter([TEXTARRAYOID, ENUM_OID])); + let tags = b"\x00\x00\x00\x01array-send"; + let fields: [Option<&[u8]>; 7] = [ + Some(&7i64.to_be_bytes()), + None, + None, + Some(tags), + Some(br#"{"a": 1}"#), + Some(b"=r/postgres"), + None, + ]; + let row = CopyRow::from_send(&plan.kinds, &fields).unwrap(); + let row = CopyRow::from_bytes(row.bytes().into()).expect("spooled bytes parse"); + let slots: Vec<_> = (0..7).map(|i| copy_slot(&row, &d, i)).collect(); + assert_eq!( + slots, + [ + DatumRef::Fixed(&7i64.to_le_bytes()), + DatumRef::Null, + DatumRef::Null, + DatumRef::Send(tags), + DatumRef::Text(br#"{"a": 1}"#), + DatumRef::Text(b"=r/postgres"), + DatumRef::Null, + ] + ); + assert_eq!(row.field(3), Some(&tags[..]), "mark stays off the bytes"); + } + + #[test] + fn missing_from_send_matches_stored_default() { + use crate::decode::datum::{Datum, build}; + use crate::decode::heap_decoder::resolve; + use crate::schema::MissingDefault; + let hex = |s: &str| { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect::>() + }; + // anyarray_send(attmissingval) from PG 17 after ALTER TABLE ADD ... DEFAULT + let cases = [ + ( + NUMERICOID, + -1, + "0000000100000000000006a400000001000000010000000c0002000000000002000c1388", + build::numeric("12.50"), + ), + ( + TEXTOID, + -1, + "0000000100000000000000190000000100000001000000026869", + Datum::Varlena(b"hi".to_vec()), + ), + ( + NAMEOID, + 64, + "0000000100000000000000130000000100000001000000026e6d", + build::name("nm"), + ), + ( + INT8OID, + 8, + "000000010000000000000014000000010000000100000008fffffffffffffffd", + Datum::fixed(&(-3i64).to_le_bytes()), + ), + ]; + for (type_oid, type_len, send, want) in cases { + let mut att = attr(1, "c", type_oid, false); + att.type_len = type_len; + att.missing_default = missing_from_send(type_oid, &hex(send)).map(MissingDefault::Raw); + assert_eq!( + resolve(&att, DatumRef::Missing), + Some(want), + "type {type_oid}" + ); + } + const TSVECTOROID: u32 = 3614; + let tsvector = hex("000000010000000000000e1e0000000100000001000000080000000178000000"); + assert_eq!(missing_from_send(TSVECTOROID, &tsvector), None); } #[test] diff --git a/src/backfill/spool.rs b/src/backfill/spool.rs index 24c66a85..9b02f872 100644 --- a/src/backfill/spool.rs +++ b/src/backfill/spool.rs @@ -692,6 +692,29 @@ mod tests { ); } + /// `typsend` fields keep their mark across the file, so they still read + /// as oracle binary input + #[tokio::test] + async fn file_records_keep_send_fields() { + use crate::backfill::copy_backfill::{WireKind, copy_slot}; + use crate::decode::datum::DatumRef; + let dir = tempdir().unwrap(); + let mut spool = DeferredSpool::new(dir.path().join("send.bin"), 0); + let kinds = [WireKind::Int4, WireKind::Send, WireKind::Send]; + let fields: [Option<&[u8]>; 3] = [Some(&7i32.to_be_bytes()), Some(b"\x01send"), None]; + let mut copy = tuple(1, b""); + copy.data = TupleData::Copy(CopyRow::from_send(&kinds, &fields).unwrap()); + spool.push(copy.clone()).await.unwrap(); + let out = drain_all(spool).await; + assert_eq!(out[0].data, copy.data); + let TupleData::Copy(row) = &out[0].data else { + unreachable!("COPY row") + }; + let rel = crate::schema::int4_rel(16400, "t", 'r', &[(1, "id"), (2, "e"), (3, "f")]); + assert_eq!(copy_slot(row, &rel, 1), DatumRef::Send(b"\x01send")); + assert_eq!(copy_slot(row, &rel, 2), DatumRef::Null); + } + #[tokio::test] async fn checkpoint_then_reopen_keeps_every_record() { let dir = tempdir().unwrap(); diff --git a/src/catalog/shadow_catalog.rs b/src/catalog/shadow_catalog.rs index bd54c89a..e4d23450 100644 --- a/src/catalog/shadow_catalog.rs +++ b/src/catalog/shadow_catalog.rs @@ -1084,6 +1084,7 @@ fn descriptor_from_rows( type_align: attr.attalign.to_string(), type_storage: attr.attstorage.to_string(), missing: None, + missing_send: None, }; let mut rel_attr = raw.build().map_err(CatalogError::Parse)?; if let Some(hex) = attr.attmissingval.as_deref() diff --git a/src/catalog/type_bridge.rs b/src/catalog/type_bridge.rs index 2d0898dd..a77e9427 100644 --- a/src/catalog/type_bridge.rs +++ b/src/catalog/type_bridge.rs @@ -37,8 +37,9 @@ //! //! ## Default expression //! -//! `attmissingval` arrives as on-disk PG bytes (pinned worker) or element -//! text (SQL path). Tier-3 bytes resolve to text through shadow PG first. +//! `attmissingval` arrives as on-disk PG bytes (pinned worker, SQL path for +//! types with a wire codec) or element text (other SQL-path types). Tier-3 +//! bytes resolve to text through shadow PG first. //! The default encodes into its column as a row would, and //! `column_literal` renders that cell as the `ADD COLUMN` default diff --git a/src/decode/datum.rs b/src/decode/datum.rs index e8a260f6..82365ffe 100644 --- a/src/decode/datum.rs +++ b/src/decode/datum.rs @@ -27,6 +27,8 @@ pub enum DatumRef<'a> { BadTag(&'a [u8]), /// `typoutput` text: fast defaults read over SQL, COPY fields cast to text Text(&'a [u8]), + /// `typsend` bytes of a COPY field, read back through `typreceive` + Send(&'a [u8]), } /// Owned [`DatumRef`] for values outliving their record: TOAST values, @@ -40,6 +42,7 @@ pub enum Datum { Cstring(Vec), BadTag(Vec), Text(Vec), + Send(Vec), } impl Datum { @@ -52,6 +55,7 @@ impl Datum { Self::Cstring(b) => DatumRef::Cstring(b), Self::BadTag(b) => DatumRef::BadTag(b), Self::Text(b) => DatumRef::Text(b), + Self::Send(b) => DatumRef::Send(b), } } @@ -61,7 +65,11 @@ impl Datum { Self::Null => 0, Self::Fixed(b) => b.len(), Self::External(_) => std::mem::size_of::(), - Self::Varlena(b) | Self::Cstring(b) | Self::BadTag(b) | Self::Text(b) => b.len(), + Self::Varlena(b) + | Self::Cstring(b) + | Self::BadTag(b) + | Self::Text(b) + | Self::Send(b) => b.len(), } } diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index 74177a32..3e0f4355 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -798,12 +798,14 @@ pub fn form_block_data(attrs: &[RelAttr], values: &[Option]) -> Vec { let (bytes, align) = match v { None | Some(Datum::Null) => continue, Some(Datum::Fixed(b)) => (b.to_vec(), align_of(b.len())), - Some(Datum::Varlena(b) | Datum::BadTag(b) | Datum::Text(b)) if b.len() < 127 => { + Some(Datum::Varlena(b) | Datum::BadTag(b) | Datum::Text(b) | Datum::Send(b)) + if b.len() < 127 => + { let mut v = vec![(((b.len() + 1) << 1) | 1) as u8]; v.extend_from_slice(b); (v, 1) } - Some(Datum::Varlena(b) | Datum::BadTag(b) | Datum::Text(b)) => { + Some(Datum::Varlena(b) | Datum::BadTag(b) | Datum::Text(b) | Datum::Send(b)) => { let mut v = (((b.len() + 4) as u32) << 2).to_le_bytes().to_vec(); v.extend_from_slice(b); (v, align_of(4).max(4)) @@ -1125,6 +1127,7 @@ pub fn text_datum(type_oid: u32, text: &[u8]) -> TextDatum { crate::decode::codecs::parse_tid(text) .map(|(block, offset)| crate::decode::codecs::encode_tid(block, offset)), ), + // TODO(0.2.0): remove once 0.1.x descriptor logs migrate NUMERICOID => crate::decode::codecs::numeric_from_text(text) .map_or(TextDatum::Null, |b| TextDatum::Datum(Datum::Varlena(b))), _ => TextDatum::Opaque, @@ -1266,6 +1269,7 @@ pub fn resolve(att: &RelAttr, datum: DatumRef<'_>) -> Option { DatumRef::Cstring(b) => Datum::Cstring(b.to_vec()), DatumRef::BadTag(b) => Datum::BadTag(b.to_vec()), DatumRef::Text(b) => Datum::Text(b.to_vec()), + DatumRef::Send(b) => Datum::Send(b.to_vec()), }) } @@ -1496,7 +1500,6 @@ pub fn is_replica_identity_attr(replident: &ReplIdent, attnum: i16) -> bool { } /// PG `attmissingval`: one-element, no-null `ArrayType`, payload at `MAXALIGN(24)` -#[cfg(test)] pub(crate) fn raw_missing_array(elemtype: u32, elem: &[u8]) -> Vec { let mut v = Vec::with_capacity(24 + elem.len()); v.extend_from_slice(&0u32.to_le_bytes()); // vl_len_ (ignored) diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 608ac0ee..2eba7f5e 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -3132,7 +3132,7 @@ fn encode_datum( "unresolved TOAST pointer (xact buffer should have reassembled)", )); } - DatumRef::BadTag(_) => { + DatumRef::BadTag(_) | DatumRef::Send(_) => { return Err(unsupported( "unresolved source value routed to a local column", )); @@ -3285,6 +3285,7 @@ fn oracle_cell( })); } DatumRef::Text(t) => return Ok(OracleCell::TextInput(t.to_vec())), + DatumRef::Send(b) => return Ok(OracleCell::BinaryInput(b.to_vec())), DatumRef::Missing => return Err(unresolved("fast default left unresolved")), DatumRef::External(_) | DatumRef::BadTag(_) => { return Err(unresolved("unresolved source value on an oracle column")); @@ -4161,6 +4162,26 @@ mod tests { } } + /// COPY `typsend` bytes take the oracle's binary input under the planned + /// source type only; NULL stays a default cell + #[test] + fn send_datums_become_binary_input_cells() { + let source = Some(SourceColumn { + index: 0, + codec: Codec::Pending, + type_oid: 16_500, + }); + assert_eq!( + oracle_cell(DatumRef::Send(&[0, 1]), source, 16_500).unwrap(), + OracleCell::BinaryInput(vec![0, 1]) + ); + assert_eq!( + oracle_cell(DatumRef::Null, source, 16_500).unwrap(), + OracleCell::Default + ); + assert!(oracle_cell(DatumRef::Send(b"x"), source, 16_501).is_err()); + } + #[test] fn literal_string_cells_stay_off_the_oracle_batch_estimate() { let alloc = Allocator::global(&mimalloc::MiMalloc); diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs index 8cf0d9b7..8ad74406 100644 --- a/src/emit/direct_parity.rs +++ b/src/emit/direct_parity.rs @@ -1077,7 +1077,7 @@ fn numeric_send(neg: bool, weight: i16, dscale: u16, digits: &[i16]) -> Vec /// COPY binary and text fields encode as page-walk views of same values do #[test] fn copy_fields_match_heap_views() { - use crate::backfill::copy_backfill::{CopyRow, copy_slot}; + use crate::backfill::copy_backfill::{CopyRow, copy_slot, wire_kinds}; let rel = rel(vec![ attr(1, INT4OID, 4, 'i', "int4"), attr(2, INT8OID, 8, 'd', "int8"), @@ -1141,6 +1141,7 @@ fn copy_fields_match_heap_views() { )), ], ]; + let kinds = wire_kinds(&rel, &foldhash::HashSet::default()); let mut views = TableEncoder::new(plan(&rel, &targets, &ColumnRules::default())).unwrap(); let mut copy = TableEncoder::new(plan(&rel, &targets, &ColumnRules::default())).unwrap(); for (i, row) in rows.iter().enumerate() { @@ -1162,7 +1163,7 @@ fn copy_fields_match_heap_views() { .iter() .map(|c| c.as_ref().map(|(_, f)| &f[..])) .collect(); - let fields = CopyRow::from_send(&rel, &fields).unwrap(); + let fields = CopyRow::from_send(&kinds, &fields).unwrap(); copy.append_slots(|src| Ok(copy_slot(&fields, &rel, src.index)), &stamp) .unwrap(); } diff --git a/src/ops/bridge.rs b/src/ops/bridge.rs index f22978e5..2180ada5 100644 --- a/src/ops/bridge.rs +++ b/src/ops/bridge.rs @@ -23,7 +23,7 @@ use tokio::sync::Mutex; use crate::toast::FetchedValue; /// Frame and op layouts. Must equal `WS_PROTO_VERSION` in `pgext/walshadow.h` -pub const PROTO_VERSION: u32 = 7; +pub const PROTO_VERSION: u32 = 8; /// Catalog column plans. Must equal `WS_PROJECTION_VERSION` pub const PROJECTION_VERSION: u32 = 2; diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 9e87e284..dc68ca89 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -14,6 +14,7 @@ const CELL_DEFAULT: u8 = 0x00; const CELL_DISK_RAW: u8 = 0x01; const CELL_TEXT: u8 = 0x02; const CELL_LITERAL: u8 = 0x03; +const CELL_BINARY: u8 = 0x04; crate::atomic_stats! { pub struct OracleStats { @@ -53,14 +54,29 @@ pub enum OracleCell { TextInput(Vec), /// Pre-rendered text bypassing source type Literal(Vec), + /// `typsend` bytes for `typreceive` + BinaryInput(Vec), } impl OracleCell { /// Tag plus optional length-prefixed body pub fn wire_bytes(&self) -> usize { + let (tag, body) = self.tagged(); + if tag == CELL_DEFAULT { + 1 + } else { + 5 + body.len() + } + } + + /// Wire tag and body, empty for `Default` + fn tagged(&self) -> (u8, &[u8]) { match self { - Self::Default => 1, - Self::DiskRaw(b) | Self::TextInput(b) | Self::Literal(b) => 5 + b.len(), + Self::Default => (CELL_DEFAULT, &[]), + Self::DiskRaw(b) => (CELL_DISK_RAW, b), + Self::TextInput(b) => (CELL_TEXT, b), + Self::Literal(b) => (CELL_LITERAL, b), + Self::BinaryInput(b) => (CELL_BINARY, b), } } } @@ -121,9 +137,10 @@ impl OracleColumnBuf { pub fn push(&mut self, cell: OracleCell) { self.payload_capacity += match &cell { OracleCell::Default => 0, - OracleCell::DiskRaw(v) | OracleCell::TextInput(v) | OracleCell::Literal(v) => { - v.capacity() - } + OracleCell::DiskRaw(v) + | OracleCell::TextInput(v) + | OracleCell::Literal(v) + | OracleCell::BinaryInput(v) => v.capacity(), }; let bytes = cell.wire_bytes(); self.wire_bytes += bytes; @@ -223,14 +240,9 @@ impl OracleColumnBuf { push_u8(out, u8::from(self.nullable_target)); push_u32(out, self.cells.len() as u32); for cell in &self.cells { - let (tag, body) = match cell { - OracleCell::Default => (0, &[][..]), - OracleCell::DiskRaw(b) => (1, &b[..]), - OracleCell::TextInput(b) => (2, &b[..]), - OracleCell::Literal(b) => (3, &b[..]), - }; + let (tag, body) = cell.tagged(); push_u8(out, tag); - if tag != 0 { + if tag != CELL_DEFAULT { push_bytes(out, body); } } @@ -244,10 +256,11 @@ impl OracleColumnBuf { buf.cells.reserve(n.min(1 << 20)); for _ in 0..n { buf.push(match c.u8()? { - 0 => OracleCell::Default, - 1 => OracleCell::DiskRaw(c.bytes()?), - 2 => OracleCell::TextInput(c.bytes()?), - 3 => OracleCell::Literal(c.bytes()?), + CELL_DEFAULT => OracleCell::Default, + CELL_DISK_RAW => OracleCell::DiskRaw(c.bytes()?), + CELL_TEXT => OracleCell::TextInput(c.bytes()?), + CELL_LITERAL => OracleCell::Literal(c.bytes()?), + CELL_BINARY => OracleCell::BinaryInput(c.bytes()?), tag => { return Err(crate::xact::spill::SpillError::Format { offset: 0, @@ -478,21 +491,10 @@ fn encode_request(columns: &[OracleRequestColumn<'_>], n_rows: usize) -> Vec } for row in 0..n_rows { for c in columns { - let cell = &c.buf.cells()[row]; - match cell { - OracleCell::Default => out.push(CELL_DEFAULT), - OracleCell::DiskRaw(b) => { - out.push(CELL_DISK_RAW); - put_lenstr(&mut out, b); - } - OracleCell::TextInput(b) => { - out.push(CELL_TEXT); - put_lenstr(&mut out, b); - } - OracleCell::Literal(b) => { - out.push(CELL_LITERAL); - put_lenstr(&mut out, b); - } + let (tag, body) = c.buf.cells()[row].tagged(); + out.push(tag); + if tag != CELL_DEFAULT { + put_lenstr(&mut out, body); } } } @@ -650,7 +652,14 @@ mod tests { ); b.push(OracleCell::Literal(b"x".to_vec())); b.push(OracleCell::TextInput(b"{}".to_vec())); - let cols = [col(0, "t", "Array(Int32)", &a), col(3, "j", "JSON", &b)]; + let mut e = OracleColumnBuf::string(16_400, -1); + e.push(OracleCell::BinaryInput(b"ok".to_vec())); + e.push(OracleCell::Default); + let cols = [ + col(0, "t", "Array(Int32)", &a), + col(3, "j", "JSON", &b), + col(4, "e", "String", &e), + ]; let out = encode_request(&cols, 2); let mut c = FRAME_PREFIX_BYTES; @@ -660,13 +669,15 @@ mod tests { v }; assert_eq!(u32_at(&mut c), 2, "n_rows"); - assert_eq!(u32_at(&mut c), 2, "n_columns"); + assert_eq!(u32_at(&mut c), 3, "n_columns"); // First column's metadata, then the second's, before any cell assert_eq!(u32_at(&mut c), 1007, "source oid"); assert_eq!(u32_at(&mut c) as i32, -1, "typmod"); c += 4 + 1 + 4 + "Array(Int32)".len(); assert_eq!(u32_at(&mut c), 3802, "second source oid"); c += 4 + 4 + 1 + 4 + "JSON".len(); + assert_eq!(u32_at(&mut c), 16_400, "third source oid"); + c += 4 + 4 + 1 + 4 + "String".len(); assert_eq!( &out[c..], &[ @@ -683,6 +694,13 @@ mod tests { 0, 1, b'x', + CELL_BINARY, + 0, + 0, + 0, + 2, + b'o', + b'k', CELL_DEFAULT, CELL_TEXT, 0, @@ -691,10 +709,33 @@ mod tests { 2, b'{', b'}', + CELL_DEFAULT, ] ); } + /// Every cell kind survives the spill encoding + #[test] + fn column_cells_round_trip_through_spill() { + let mut buf = OracleColumnBuf::string(16_400, 7); + for cell in [ + OracleCell::Default, + OracleCell::DiskRaw(vec![1, 2]), + OracleCell::TextInput(b"{}".to_vec()), + OracleCell::Literal(b"x".to_vec()), + OracleCell::BinaryInput(vec![0, 0, 0, 1]), + OracleCell::BinaryInput(Vec::new()), + ] { + buf.push(cell); + } + let mut out = Vec::new(); + buf.encode_into(&mut out); + let back = OracleColumnBuf::decode(&mut crate::xact::spill::Cursor::new(&out)).unwrap(); + assert_eq!(back.cells(), buf.cells()); + assert_eq!(back.approx_size(), buf.approx_size()); + assert!(back.same_column(&buf)); + } + #[test] fn request_column_bytes_matches_framed_size() { let mut buf = OracleColumnBuf::new( diff --git a/src/pg.rs b/src/pg.rs index bec0977c..33c34781 100644 --- a/src/pg.rs +++ b/src/pg.rs @@ -3,7 +3,7 @@ use tokio_postgres::types::{Oid, PgLsn}; use tokio_postgres::{Client, Row}; -use crate::schema::RelAttr; +use crate::schema::{MissingDefault, RelAttr}; pub use walrus::pg::backup::parse_pg_lsn; @@ -43,7 +43,8 @@ pub const ATTR_SQL: &str = "SELECT \ a.attlen::int2, \ a.attalign::text, \ a.attstorage::text, \ - CASE WHEN a.atthasmissing THEN a.attmissingval::text END \ + CASE WHEN a.atthasmissing THEN a.attmissingval::text END, \ + CASE WHEN a.atthasmissing AND t.typsend::oid <> 0 THEN anyarray_send(a.attmissingval) END \ FROM pg_attribute a \ LEFT JOIN pg_type t ON t.oid = a.atttypid \ WHERE a.attrelid = $1 AND a.attnum >= 1 \ @@ -65,6 +66,9 @@ pub struct RawAttr { pub type_storage: String, /// `attmissingval::text` array literal pub missing: Option, + /// `anyarray_send(attmissingval)`, preferred over `missing` where the + /// type's send form converts to on-disk bytes + pub missing_send: Option>, } impl RawAttr { @@ -82,10 +86,22 @@ impl RawAttr { type_align: row.get(9), type_storage: row.get(10), missing: row.get(11), + missing_send: row.get(12), } } pub fn build(self) -> Result { + let missing_default = self + .missing_send + .as_deref() + .and_then(|send| crate::backfill::copy_backfill::missing_from_send(self.type_oid, send)) + .map(MissingDefault::Raw) + .or_else(|| { + self.missing + .as_deref() + .and_then(parse_array_one_element) + .map(MissingDefault::Text) + }); Ok(RelAttr { attnum: self.attnum, name: self.name, @@ -98,11 +114,7 @@ impl RawAttr { type_len: self.type_len, type_align: single_char(&self.type_align, "attalign")?, type_storage: single_char(&self.type_storage, "attstorage")?, - missing_default: self - .missing - .as_deref() - .and_then(parse_array_one_element) - .map(crate::schema::MissingDefault::Text), + missing_default, }) } } @@ -190,6 +202,7 @@ mod tests { type_align: "i".into(), type_storage: "x".into(), missing: None, + missing_send: None, }; let attr = raw.build().unwrap(); assert!(attr.dropped); @@ -214,6 +227,7 @@ mod tests { type_align: "ii".into(), type_storage: "p".into(), missing: None, + missing_send: None, }; assert!(raw.build().is_err()); } diff --git a/src/schema.rs b/src/schema.rs index 145f5947..2f79ce75 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -211,10 +211,11 @@ pub struct RelAttr { pub missing_default: Option, } -/// Fast default (`attmissingval`), read by `missing_datum`. Shadow -/// catalog reads yield `Raw` on-disk bytes; `Text` (element's `anyarray_out` -/// form) comes from source-PG SQL, which has no way to the bytes, oracle -/// rendered DDL defaults, and descriptor logs older than raw shadow reads +/// Fast default (`attmissingval`), read by `missing_datum`. Shadow catalog +/// reads yield `Raw` on-disk bytes, as do source-PG SQL reads of types whose +/// send form converts to on-disk bytes; `Text` (element's `anyarray_out` form) +/// covers other SQL-read types, oracle rendered DDL defaults, and descriptor +/// logs older than raw reads #[derive(Debug, Clone, PartialEq, Eq)] pub enum MissingDefault { Raw(Vec), diff --git a/src/toast/read_ahead.rs b/src/toast/read_ahead.rs index 7b156e18..c62ea6b1 100644 --- a/src/toast/read_ahead.rs +++ b/src/toast/read_ahead.rs @@ -58,7 +58,7 @@ pub struct PendingReads { entries: HashMap, } -/// Fetch and decompress transients; rendered values top up once their size is known +/// Fetch and decompress transients; held values top up once their size is known fn read_cost(p: &ToastPointer) -> usize { pointer_extsize(p) .saturating_mul(2) @@ -66,9 +66,9 @@ fn read_cost(p: &ToastPointer) -> usize { .saturating_add(ENTRY_OVERHEAD) } -/// Bytes a value rendered to `rendered` bytes holds until planning consumes it -pub fn retained_bytes(rendered: usize, p: &ToastPointer) -> u64 { - rendered.max(pointer_footprint(p)).saturating_mul(2) as u64 +/// Bytes a value of `held` raw bytes retains until planning consumes it +pub fn retained_bytes(held: usize, p: &ToastPointer) -> u64 { + held.max(pointer_footprint(p)).saturating_mul(2) as u64 } impl PendingReads { diff --git a/tests/oracle.rs b/tests/oracle.rs index 7c9d43c7..ae236368 100644 --- a/tests/oracle.rs +++ b/tests/oracle.rs @@ -567,6 +567,118 @@ async fn oracle_encodes_tier3_disk_bytes() { assert_eq!(oracle.stats.errors.load(Ordering::Relaxed), 0); } +/// `array_send` of one-dimensional int4 `values`, big-endian throughout +fn array_int4_send(values: &[i32]) -> Vec { + let mut out = Vec::new(); + for v in [1i32, 0, 23, values.len() as i32, 1] { + out.extend_from_slice(&v.to_be_bytes()); + } + for v in values { + out.extend_from_slice(&4i32.to_be_bytes()); + out.extend_from_slice(&v.to_be_bytes()); + } + out +} + +/// COPY `typsend` bytes decode through `typreceive`; NULL takes the +/// column default; leftover or short bytes and types without a receive +/// function fail the request, which the worker survives +#[tokio::test(flavor = "current_thread")] +async fn oracle_receives_binary_cells() { + let tmp = tempfile::tempdir().unwrap(); + let Some(guard) = start_pg(&tmp, ports::PG_SHADOW_PORT) else { + return; + }; + let oracle = oracle_on(&guard.sh).await; + let encode = async |oid, ty, cells: Vec| { + let n = cells.len(); + let b = buf(oid, cells); + let columns = [OracleRequestColumn { + ordinal: 0, + name: "value", + target_type: ty, + buf: &b, + }]; + oracle + .encode_batch(Oracle::ANY_DATABASE, &columns, n, alloc()) + .await + }; + + let block = encode( + INT4ARRAYOID, + "Array(Int32)", + vec![ + OracleCell::BinaryInput(array_int4_send(&[1, 2, 3])), + OracleCell::Default, + OracleCell::BinaryInput(array_int4_send(&[])), + ], + ) + .await + .expect("binary cells decode"); + let tags = block.column(0).unwrap(); + assert_eq!(tags.array_offsets().unwrap(), [3, 3, 3]); + assert_eq!( + tags.array_values().unwrap().fixed().unwrap().1, + [1i32, 2, 3] + .into_iter() + .flat_map(i32::to_le_bytes) + .collect::>() + ); + // int4_send is four big-endian bytes, rendered through typoutput + let block = encode( + 23, + "Nullable(String)", + vec![ + OracleCell::BinaryInput((-7i32).to_be_bytes().to_vec()), + OracleCell::Default, + ], + ) + .await + .expect("scalar binary cell decodes"); + let value = block.column(0).unwrap(); + assert_eq!(value.null_map().unwrap(), [0, 1]); + assert_eq!( + value.nullable_inner().unwrap().string().unwrap(), + (&[2, 2][..], &b"-7"[..]) + ); + + let mut trailing = array_int4_send(&[1]); + trailing.push(0); + let mut short = array_int4_send(&[1]); + short.pop(); + let cases = [ + ( + INT4ARRAYOID, + "Array(Int32)", + trailing, + "incorrect binary data format", + ), + ( + INT4ARRAYOID, + "Array(Int32)", + short, + "insufficient data left in message", + ), + // aclitem has typinput but no typreceive + (1033, "String", b"x".to_vec(), "no binary input function"), + ]; + for (oid, ty, bad, expected) in cases { + let err = encode(oid, ty, vec![OracleCell::BinaryInput(bad)]) + .await + .expect_err("malformed binary cell fails"); + let msg = err.to_string(); + assert!(msg.contains(expected), "{msg}"); + assert!(!err.retryable()); + } + encode( + INT4ARRAYOID, + "Array(Int32)", + vec![OracleCell::BinaryInput(array_int4_send(&[4]))], + ) + .await + .expect("worker still serves"); +} + #[tokio::test(flavor = "current_thread")] async fn oracle_defaults_fill_absent_cells() { let tmp = tempfile::tempdir().unwrap(); diff --git a/tests/runtime_config_e2e.rs b/tests/runtime_config_e2e.rs index 12d18aa5..645d7947 100644 --- a/tests/runtime_config_e2e.rs +++ b/tests/runtime_config_e2e.rs @@ -82,6 +82,11 @@ //! * `ALTER DATABASE … SET intervalstyle = iso_8601` before a COPY load. //! * Expect: COPY and WAL rows render the same interval identically. //! +//! 13. `copy_load_matches_wal_for_oracle_types` +//! * COPY loads rows of types only the oracle converts, binary I/O ones +//! by `typsend`/`typreceive`, aclitem by text; WAL inserts the same. +//! * Expect: COPY and WAL rows, NULL rows included, read identically. +//! //! Source-side `config_*` install runs the real `sql/runtime_config_install.sql` //! inside the bootstrap schema dump, so the drills double as install-script @@ -1315,18 +1320,29 @@ async fn copy_load_converges( ch.query("CREATE DATABASE IF NOT EXISTS walshadow_test") .expect("create db"); - let mut pipeline = fx::build_pipeline(fx::BuildPipelineArgs { - tmp: &tmp, - source: &source, - shadow: &shadow, - shadow_filter_dir: &shadow_filter_dir, - shadow_stream_state, - ch_database: "walshadow_test", - ch_tcp_port: slot.ch_tcp, - mappings: vec![], - app_name, - ddl: Some(overlay_ddl_args()), - }) + // Worker binds only once recovery reaches consistency, so budget the dial + let socket = shadow.bridge_socket().expect("bridge configured"); + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) + .await + .expect("bridge connect"); + let mut pipeline = fx::build_pipeline_tuned( + fx::BuildPipelineArgs { + tmp: &tmp, + source: &source, + shadow: &shadow, + shadow_filter_dir: &shadow_filter_dir, + shadow_stream_state, + ch_database: "walshadow_test", + ch_tcp_port: slot.ch_tcp, + mappings: vec![], + app_name, + ddl: Some(overlay_ddl_args()), + }, + |_| {}, + Some(std::sync::Arc::new(walshadow::oracle::Oracle::new( + std::sync::Arc::new(bridge), + ))), + ) .await; let driver = fx::spawn_workload(&source, workload); @@ -1435,3 +1451,57 @@ async fn copy_load_pins_output_settings() { "COPY row renders like the WAL row" ); } + +/// Types only the oracle converts: binary I/O ones travel as `typsend` +/// bytes, aclitem (no `typsend`) as text. Each COPY row must read like the +/// WAL row holding the same values +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn copy_load_matches_wal_for_oracle_types() { + if !fx::tools::requirements_available() { + return; + } + let values = "'{1,NULL,3}', '{a,\"b c\"}', 'happy', '(7,\"x y\")', 42, \ + 'k=>v, n=>NULL', 'fat:2 rat:1', '[3,9)', 'postgres=r/postgres'"; + let nulls = "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL"; + let schema_sql = format!( + "{INSTALL_SQL}\n\ + CREATE EXTENSION hstore;\n\ + CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');\n\ + CREATE TYPE pair AS (a int, b text);\n\ + CREATE DOMAIN posint AS int CHECK (VALUE > 0);\n\ + CREATE SCHEMA app;\n\ + CREATE TABLE app.wide (id bigint PRIMARY KEY, ints int4[], texts text[], m mood, \ + p pair, d posint, h hstore, tv tsvector, r int8range, acl aclitem);\n\ + INSERT INTO app.wide VALUES (1, {values}), (3, {nulls});\n" + ); + let (ch, got) = copy_load_converges( + &schema_sql, + vec![ + "INSERT INTO walshadow.config_table (namespace, relname, replicate, initial_load) \ + VALUES ('app', 'wide', true, 'copy')" + .into(), + format!("INSERT INTO app.wide VALUES (2, {values}), (4, {nulls})"), + "SELECT pg_switch_wal()".into(), + ], + "walshadow-copy-binary", + "SELECT count() FROM walshadow_test.app_wide FINAL WHERE _is_deleted = 0", + "4", + ) + .await; + assert_eq!(got, "4", "COPY and WAL rows reach CH"); + let row = |id: u32| { + ch.query(&format!( + "SELECT ints, texts, m, p, d, h, tv, r, acl FROM walshadow_test.app_wide FINAL \ + WHERE _is_deleted = 0 AND id = {id}" + )) + .expect("ch row") + }; + let (copy, wal) = (row(1), row(2)); + assert_eq!(copy, wal, "COPY row reads like the WAL row"); + assert_eq!( + copy, + "[1,NULL,3]\t['a','b c']\thappy\t(7,\"x y\")\t42\t{'k':'v','n':NULL}\t\ + \\'fat\\':2 \\'rat\\':1\t[3,9)\tpostgres=r/postgres" + ); + assert_eq!(row(3), row(4), "NULL COPY row reads like the NULL WAL row"); +} diff --git a/tests/shadow_catalog.rs b/tests/shadow_catalog.rs index aa1d82d4..ad04789b 100644 --- a/tests/shadow_catalog.rs +++ b/tests/shadow_catalog.rs @@ -366,6 +366,81 @@ async fn dropped_column_keeps_physical_slot() { assert_eq!(b.attributes, desc.attributes); } +/// Source SQL reads fast defaults through `anyarray_send`, landing the stored +/// datums shadow reads do; types without a wire codec stay element text +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn source_sql_fast_defaults_match_shadow_reads() { + use walshadow::datum::DatumRef; + use walshadow::heap_decoder::resolve; + use walshadow::schema::MissingDefault; + if !pg_available() { + return; + } + let tmp = tempfile::tempdir().unwrap(); + let shadow = make_shadow(&tmp, ports::PG_SHADOW_PORT); + shadow.initdb().expect("initdb"); + shadow.write_base_conf().expect("conf"); + shadow.start().expect("start"); + let _stop = stop_on_drop(&shadow); + + shadow + .apply_schema_dump( + "CREATE SCHEMA fd;\n\ + CREATE TABLE fd.t (id int);\n\ + INSERT INTO fd.t VALUES (1);\n\ + ALTER TABLE fd.t ADD n numeric DEFAULT 12.50, ADD s text DEFAULT 'hi', \ + ADD nm name DEFAULT 'nm', ADD b int8 DEFAULT -3, \ + ADD ts timestamp DEFAULT '2024-01-02 03:04:05', ADD v tsvector DEFAULT 'x';\n", + ) + .expect("schema dump"); + + let db = current_db_oid(&shadow); + let filenode = user_relation_filenode(&shadow, "fd.t"); + let mut cat = open_catalog(&shadow, Duration::from_secs(5)).await; + let stored = cat + .descriptor_by_name(&RelName::new("fd", "t")) + .await + .expect("descriptor_by_name") + .expect("fd.t exists"); + + let cfg = shadow.config(); + let conninfo = socket_conninfo( + cfg.socket_dir.to_str().unwrap(), + cfg.port, + "postgres", + "postgres", + ); + let (client, conn) = tokio_postgres::connect(&conninfo, tokio_postgres::NoTls) + .await + .expect("sql connect"); + tokio::spawn(conn); + let seeded = walshadow::backfill_bootstrap::seed_catalog_from_source(&client) + .await + .expect("seed catalog") + .get(db, filenode) + .expect("fd.t seeded"); + + assert_eq!(seeded.attributes.len(), stored.attributes.len()); + for (sql, disk) in seeded.attributes.iter().zip(&stored.attributes).skip(1) { + let resolved = resolve(sql, DatumRef::Missing); + assert!(resolved.is_some(), "{} default missing", sql.name); + if sql.name == "v" { + assert_eq!( + sql.missing_default, + Some(MissingDefault::Text("'x'".into())) + ); + continue; + } + assert!( + matches!(sql.missing_default, Some(MissingDefault::Raw(_))), + "{}: {:?}", + sql.name, + sql.missing_default + ); + assert_eq!(resolved, resolve(disk, DatumRef::Missing), "{}", sql.name); + } +} + /// Replica-identity carriage: `RelDescriptor::replident` carries the resolved /// `pg_class.relreplident` and, for `USING INDEX`, the index oid plus /// `pg_index.indkey` attnum list. Heap decoder reads both off the From 44f88c91b39df70ca14e9668de05a5627185de54 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:56:31 +0000 Subject: [PATCH 17/20] offload work from pump --- architecture/README.md | 8 +- benches/spool_read.rs | 7 +- docs/configuration.md | 3 +- src/backfill/visibility_pending.rs | 2 +- src/backfill/wal_replay.rs | 21 +- src/emit/ch_emitter.rs | 56 +++- src/emit/direct_parity.rs | 10 +- src/emit/pipeline/batcher.rs | 239 +++++++++++--- src/emit/pipeline/bootstrap.rs | 19 +- src/emit/pipeline/encode.rs | 103 ++++++ src/emit/pipeline/mod.rs | 1 + src/emit/pipeline/plan_spool.rs | 501 ++++++++++++++++++++++++----- src/emit/pipeline/planner.rs | 156 +++++++-- src/emit/pipeline/reorder.rs | 244 ++++++++++---- src/emit/pipeline/resolver.rs | 4 - src/emit/pipeline/tail.rs | 17 +- src/ops/oracle.rs | 10 + src/xact/xact_buffer.rs | 121 ++++--- 18 files changed, 1175 insertions(+), 347 deletions(-) create mode 100644 src/emit/pipeline/encode.rs diff --git a/architecture/README.md b/architecture/README.md index b746e863..e3f420d4 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -40,9 +40,11 @@ planning committed transactions ![Commit pipeline: reorder plans column chunks, one batcher feeds a conversion and sealing pool, inserters send prepared blocks, and acknowledgements advance a contiguous watermark](workers.svg) `BufferingDecoderSink` and `ReorderSink` share one record-queue worker -`ReorderSink` plans each commit, encoding its rows into per-table column -chunks, and places them onto the batcher in order. Batcher merges chunks of -many commits into Native blocks and groups blocks into per-table INSERTs. A +`ReorderSink` plans each commit and places it onto the batcher in order. +Planning routes, validates and detoasts rows but leaves them WAL bytes; each +outbox flush hands the rows of many commits to one encode job on the runtime, +which writes per-table column chunks. Batcher awaits jobs in order, merges +chunks into Native blocks and groups blocks into per-table INSERTs. A separate conversion pool resolves PostgreSQL types, then serializes each block once and keeps prepared bytes until its INSERT is durable. Compression follows `[ch] compression`; inserters rebuild blocks only when a connection rejects diff --git a/benches/spool_read.rs b/benches/spool_read.rs index d905abf8..38f4a7dc 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -401,8 +401,9 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { while let Some(item) = rd.next_item()? { match item { PlanItem::Segment(s) => { - n += s.chunks.iter().map(|c| c.rows()).sum::(); - segments.push(s.chunks); + let chunks = s.into_chunks()?; + n += chunks.iter().map(|c| c.rows()).sum::(); + segments.push(chunks); } PlanItem::Control { .. } => controls += 1, PlanItem::Truncate { .. } => unreachable!(), @@ -446,7 +447,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { let mut rrd = reference.seal(u64::MAX, 0)?.replay()?; let mut want = Vec::new(); while let Some(PlanItem::Segment(s)) = rrd.next_item()? { - want.extend(s.chunks); + want.extend(s.into_chunks()?); } assert!( merged(want) == merged(segments.into_iter().flatten().collect()), diff --git a/docs/configuration.md b/docs/configuration.md index db994070..80f2baa9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -305,7 +305,8 @@ Both settings apply to initial load and WAL processing in every value mode ### Memory budget `[memory] resident_payload_max` controls backpressure for payload bytes held -while decoding and inserting rows. New work waits when budget is full. Default +while decoding, planning and batching rows; a ClickHouse tail's own budget +covers them once sealed into an INSERT. New work waits when budget is full. Default is one half of cgroup memory limit, or host memory without a cgroup limit, with a 512 MiB minimum. This default assumes a dedicated host or container. Set an explicit limit in shared environments diff --git a/src/backfill/visibility_pending.rs b/src/backfill/visibility_pending.rs index 5248602e..968cbfa5 100644 --- a/src/backfill/visibility_pending.rs +++ b/src/backfill/visibility_pending.rs @@ -1024,7 +1024,7 @@ mod tests { enc.append_view(&desc, &view.datums, &patch, &stamp) }) .unwrap(); - let chunks = chunks.take().unwrap(); + let chunks = chunks.take(); // UInt32 xids and UInt16 infomask read back as same-width ints assert_eq!( chunks[0].1.cells(), diff --git a/src/backfill/wal_replay.rs b/src/backfill/wal_replay.rs index bd0af8e8..07804656 100644 --- a/src/backfill/wal_replay.rs +++ b/src/backfill/wal_replay.rs @@ -390,27 +390,17 @@ impl WalReplaySink { return Ok(()); } let ship = permit.is_some(); - self.stage(seq, permit.map(Arc::new))?; + self.stage(seq, permit.map(Arc::new)); if ship || self.staged_rows >= STAGED_ROWS { self.flush_pending().await?; } Ok(()) } - fn stage( - &mut self, - seq: u64, - permit: Option>, - ) -> std::result::Result<(), SinkError> { - let chunks: Vec<_> = self - .chunks - .take() - .map_err(encode_err)? - .into_iter() - .map(|(_, c)| c) - .collect(); + fn stage(&mut self, seq: u64, permit: Option>) { + let chunks: Vec<_> = self.chunks.take().into_iter().map(|(_, c)| c).collect(); if chunks.is_empty() { - return Ok(()); + return; } self.staged_rows += chunks.iter().map(|c| c.rows()).sum::(); self.staged.push(PlacedSegment { @@ -418,7 +408,6 @@ impl WalReplaySink { chunks, permit, }); - Ok(()) } /// Mirror all chunks from restored TOAST page image. Bypass transaction @@ -602,7 +591,7 @@ impl WalReplaySink { } drain.finish().await.map_err(SinkError::from)?; if let Some((seq, rows)) = self.open.take() { - self.stage(seq.seq(), None)?; + self.stage(seq.seq(), None); self.seqs.place(seq, rows); if self.staged_rows >= STAGED_ROWS { self.flush_pending().await?; diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 2eba7f5e..a214e01d 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -1905,6 +1905,10 @@ pub(crate) enum Append { Full, } +/// One row's mapped datums, inline up to wide tables so encoding a row +/// does not allocate +type RowCells<'a> = SmallVec<[DatumRef<'a>; 64]>; + /// Per-table per-xact accumulator, one block buffer per CH column. pub(crate) struct TableEncoder { pub(crate) plan: TablePlan, @@ -1985,6 +1989,33 @@ impl ColumnBuf { }) } + /// Empty buffer of this shape + fn empty_like(&self) -> Self { + match self { + Self::Fixed { width, .. } => Self::Fixed { + width: *width, + bytes: Vec::new(), + }, + Self::String { absent, .. } => Self::String { + offsets: Vec::new(), + data: Vec::new(), + absent, + }, + Self::NullableFixed { width, .. } => Self::NullableFixed { + width: *width, + null_map: Vec::new(), + inner: Vec::new(), + }, + Self::NullableString { absent, .. } => Self::NullableString { + offsets: Vec::new(), + data: Vec::new(), + null_map: Vec::new(), + absent, + }, + Self::Oracle(o) => Self::Oracle(o.empty_like()), + } + } + pub(crate) fn allocated_bytes(&self) -> usize { match self { Self::Fixed { bytes, .. } => bytes.capacity(), @@ -2292,10 +2323,8 @@ impl ColumnBuf { } } -/// Fresh per-column buffers matching `plan` (mapped + synthetic). -/// Shared by [`TableEncoder::new`] and [`TableEncoder::take_block`] so -/// synthetic-column widths live in one place. -pub(crate) fn fresh_buffers(plan: &TablePlan) -> Result, EmitterError> { +/// Fresh per-column buffers matching `plan` (mapped + synthetic) +fn fresh_buffers(plan: &TablePlan) -> Result, EmitterError> { let mut buffers = Vec::with_capacity(plan.columns.len() + 4); for c in &plan.columns { buffers.push(match c.encoding { @@ -2359,16 +2388,17 @@ impl TableEncoder { }) } - /// Swap accumulated slabs out for fresh empties, returning old slabs - /// + row count. Transfers ownership rather than reusing allocations. - pub(crate) fn take_block(&mut self) -> Result<(Vec, usize), EmitterError> { - let fresh = fresh_buffers(&self.plan)?; + /// Swap accumulated slabs out for empties of the same shape, returning + /// old slabs + row count. Transfers ownership rather than reusing + /// allocations + pub(crate) fn take_block(&mut self) -> (Vec, usize) { + let fresh = self.buffers.iter().map(ColumnBuf::empty_like).collect(); let old = std::mem::replace(&mut self.buffers, fresh); let rows = self.rows; self.rows = 0; self.approx_bytes = 0; self.oracle_bytes = self.oracle_overhead; - Ok((old, rows)) + (old, rows) } /// Whether `block` lays out the same columns as these buffers @@ -2440,7 +2470,7 @@ impl TableEncoder { HeapOp::Delete => decoded.old.as_ref(), _ => decoded.new.as_ref(), }; - let cells: SmallVec<[DatumRef<'_>; 16]> = mapping + let cells: RowCells<'_> = mapping .columns .iter() .map(|col| { @@ -2470,7 +2500,7 @@ impl TableEncoder { patch: &[(usize, Datum)], stamp: &RowStamp, ) -> Result { - let cells: SmallVec<[DatumRef<'_>; 16]> = self + let cells: RowCells<'_> = self .plan .columns .iter() @@ -2498,7 +2528,7 @@ impl TableEncoder { mut slot: impl FnMut(SourceColumn) -> Result, EmitterError>, stamp: &RowStamp, ) -> Result { - let mut cells: SmallVec<[DatumRef<'a>; 16]> = SmallVec::new(); + let mut cells = RowCells::new(); for c in &self.plan.columns { cells.push( c.source @@ -5230,7 +5260,7 @@ mod tests { (1, before), "row not appended" ); - assert_eq!(enc.take_block().unwrap().1, 1); + assert_eq!(enc.take_block().1, 1); assert_eq!( enc.append_row(&committed(2, Some(&big)), &m, OP_INSERT) .unwrap(), diff --git a/src/emit/direct_parity.rs b/src/emit/direct_parity.rs index 8ad74406..0dbfb5e1 100644 --- a/src/emit/direct_parity.rs +++ b/src/emit/direct_parity.rs @@ -223,8 +223,8 @@ fn assert_parity( return; } } - let (a, na) = cells.take_block().unwrap(); - let (b, nb) = direct.take_block().unwrap(); + let (a, na) = cells.take_block(); + let (b, nb) = direct.take_block(); assert_eq!(na, nb); let wire = |bufs: &[ColumnBuf]| { bufs.iter() @@ -809,7 +809,7 @@ fn predeclared_columns_match() { let view = view_block_data(&data, &rel).unwrap(); direct.append_view(&rel, &view.datums, &[], &stamp).unwrap(); let wire = |e: &mut TableEncoder| { - let (bufs, _) = e.take_block().unwrap(); + let (bufs, _) = e.take_block(); bufs.iter() .map(|b| format!("{:?}", b.wire())) .collect::>() @@ -902,7 +902,7 @@ fn plan_both(rel: &RelDescriptor, targets: &[&str], changes: &[Change], values: let mut bytes = Vec::new(); while let Some(item) = rd.next_item().unwrap() { if let PlanItem::Segment(s) = item { - bytes.extend(s.chunks.iter().map(|c| (c.rows(), c.to_bytes()))); + bytes.extend(s.chunks().iter().map(|c| (c.rows(), c.to_bytes()))); } } out.push(bytes); @@ -1168,7 +1168,7 @@ fn copy_fields_match_heap_views() { .unwrap(); } let wire = |enc: &mut TableEncoder| { - let (bufs, rows) = enc.take_block().unwrap(); + let (bufs, rows) = enc.take_block(); let wire: Vec<_> = bufs.iter().map(|b| format!("{:?}", b.wire())).collect(); (rows, wire) }; diff --git a/src/emit/pipeline/batcher.rs b/src/emit/pipeline/batcher.rs index 906218d5..c7233f52 100644 --- a/src/emit/pipeline/batcher.rs +++ b/src/emit/pipeline/batcher.rs @@ -34,6 +34,7 @@ use crate::decode::heap_decoder::CommittedTuple; use crate::emit::ch_emitter::{ Append, ColumnBuf, ColumnEncoding, EmitterStats, EncodedChunk, TableEncoder, TablePlan, }; +use crate::emit::pipeline::encode::EncodedRun; use crate::emit::pipeline::{DEFAULT_PIPELINE_FLUSH, Fatal}; use crate::emit::route::RouteSnapshot; use crate::schema::{RelDescriptor, TableKey}; @@ -54,7 +55,7 @@ pub struct RowTable { pub struct PlacedSegment { pub seq: u64, pub chunks: Vec, - /// Released post-insert-ack, when every covering `InsertBatch` drops + /// Released once every table holding these rows emits its batch pub permit: Option>, } @@ -78,7 +79,7 @@ impl PlacedSegment { } Ok(Self { seq, - chunks: builder.take()?.into_iter().map(|(_, c)| c).collect(), + chunks: builder.take().into_iter().map(|(_, c)| c).collect(), permit: None, }) } @@ -183,8 +184,6 @@ pub(crate) struct InsertBatch { pub(crate) blocks: Vec, pub(crate) n_rows: usize, pub(crate) per_seq: Vec<(u64, u64)>, - /// Admission permit shares covering these rows, dropped post-insert-ack - pub(crate) permits: Vec>, /// Output bytes held, shrunk once blocks compress pub(crate) encoded: crate::budget::MemoryPermit, } @@ -197,6 +196,9 @@ pub enum BatcherMsg { /// channel send + cross-thread wakeup that dominates coordination /// under sustained load Segments(Vec), + /// Encode job for segments staged across commits, merged once it + /// finishes; later messages wait behind it + Encoded(JoinHandle>), /// Seal every open table, push to inserters, reply. Barrier (drain before /// DDL/TRUNCATE) and shutdown. FlushAll(oneshot::Sender<()>), @@ -222,6 +224,7 @@ struct Table { allocated_bytes: usize, meta: Arc, seq_counts: Vec<(u64, u64)>, + /// Admission covering open rows, until the output pool takes them slice_permits: Vec>, deadline: Option, } @@ -237,7 +240,7 @@ impl Table { if self.enc.rows == 0 { return Ok(()); } - let (buffers, n_rows) = self.enc.take_block().map_err(|e| e.to_string())?; + let (buffers, n_rows) = self.enc.take_block(); self.sealed_rows += n_rows; self.sealed_bytes += buffers .iter() @@ -363,6 +366,33 @@ pub(crate) fn spawn( break; } } + Some(BatcherMsg::Encoded(job)) => { + let run = match job.await { + Ok(Ok(run)) => run, + Ok(Err(e)) => { + fatal.set(format!("encode: {e}")); + break; + } + Err(e) => { + fatal.set(format!("encode job: {e}")); + break; + } + }; + let live = effective_cfg(&cfg, snapshot(config_rx.as_ref()).as_deref()); + let ctx = RowCtx { cfg: live, out: &out, alloc, epoch, stats }; + let permit = run.permit.map(Arc::new); + let mut res = Ok(()); + for (chunk, seqs) in run.chunks { + res = merge_chunk(&mut tables, &ctx, chunk, &seqs, permit.as_ref()).await; + if res.is_err() { + break; + } + } + if let Err(e) = res { + fatal.set(format!("batcher: {e}")); + break; + } + } Some(BatcherMsg::FlushAll(reply)) => { if let Err(e) = flush_all(&mut tables, &out, &mut epoch, stats).await { fatal.set(format!("batcher barrier flush: {e}")); @@ -434,58 +464,60 @@ async fn handle_segment( ctx: &RowCtx<'_>, segment: PlacedSegment, ) -> Result<(), String> { - for mut chunk in segment.chunks { - ctx.stats - .insertbatch_rows_in - .fetch_add(chunk.rows as u64, Ordering::Relaxed); - let rt = RowTable { - rel: chunk.rel.clone(), - route: chunk.route.clone(), - }; - let slot = tables.slot(ctx, &rt)?; - let t = &mut tables.open[slot]; - if !t.enc.accepts(&chunk.buffers) { - // Same destination, other layout: seal what it holds first - emit_batch(t, ctx.out, ctx.stats).await?; - (t.enc, t.meta) = table_encoder(ctx, t.meta.table_key.clone(), &rt)?; - } - let rows = chunk.rows; - if t.enc - .append_block(&mut chunk.buffers, rows) - .map_err(|e| e.to_string())? - == Append::Full - { - t.seal_block()?; - t.enc - .append_block(&mut chunk.buffers, rows) - .map_err(|e| e.to_string())?; - } - appended( - tables, - ctx, - slot, - segment.seq, - rows as u64, - segment.permit.as_ref(), - ) - .await?; + for chunk in segment.chunks { + let seqs = [(segment.seq, chunk.rows as u64)]; + merge_chunk(tables, ctx, chunk, &seqs, segment.permit.as_ref()).await?; } Ok(()) } -/// Account `rows` of `seq` just appended to `slot`, sealing what limits +/// Merge one chunk whose rows belong to `seqs` into its destination's batch +async fn merge_chunk( + tables: &mut Tables, + ctx: &RowCtx<'_>, + mut chunk: EncodedChunk, + seqs: &[(u64, u64)], + permit: Option<&Arc>, +) -> Result<(), String> { + ctx.stats + .insertbatch_rows_in + .fetch_add(chunk.rows as u64, Ordering::Relaxed); + let rt = RowTable { + rel: chunk.rel.clone(), + route: chunk.route.clone(), + }; + let slot = tables.slot(ctx, &rt)?; + let t = &mut tables.open[slot]; + if !t.enc.accepts(&chunk.buffers) { + // Same destination, other layout: seal what it holds first + emit_batch(t, ctx.out, ctx.stats).await?; + (t.enc, t.meta) = table_encoder(ctx, t.meta.table_key.clone(), &rt)?; + } + let rows = chunk.rows; + if t.enc + .append_block(&mut chunk.buffers, rows) + .map_err(|e| e.to_string())? + == Append::Full + { + t.seal_block()?; + t.enc + .append_block(&mut chunk.buffers, rows) + .map_err(|e| e.to_string())?; + } + appended(tables, ctx, slot, seqs, permit).await +} + +/// Account rows of `seqs` just appended to `slot`, sealing what limits /// call for async fn appended( tables: &mut Tables, ctx: &RowCtx<'_>, slot: usize, - seq: u64, - rows: u64, + seqs: &[(u64, u64)], slice_permit: Option<&Arc>, ) -> Result<(), String> { let t = &mut tables.open[slot]; - // Rows sharing a permit arrive consecutively, so check only last entry. - // Sealing clears permit lists so each batch retains its own permits + // Rows sharing a permit arrive consecutively, so check only last entry if let Some(p) = slice_permit && t.slice_permits .last() @@ -493,9 +525,11 @@ async fn appended( { t.slice_permits.push(p.clone()); } - match t.seq_counts.last_mut() { - Some((s, c)) if *s == seq => *c += rows, - _ => t.seq_counts.push((seq, rows)), + for &(seq, rows) in seqs { + match t.seq_counts.last_mut() { + Some((s, c)) if *s == seq => *c += rows, + _ => t.seq_counts.push((seq, rows)), + } } if t.deadline.is_none() { t.deadline = Some(Instant::now() + ctx.cfg.flush_timeout); @@ -552,7 +586,6 @@ async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> R return Ok(()); } let per_seq = std::mem::take(&mut t.seq_counts); - let permits = std::mem::take(&mut t.slice_permits); // Separate pool avoids waiting on decoded payload held by this batch. // Oversized batch waits for whole compartment, then holds rest unmetered let mut encoded = out @@ -560,12 +593,13 @@ async fn emit_batch(t: &mut Table, out: &BatchOutput, stats: &EmitterStats) -> R .admit(bytes.min(out.budget.admission_max())) .await; encoded.settle(bytes as u64); + // Output pool covers these rows from here to insert ack + t.slice_permits.clear(); let batch = InsertBatch { meta: t.meta.clone(), blocks, n_rows, per_seq, - permits, encoded, }; out.sender @@ -1204,6 +1238,111 @@ mod tests { BatcherMsg::Segments(vec![segment]) } + /// Resident admission covers rows until their batch emits; the tail's + /// output pool holds them from there to insert ack + #[tokio::test] + async fn emit_releases_resident_admission() { + let resident = crate::budget::MemoryBudget::new(1 << 20); + let (tx, rx, fatal, handle) = + spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); + let (rel, route) = text_table("Nullable(String)"); + let mut seg = PlacedSegment::of_rows(0, &rel, &route, &[text_row(0, 0, "a").1]).unwrap(); + seg.permit = Some(Arc::new(resident.try_admit(4096).unwrap())); + tx.send(BatcherMsg::Segments(vec![seg])).await.unwrap(); + assert!(resident.resident_bytes() > 0); + let (reply_tx, reply_rx) = oneshot::channel(); + tx.send(BatcherMsg::FlushAll(reply_tx)).await.unwrap(); + reply_rx.await.unwrap(); + let batch = rx.recv().await.unwrap(); + assert_eq!( + resident.resident_bytes(), + 0, + "released while batch awaits insert" + ); + drop((batch, tx)); + handle.await.unwrap(); + assert!(!fatal.is_set()); + } + + /// One encode job over several commits' pending segments merges like + /// their placed segments, rows counted per seq; a job encoding other + /// than the planned rows fails the pipeline + #[tokio::test] + async fn encode_jobs_merge_like_segments() { + use crate::emit::pipeline::encode::{self, Encoders, PendingSegment}; + use crate::emit::pipeline::plan_spool::{PlanItem, PlanWriter, Segment, SegmentBody}; + let tmp = tempfile::tempdir().unwrap(); + let (rel, route) = text_table("Nullable(String)"); + let pending = |seq: u64, rows: &[(i32, &str)]| { + let path = tmp.path().join(format!("{seq}.plan")); + let mut w = PlanWriter::create(path, 1 << 20, 1 << 20).deferring(); + let table = w.table(&rel, 0, || Some(route.clone())); + for &(id, text) in rows { + w.push_cells(text_row(seq, id, text).1.decoded, table) + .unwrap(); + } + let mut rd = w.seal(0, 0).unwrap().replay().unwrap(); + let Some(PlanItem::Segment(Segment { + body: SegmentBody::Pending(records), + rows, + .. + })) = rd.next_item().unwrap() + else { + panic!("one pending segment"); + }; + PendingSegment { seq, rows, records } + }; + let commits: [(u64, &[(i32, &str)]); 3] = [ + (0, &[(0, "a"), (1, "")]), + (1, &[(2, "ccc")]), + (2, &[(3, "dd")]), + ]; + let mut wires = Vec::new(); + for jobbed in [false, true] { + let (tx, rx, fatal, handle) = + spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); + if jobbed { + let segments = commits.iter().map(|(seq, rows)| pending(*seq, rows)); + let job = encode::spawn( + segments.collect(), + None, + Encoders::default(), + Arc::default(), + ); + tx.send(BatcherMsg::Encoded(job)).await.unwrap(); + } else { + for (seq, rows) in commits { + let rows = rows.iter().map(|&(id, t)| text_row(seq, id, t)).collect(); + tx.send(segment(seq, &rel, &route, rows)).await.unwrap(); + } + } + drop(tx); + let b = rx.recv().await.unwrap(); + assert!(rx.recv().await.is_err(), "one batch"); + assert_eq!(b.per_seq, [(0, 2), (1, 1), (2, 1)]); + wires.push( + b.blocks[0] + .buffers + .iter() + .map(|c| format!("{:?}", c.wire())) + .collect::>(), + ); + handle.await.unwrap(); + assert!(!fatal.is_set()); + } + assert_eq!(wires[0], wires[1]); + + let (tx, _rx, fatal, handle) = + spawn_deadline_batcher(partial_block_cfg(Duration::from_secs(60)), None); + let mut short = pending(0, &[(0, "a")]); + short.rows = 2; + let job = encode::spawn(vec![short], None, Encoders::default(), Arc::default()); + tx.send(BatcherMsg::Encoded(job)).await.unwrap(); + handle.await.unwrap(); + let msg = fatal.message().unwrap(); + assert!(msg.contains("encoded 1 rows, planned 2"), "{msg}"); + } + fn text_row(seq: u64, id: i32, text: &str) -> (u64, CommittedTuple) { let mut r = tuple(seq, id); r.1.decoded.new.as_mut().unwrap().columns = vec![Some(build::text(text))]; diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index 60858b3c..6f4445bf 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -734,31 +734,25 @@ impl EncodeState { let mut view = TupleView::default(); for row in &mut slab.rows { if row.seq != self.seq { - self.seal()?; + self.seal(); self.seq = row.seq; } let patch = std::mem::take(&mut row.patch); let (rel, route) = &slab.tables[row.table as usize]; encode_row(&mut self.chunks, &mut view, &row.tuple, rel, route, patch)?; if self.chunks.due() { - self.seal()?; + self.seal(); } } Ok(()) } - fn seal(&mut self) -> Result<(), String> { + fn seal(&mut self) { let rows = self.chunks.rows(); if rows == 0 { - return Ok(()); + return; } - let chunks = self - .chunks - .take() - .map_err(|e| format!("bootstrap: encode: {e}"))? - .into_iter() - .map(|(_, c)| c) - .collect(); + let chunks = self.chunks.take().into_iter().map(|(_, c)| c).collect(); self.chunks.clear(); self.ready.push(PlacedSegment { seq: self.seq, @@ -766,7 +760,6 @@ impl EncodeState { permit: None, }); self.ready_rows += rows; - Ok(()) } } @@ -818,7 +811,7 @@ impl Encoder { /// seqs whose rows all shipped async fn ship(&mut self) -> Result<(), String> { let state = self.state.as_mut().expect("encode state home"); - state.seal()?; + state.seal(); // Every row it covered ships now; holding it longer stalls next acquire let permit = self.permit.take(); if !state.ready.is_empty() { diff --git a/src/emit/pipeline/encode.rs b/src/emit/pipeline/encode.rs new file mode 100644 index 00000000..c2a9475a --- /dev/null +++ b/src/emit/pipeline/encode.rs @@ -0,0 +1,103 @@ +//! Encode planned records off the commit path. +//! +//! Planning keeps routed records as WAL bytes, so the sink task that plans +//! every commit in order does no value encoding. Each outbox flush hands its +//! pending segments to one job on the runtime; the batcher awaits jobs in +//! channel order, so rows still merge in seq order. A job encodes one +//! table's rows from every segment into one chunk, tagging rows with their +//! seq, so small commits share column buffers instead of each sealing +//! their own. +//! +//! Planning already validated every record, so a job fails only on the +//! value codec. That failure is pipeline fatal; a control or truncate +//! behind it never applies, since both wait for earlier rows to be durable. + +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex, PoisonError}; + +use tokio::task::JoinHandle; + +use crate::budget::MemoryPermit; +use crate::emit::ch_emitter::{EmitterStats, EncodedChunk}; +use crate::emit::pipeline::plan_spool::{ChunkBuilder, EncoderCache, PendingRows, SeqRows}; + +/// Chunks one job encoded, each with the seqs of its rows +pub struct EncodedRun { + pub chunks: Vec<(EncodedChunk, SeqRows)>, + /// Admission covering the run, settled to encoded bytes + pub permit: Option, +} + +/// Pending segment placed under its seq +pub struct PendingSegment { + pub seq: u64, + /// Rows planning counted, which the ack collector expects + pub rows: u64, + pub records: PendingRows, +} + +/// Encoder caches jobs check out whole, one per concurrent job, so a route +/// compiles its table plan once per cache rather than per job +#[derive(Clone, Default)] +pub struct Encoders(Arc>>); + +impl Encoders { + /// Caches kept between jobs; a burst past this many concurrent jobs + /// compiles its own and drops them after + const KEPT: usize = 8; + + fn take(&self) -> EncoderCache { + self.0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .pop() + .unwrap_or_default() + } + + fn put(&self, cache: EncoderCache) { + let mut caches = self.0.lock().unwrap_or_else(PoisonError::into_inner); + if caches.len() < Self::KEPT { + caches.push(cache); + } + } +} + +/// Encode `segments` on the runtime +pub fn spawn( + segments: Vec, + permit: Option, + encoders: Encoders, + stats: Arc, +) -> JoinHandle> { + tokio::spawn(async move { + let run = encode(&segments, permit, &encoders); + if run.is_err() { + stats.plan_failures_encode.fetch_add(1, Ordering::Relaxed); + } + run + }) +} + +fn encode( + segments: &[PendingSegment], + mut permit: Option, + encoders: &Encoders, +) -> Result { + let mut b = ChunkBuilder::new(usize::MAX, usize::MAX).with_encoders(encoders.take()); + for s in segments { + b.tag(s.seq); + let rows = s.records.encode_into(&mut b).map_err(|e| e.to_string())?; + if rows as u64 != s.rows { + return Err(format!( + "seq {} encoded {rows} rows, planned {}", + s.seq, s.rows + )); + } + } + let chunks = b.take_runs(); + encoders.put(b.into_encoders()); + if let Some(p) = permit.as_mut() { + p.settle(chunks.iter().map(|(c, _)| c.allocated_bytes() as u64).sum()); + } + Ok(EncodedRun { chunks, permit }) +} diff --git a/src/emit/pipeline/mod.rs b/src/emit/pipeline/mod.rs index d4808383..f0f3ced6 100644 --- a/src/emit/pipeline/mod.rs +++ b/src/emit/pipeline/mod.rs @@ -13,6 +13,7 @@ pub mod ack; pub mod batcher; pub mod bootstrap; +pub mod encode; pub mod inserter; pub mod plan_spool; pub mod planner; diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index 962aec85..5c225d14 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -1,9 +1,11 @@ //! Plan one committed transaction before executing it. //! -//! Encode routed rows into per-table column chunks while planning. Seal open -//! chunks into a segment at row and byte limits and before every control or -//! truncate; each segment dispatches as one seq. Keep chunks in memory until -//! their size exceeds memory limit, then write them to a spool file. Keep +//! Seal routed rows into a segment at row and byte limits and before every +//! control or truncate; each segment dispatches as one seq. A deferring +//! writer keeps validated records as WAL bytes so encoding runs once +//! dispatched, off the commit path; others encode per-table column chunks +//! while planning. Keep segments in memory until their size exceeds memory +//! limit, then encode and write them to a spool file. Keep //! table metadata, controls, and truncates in memory, assuming control state //! stays small as in live transaction buffer. Count and discard rows without //! mappings, but keep their truncates. On planning failure, delete spool file @@ -27,13 +29,13 @@ use thiserror::Error; use crate::ch::EmitterError; use crate::decode::datum::{Datum, DatumRef}; -use crate::decode::heap_decoder::{HeapOp, TupleView, inflate, missing_datum}; +use crate::decode::heap_decoder::{HeapOp, TupleView, inflate, missing_datum, view_heap_record}; use crate::emit::ch_emitter::{ Append, EncodedChunk, OP_DELETE, OP_INSERT, OP_UPDATE, RowStamp, TableEncoder, TablePlan, }; use crate::emit::route::RouteSnapshot; use crate::schema::RelDescriptor; -use crate::xact::spill::{self, Cursor, IO_CHUNK, SpillError, Truncate}; +use crate::xact::spill::{self, Cursor, DescribedInput, IO_CHUNK, SpillError, Truncate}; use crate::xact::xact_buffer::{DrainEntry, ToastRowBatch}; use foldhash::HashMap; @@ -75,13 +77,109 @@ pub struct PlannedTruncate { /// Rows sealed together; executor dispatches one seq per segment pub struct Segment { - pub chunks: Vec, + pub body: SegmentBody, /// Destination rows, delete markers for key changes included pub rows: u64, - /// Allocated column bytes, the admission a dispatched segment holds + /// Allocated column bytes or held record bytes, the admission a + /// dispatched segment holds pub bytes: usize, } +pub enum SegmentBody { + Encoded(Vec), + /// Validated records still WAL bytes, encoded once dispatched + Pending(PendingRows), +} + +impl Segment { + #[cfg(any(test, feature = "test-support"))] + /// [`Self::into_chunks`], panicking on a codec failure + pub fn chunks(self) -> Vec { + self.into_chunks().expect("segment encodes") + } + + /// Column chunks, encoding pending records here + pub fn into_chunks(self) -> std::result::Result, EmitterError> { + match self.body { + SegmentBody::Encoded(chunks) => Ok(chunks), + SegmentBody::Pending(rows) => { + let mut b = ChunkBuilder::new(usize::MAX, usize::MAX); + rows.encode_into(&mut b)?; + Ok(b.take().into_iter().map(|(_, c)| c).collect()) + } + } + } +} + +/// Routed heap record planned without encoding: routed, validated and +/// detoasted, so encoding can only fail on the value codec +pub struct PendingHeap { + table: u32, + route: Arc, + input: Box, + /// Resolved external values per image as `[new, old]`, empty when no + /// image has any + patches: Vec<[Vec<(usize, Datum)>; 2]>, +} + +impl PendingHeap { + /// Encode every image into `b` under `key`, returning rows appended + fn encode_into( + &self, + b: &mut ChunkBuilder, + key: K, + commit_ts: i64, + ) -> std::result::Result { + let rel = &self.input.descriptor; + let lsn = self.input.input.source_lsn; + let images = view_heap_record(self.input.input.record_bytes(), rel) + .map_err(|e| EmitterError::Type(format!("record at {lsn:#X}: {e}")))?; + let mut rows = 0; + for (i, h) in images.iter().enumerate() { + let (new_patch, old_patch) = self + .patches + .get(i) + .map_or((&[][..], &[][..]), |[n, o]| (&n[..], &o[..])); + let change = ViewChange { + op: h.op, + xid: self.input.input.xid, + source_lsn: lsn, + new: h.new.as_ref(), + old: h.old.as_ref(), + new_patch, + old_patch, + }; + rows += b.push_views(key, rel, &self.route, change, commit_ts)?; + } + Ok(rows) + } +} + +/// One segment's pending records +#[derive(Default)] +pub struct PendingRows { + commit_ts: i64, + heaps: Vec, +} + +impl PendingRows { + /// Encode into `b`, keying tables by relation and route identity so + /// several segments' rows of one table share a chunk. Returns rows + /// appended + pub fn encode_into( + &self, + b: &mut ChunkBuilder<(usize, usize)>, + ) -> std::result::Result { + self.heaps.iter().try_fold(0, |rows, h| { + let key = ( + Arc::as_ptr(&h.input.descriptor) as usize, + Arc::as_ptr(&h.route) as usize, + ); + Ok(rows + h.encode_into(b, key, self.commit_ts)?) + }) + } +} + #[derive(Debug, Error)] pub enum PlanSpoolError { #[error("plan spool io: {0}")] @@ -145,12 +243,10 @@ impl Drop for PlanFile { /// One plan step in drain order enum Step { - /// `chunks` stay `None` once written to the spool file Segment { - count: usize, rows: u64, bytes: usize, - chunks: Option>, + body: StepBody, }, Control { event: DrainEntry, @@ -162,14 +258,31 @@ enum Step { }, } +enum StepBody { + Pending(PendingRows), + Encoded(Vec<(u32, EncodedChunk)>), + /// `frames` chunks written to the spool file + Spooled { + frames: usize, + }, +} + /// Per-table encoder of the open segment struct OpenTable { key: K, rel: Arc, route: Arc, enc: TableEncoder, + /// `(seq, rows)` of the open chunk, in append order, when tagging + seqs: SeqRows, } +/// `(seq, rows)` runs one chunk carries, in append order +pub type SeqRows = Vec<(u64, u64)>; + +/// Chunk sealed out of a [`ChunkBuilder`], with the seqs its rows belong to +type Taken = (K, EncodedChunk, SeqRows); + /// Empty encoders kept across plans, so a route compiles its table plan /// once rather than per transaction. One per relation name: a new /// descriptor or route version replaces the old @@ -215,7 +328,9 @@ pub struct ChunkBuilder { open: Vec>, encoders: EncoderCache, /// Chunks of the open segment sealed early at an oracle frame limit - sealed: Vec<(K, EncodedChunk)>, + sealed: Vec>, + /// Seq rows append under, see [`Self::tag`] + seq: Option, rows: usize, segment_rows: usize, segment_bytes: usize, @@ -229,6 +344,7 @@ impl ChunkBuilder { open: Vec::new(), encoders: EncoderCache::default(), sealed: Vec::new(), + seq: None, rows: 0, segment_rows: segment_rows.max(1), segment_bytes: segment_bytes.max(1), @@ -241,6 +357,12 @@ impl ChunkBuilder { self } + /// Tag rows appended from here on with `seq`, so one chunk can carry + /// rows of several seqs; see [`Self::take_runs`] + pub fn tag(&mut self, seq: u64) { + self.seq = Some(seq); + } + /// Rows in the open segment pub fn rows(&self) -> usize { self.rows @@ -256,7 +378,7 @@ impl ChunkBuilder { + self .sealed .iter() - .map(|(_, c)| c.allocated_bytes()) + .map(|(_, c, _)| c.allocated_bytes()) .sum::() } @@ -281,15 +403,23 @@ impl ChunkBuilder { rel: rel.clone(), route: route.clone(), enc, + seqs: SeqRows::new(), }); self.open.len() - 1 } }; if row(&mut self.open[i].enc)? == Append::Full { - let chunk = self.take_chunk(i)?; + let chunk = self.take_chunk(i); self.sealed.push(chunk); row(&mut self.open[i].enc)?; } + if let Some(seq) = self.seq { + let seqs = &mut self.open[i].seqs; + match seqs.last_mut() { + Some((s, n)) if *s == seq => *n += 1, + _ => seqs.push((seq, 1)), + } + } self.rows += 1; Ok(()) } @@ -317,10 +447,7 @@ impl ChunkBuilder { leaf, }; let mut rows = 0; - if !route.drops_deletes() - && let (Some(old), Some(new)) = (change.old, change.new) - && key_moved(rel, &change, old, new) - { + if let Some(old) = moved_key(rel, route, &change) { let stamp = stamp(change.source_lsn - 1, OP_DELETE); self.append(key, rel, route, |enc| { enc.append_view(rel, &old.datums, change.old_patch, &stamp) @@ -339,28 +466,44 @@ impl ChunkBuilder { Ok(rows + 1) } - fn take_chunk(&mut self, i: usize) -> std::result::Result<(K, EncodedChunk), EmitterError> { + fn take_chunk(&mut self, i: usize) -> Taken { let o = &mut self.open[i]; - let (buffers, rows) = o.enc.take_block()?; + let (buffers, rows) = o.enc.take_block(); let chunk = EncodedChunk { rel: o.rel.clone(), route: o.route.clone(), buffers, rows, }; - Ok((o.key, chunk)) + (o.key, chunk, std::mem::take(&mut o.seqs)) } /// Seal the open segment's chunks, early-sealed ones first - pub fn take(&mut self) -> std::result::Result, EmitterError> { + fn take_tagged(&mut self) -> Vec> { let mut chunks = std::mem::take(&mut self.sealed); for i in 0..self.open.len() { if self.open[i].enc.rows > 0 { - chunks.push(self.take_chunk(i)?); + chunks.push(self.take_chunk(i)); } } self.rows = 0; - Ok(chunks) + chunks + } + + /// Seal the open segment's chunks, early-sealed ones first + pub fn take(&mut self) -> Vec<(K, EncodedChunk)> { + self.take_tagged() + .into_iter() + .map(|(k, c, _)| (k, c)) + .collect() + } + + /// [`Self::take`] with each chunk's rows per [`Self::tag`]ged seq + pub fn take_runs(&mut self) -> Vec<(EncodedChunk, SeqRows)> { + self.take_tagged() + .into_iter() + .map(|(_, c, seqs)| (c, seqs)) + .collect() } /// Park open encoders, emptied by [`Self::take`], so later keys start @@ -400,6 +543,12 @@ pub struct PlanWriter { /// second live table appears, so single-relation plans never hash table_ids: Option>, chunks: ChunkBuilder, + /// Keep routed records as WAL bytes until dispatch, see [`Self::defers`] + defer: bool, + /// Open segment's records when deferring + pending: Vec, + pending_rows: usize, + pending_bytes: usize, commit_ts: i64, steps: Vec, row_batches: Vec, @@ -423,6 +572,10 @@ impl PlanWriter { last_table: 0, table_ids: None, chunks: ChunkBuilder::new(usize::MAX, usize::MAX), + defer: false, + pending: Vec::new(), + pending_rows: 0, + pending_bytes: 0, commit_ts: 0, steps: Vec::new(), row_batches: Vec::new(), @@ -445,6 +598,19 @@ impl PlanWriter { self } + /// Plan routed records without encoding them, so encoding runs once + /// dispatched instead of on the commit path + pub fn deferring(mut self) -> Self { + self.defer = true; + self + } + + /// Whether the next record should go through [`Self::push_pending`]: + /// only memory-resident plans defer, a spilled one writes chunks + pub fn defers(&self) -> bool { + self.defer && self.file.is_none() + } + /// Reuse encoders an earlier plan returned through [`Self::seal_keeping`] pub fn with_encoders(mut self, encoders: EncoderCache) -> Self { self.chunks.encoders = encoders; @@ -523,6 +689,8 @@ impl PlanWriter { &input.descriptor, ) .expect("formed record views"); + let defer = self.defers(); + let mut rows = 0; for h in &images { let change = ViewChange { op: h.op, @@ -533,7 +701,15 @@ impl PlanWriter { new_patch: &[], old_patch: &[], }; - self.push_views(change, table)?; + if defer { + rows += self.count_change(&change, table)?; + } else { + self.push_views(change, table)?; + } + } + drop(images); + if defer { + self.push_pending(table, Box::new(input), Vec::new(), rows)?; } Ok(()) } @@ -550,13 +726,68 @@ impl PlanWriter { self.pushed(rows) } - /// Count `rows` just encoded, none for a dropped delete, and seal a due - /// segment - fn pushed(&mut self, rows: usize) -> Result<()> { + /// Count one change [`Self::push_pending`] will carry, validating its + /// operation. Returns the rows it encodes to + pub fn count_change(&mut self, change: &ViewChange<'_, '_>, table: u32) -> Result { + let t = &mut self.tables[table as usize]; + t.heaps += 1; + let route = t.route.as_ref().expect("routed table"); + let rows = if route.drops_deletes() && change.op == HeapOp::Delete { + 0 + } else { + op_code(change.op)?; + 1 + usize::from(moved_key(&t.descriptor, route, change).is_some()) + }; + self.count(rows); + Ok(rows) + } + + /// Carry a record whose images [`Self::count_change`] counted as `rows` + pub fn push_pending( + &mut self, + table: u32, + input: Box, + patches: Vec<[Vec<(usize, Datum)>; 2]>, + rows: usize, + ) -> Result<()> { + if rows == 0 { + return Ok(()); + } + let route = self.tables[table as usize].route.clone(); + self.pending_bytes += size_of::() + + input.approx_bytes() + + patches + .iter() + .flatten() + .flatten() + .map(|(_, d)| size_of::<(usize, Datum)>() + d.approx_bytes()) + .sum::(); + self.pending_rows += rows; + self.pending.push(PendingHeap { + table, + route: route.expect("routed table"), + input, + patches, + }); + if self.pending_rows >= self.chunks.segment_rows + || self.pending_bytes >= self.chunks.segment_bytes + { + self.seal_segment()?; + } + Ok(()) + } + + /// Count `rows` of one change, none for a dropped delete + fn count(&mut self, rows: usize) { if rows == 0 { self.deletes_discarded += 1; } self.routed_rows += rows as u64; + } + + /// Count `rows` just encoded and seal a due segment + fn pushed(&mut self, rows: usize) -> Result<()> { + self.count(rows); if self.chunks.due() { self.seal_segment()?; } @@ -565,33 +796,61 @@ impl PlanWriter { /// Close the open segment into a step, spilling once memory runs out fn seal_segment(&mut self) -> Result<()> { - let chunks = self.chunks.take()?; - if chunks.is_empty() { - return Ok(()); + let (rows, bytes, body) = if self.pending.is_empty() { + let chunks = self.chunks.take(); + if chunks.is_empty() { + return Ok(()); + } + let rows = chunks.iter().map(|(_, c)| c.rows as u64).sum(); + let bytes = chunks.iter().map(|(_, c)| c.allocated_bytes()).sum(); + (rows, bytes, StepBody::Encoded(chunks)) + } else { + let rows = std::mem::take(&mut self.pending_rows) as u64; + let bytes = std::mem::take(&mut self.pending_bytes); + let records = PendingRows { + commit_ts: self.commit_ts, + heaps: std::mem::take(&mut self.pending), + }; + (rows, bytes, StepBody::Pending(records)) + }; + let mut step = Step::Segment { rows, bytes, body }; + if self.file.is_none() && self.bytes + bytes as u64 <= self.mem_max.min(self.cap) { + self.bytes += bytes as u64; + } else { + self.spill()?; + self.spool_step(&mut step)?; } - let rows = chunks.iter().map(|(_, c)| c.rows as u64).sum(); - let bytes = chunks - .iter() - .map(|(_, c)| c.allocated_bytes()) - .sum::(); - let count = chunks.len(); - let chunks = - if self.file.is_none() && self.bytes + bytes as u64 <= self.mem_max.min(self.cap) { - self.bytes += bytes as u64; - Some(chunks) - } else { - self.spill()?; - for (table, c) in &chunks { - self.write_chunk(*table, c)?; + self.steps.push(step); + Ok(()) + } + + /// Write a resident segment's chunks to the spool file, encoding + /// pending records first + fn spool_step(&mut self, step: &mut Step) -> Result<()> { + let Step::Segment { bytes, body, .. } = step else { + return Ok(()); + }; + let chunks = match std::mem::replace(body, StepBody::Spooled { frames: 0 }) { + StepBody::Encoded(chunks) => chunks, + StepBody::Pending(records) => { + for h in &records.heaps { + h.encode_into(&mut self.chunks, h.table, records.commit_ts)?; } - None - }; - self.steps.push(Step::Segment { - count, - rows, - bytes, - chunks, - }); + let chunks = self.chunks.take(); + *bytes = chunks.iter().map(|(_, c)| c.allocated_bytes()).sum(); + chunks + } + spooled @ StepBody::Spooled { .. } => { + *body = spooled; + return Ok(()); + } + }; + for (table, c) in &chunks { + self.write_chunk(*table, c)?; + } + *body = StepBody::Spooled { + frames: chunks.len(), + }; Ok(()) } @@ -607,16 +866,7 @@ impl PlanWriter { self.file = Some((out, PlanFile(path))); self.bytes = 4; let mut steps = std::mem::take(&mut self.steps); - let written = steps.iter_mut().try_for_each(|step| { - if let Step::Segment { chunks, .. } = step - && let Some(chunks) = chunks.take() - { - chunks - .iter() - .try_for_each(|(table, c)| self.write_chunk(*table, c))?; - } - Ok::<_, PlanSpoolError>(()) - }); + let written = steps.iter_mut().try_for_each(|step| self.spool_step(step)); self.steps = steps; written } @@ -771,6 +1021,19 @@ fn key_moved<'v>( } } +/// Old image when `change` moves its key and the route keeps deletes: the +/// change encodes as a delete marker plus the new row +fn moved_key<'v, 'a>( + rel: &RelDescriptor, + route: &RouteSnapshot, + change: &ViewChange<'v, 'a>, +) -> Option<&'v TupleView<'a>> { + let (Some(old), Some(new)) = (change.old, change.new) else { + return None; + }; + (!route.drops_deletes() && key_moved(rel, change, old, new)).then_some(old) +} + /// Stored-byte inequality, as PostgreSQL's `datumIsEqual` judges index /// keys; absent never differs. Floats compare by value, so NaN differs from /// itself and signed zeros match @@ -933,30 +1196,26 @@ impl PlanReader { Ok(Some(match step { Step::Control { event, row_idx } => PlanItem::Control { event, row_idx }, Step::Truncate { truncate, upto } => PlanItem::Truncate { truncate, upto }, - Step::Segment { - count, - rows, - bytes, - chunks, - } => { - let chunks = match chunks { - Some(chunks) => chunks.into_iter().map(|(_, c)| c).collect(), - None => { + Step::Segment { rows, bytes, body } => { + let body = match body { + StepBody::Pending(records) => SegmentBody::Pending(records), + StepBody::Encoded(chunks) => { + SegmentBody::Encoded(chunks.into_iter().map(|(_, c)| c).collect()) + } + StepBody::Spooled { frames } => { let Some(f) = self.input.as_mut() else { return Err(PlanSpoolError::Format { detail: "spilled segment without spool file".into(), }); }; - (0..count) - .map(|_| read_frame(f, &self.tables)) - .collect::>()? + SegmentBody::Encoded( + (0..frames) + .map(|_| read_frame(f, &self.tables)) + .collect::>()?, + ) } }; - PlanItem::Segment(Segment { - chunks, - rows, - bytes, - }) + PlanItem::Segment(Segment { body, rows, bytes }) } })) } @@ -1003,7 +1262,7 @@ fn read_frame(f: &mut FileInput, tables: &[PlanTable]) -> Result { mod tests { use super::*; use crate::decode::datum::{Datum, build}; - use crate::decode::heap_decoder::{DecodedHeap, DecodedTuple}; + use crate::decode::heap_decoder::{CommittedTuple, DecodedHeap, DecodedTuple}; use crate::emit::ch_emitter::Wire; use crate::mapping::{ColumnMapping, TableMapping, TableTarget}; use crate::schema::{INT4OID, RelAttr, RelName, SchemaEvent}; @@ -1243,11 +1502,12 @@ mod tests { order.push(match item { PlanItem::Control { event, .. } => format!("{event:?}"), PlanItem::Segment(s) => { - for c in &s.chunks { + let chunks = s.chunks(); + for c in &chunks { assert!(Arc::ptr_eq(&c.rel, &d1), "descriptor survives plan"); assert!(Arc::ptr_eq(&c.route, &r)); } - format!("{:?}", rows(&s.chunks)) + format!("{:?}", rows(&chunks)) } PlanItem::Truncate { truncate: t, .. } => { assert!(rd.tables[t.table as usize].route.is_none()); @@ -1342,8 +1602,8 @@ mod tests { let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() else { panic!("one segment"); }; - assert_eq!(rows(&s.chunks), want); assert_eq!(s.rows, want.len() as u64); + assert_eq!(rows(&s.chunks()), want); assert_eq!(rd.deletes_discarded, discarded); } } @@ -1367,11 +1627,78 @@ mod tests { let mut seen = Vec::new(); while let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() { assert_eq!(s.rows, 2); - seen.extend(rows(&s.chunks).into_iter().map(|r| r.0.unwrap())); + seen.extend(rows(&s.chunks()).into_iter().map(|r| r.0.unwrap())); } assert_eq!(seen, (0..64).collect::>()); } + /// Deferred records replay pending and encode to the rows inline + /// planning writes; a spill encodes them into frames + #[test] + fn deferred_records_encode_like_inline() { + let tmp = tempfile::tempdir().unwrap(); + let d = descriptor(16500); + let r = route(); + let plan = |name: &str, defer: bool, mem_max: u64| { + let mut w = PlanWriter::create(tmp.path().join(name), 1 << 20, mem_max) + .with_segment_limits(2, usize::MAX) + .for_commit(42); + if defer { + w = w.deferring(); + } + for v in 0..5 { + push(&mut w, &d, heap(&d, 100 + v as u64, v), Some(r.clone())).unwrap(); + } + let mut rd = w.seal(0x2000, 42).unwrap().replay().unwrap(); + let (mut out, mut pending) = (Vec::new(), 0); + while let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() { + pending += usize::from(matches!(s.body, SegmentBody::Pending(_))); + let want = s.rows; + let got = rows(&s.chunks()); + assert_eq!(got.len() as u64, want); + out.push(got); + } + (out, pending) + }; + let (inline, none) = plan("inline", false, 1 << 20); + assert_eq!((inline.len(), none), (3, 0)); + assert_eq!(plan("deferred", true, 1 << 20), (inline.clone(), 3)); + assert_eq!(plan("spilled", true, 0), (inline, 0)); + } + + /// Rows tagged with seqs share one chunk per table, each run counted + #[test] + fn tagged_rows_share_chunk_per_table() { + let (d1, d2, r) = (descriptor(1), descriptor(2), route()); + let mut b = ChunkBuilder::new(usize::MAX, usize::MAX); + for (seq, rel) in [(5, &d1), (5, &d2), (6, &d1), (6, &d1), (7, &d2)] { + b.tag(seq); + let h = heap(rel, 100 + seq, seq as i32); + b.append(Arc::as_ptr(rel) as usize, rel, &r, |enc| { + enc.append_routed( + &CommittedTuple { + decoded: h.clone(), + commit_ts: 42, + commit_lsn: 0, + }, + &r.mapping, + OP_INSERT, + None, + ) + }) + .unwrap(); + } + let runs: Vec<_> = b + .take_runs() + .into_iter() + .map(|(c, seqs)| (c.rel.oid, c.rows, seqs)) + .collect(); + assert_eq!( + runs, + [(1, 3, vec![(5, 1), (6, 2)]), (2, 2, vec![(5, 1), (7, 1)])] + ); + } + /// Sealing returns emptied encoders that serve the next plan under the /// same route; another route version compiles its own #[test] @@ -1401,7 +1728,7 @@ mod tests { let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() else { panic!("one segment"); }; - assert_eq!(rows(&s.chunks), [(Some(i as i32), 100 + i as u64, 7, 0)]); + assert_eq!(rows(&s.chunks()), [(Some(i as i32), 100 + i as u64, 7, 0)]); } } @@ -1428,7 +1755,7 @@ mod tests { let Some(PlanItem::Segment(s)) = rd.next_item().unwrap() else { panic!("one segment"); }; - s.chunks + s.chunks() .iter() .flat_map(|c| c.buffers.iter().map(|b| format!("{:?}", b.wire()))) .collect::>() diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index f8240104..6db16017 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -29,8 +29,9 @@ //! reconstruction path exists and PG only elides below //! `wal_level=logical`, so a routed partial fails the plan //! - needed toast values resolve deterministically → detoast at planning -//! - mapped output columns have deterministic codecs → plan-time codec -//! policy layers onto this stage (deterministic-codec phase) +//! - mapped output columns encode → not checked here: memory-resident plans +//! carry records to dispatch, where an encode job runs the codec and a +//! failure is pipeline fatal; spilled plans encode at planning //! - route snapshot complete → `RouteSnapshot::freeze` populates every //! field by construction; `None` is the counted unmapped discard, which //! skips detoast and codec validation entirely @@ -40,7 +41,7 @@ use std::sync::Arc; use crate::decode::datum::Datum; -use crate::decode::heap_decoder::view_heap_record; +use crate::decode::heap_decoder::{HeapImages, TupleView, view_heap_record}; use crate::emit::pipeline::plan_spool::{ DEFAULT_PLAN_MEM_MAX, EncoderCache, PlanPath, PlanSpoolError, PlanWriter, SealedPlan, ViewChange, @@ -149,6 +150,7 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { ) -> Self { Self { writer: PlanWriter::create(path, disk_cap, DEFAULT_PLAN_MEM_MAX) + .deferring() .with_segment_limits(segment.0, segment.1) .for_commit(commit_ts) .with_encoders(view.take_encoders()), @@ -209,7 +211,7 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { self.writer.discard(table, u64::from(input.input.tuples)); continue; } - self.plan_views(&input, table, spool, &ref_maps).await?; + self.plan_views(input, table, spool, &ref_maps).await?; } } } @@ -218,51 +220,113 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { Ok(()) } - /// Encode an input from tuple images in place, external values resolved - /// into per-image patches + /// Plan an input from tuple images in place. Records without external + /// values plan synchronously; the rest detoast into per-image patches async fn plan_views( &mut self, - input: &DescribedInput, + input: Box, table: u32, spool: Option<&BodySpoolFile>, ref_maps: &[&ChunkRefMap], ) -> Result<(), PlanError> { - let rel = &input.descriptor; - let lsn = input.input.source_lsn; - let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { - XactBufferError::OrdinaryFailClosed { - relid: rel.oid, - lsn, - rm: input.input.rm, - info: input.input.info, - reason: FailClosedReason::Malformed(e.to_string()), - } - })?; + let Some(input) = self.plan_inline(input, table)? else { + return Ok(()); + }; + // Boxed: detoast state would bloat every commit future + Box::pin(self.plan_detoasted(input, table, spool, ref_maps)).await + } + + /// Plan `input` unless an image holds an external value, handing it back + fn plan_inline( + &mut self, + input: Box, + table: u32, + ) -> Result>, PlanError> { + let images = images(&input)?; + if images + .iter() + .any(|h| !external_sites([h.new.as_ref(), h.old.as_ref()]).is_empty()) + { + drop(images); + return Ok(Some(input)); + } + let rows = self.push_changes(&input, table, &images, &[])?; + drop(images); + self.carry(input, table, Vec::new(), rows)?; + Ok(None) + } + + async fn plan_detoasted( + &mut self, + input: Box, + table: u32, + spool: Option<&BodySpoolFile>, + ref_maps: &[&ChunkRefMap], + ) -> Result<(), PlanError> { + let images = images(&input)?; + let (db, lsn) = (input.descriptor.rfn.db_node, input.input.source_lsn); + let mut patches = Vec::with_capacity(images.len()); for h in &images { - if h.new.as_ref().is_some_and(|t| t.partial) { - return Err(PlanError::PartialUpdate { lsn }); - } - let mut patches: [Vec<(usize, Datum)>; 2] = Default::default(); + let mut patch: [Vec<(usize, Datum)>; 2] = Default::default(); let sites = external_sites([h.new.as_ref(), h.old.as_ref()]); if !sites.is_empty() { - let db = rel.rfn.db_node; // Value permit drops once bytes land in the plan let (values, _permit) = detoast_sites(&sites, db, lsn, spool, ref_maps, self.resolver).await?; for (site, v) in sites.iter().zip(values) { - patches[site.image].push((site.idx, v)); + patch[site.image].push((site.idx, v)); } } + patches.push(patch); + } + let rows = self.push_changes(&input, table, &images, &patches)?; + drop(images); + self.carry(input, table, patches, rows) + } + + /// Plan each image as one change with its patches. A deferring writer + /// only counts them, returning rows the record encodes to + fn push_changes( + &mut self, + input: &DescribedInput, + table: u32, + images: &[HeapImages>], + patches: &[[Vec<(usize, Datum)>; 2]], + ) -> Result, PlanError> { + let defer = self.writer.defers(); + let mut rows = 0; + for (i, h) in images.iter().enumerate() { + let (new_patch, old_patch) = patches + .get(i) + .map_or((&[][..], &[][..]), |[n, o]| (&n[..], &o[..])); let change = ViewChange { op: h.op, xid: input.input.xid, - source_lsn: lsn, + source_lsn: input.input.source_lsn, new: h.new.as_ref(), old: h.old.as_ref(), - new_patch: &patches[0], - old_patch: &patches[1], + new_patch, + old_patch, }; - self.writer.push_views(change, table)?; + if defer { + rows += self.writer.count_change(&change, table)?; + } else { + self.writer.push_views(change, table)?; + } + } + Ok(defer.then_some(rows)) + } + + /// Carry a record [`Self::push_changes`] counted for deferred encoding + fn carry( + &mut self, + input: Box, + table: u32, + patches: Vec<[Vec<(usize, Datum)>; 2]>, + rows: Option, + ) -> Result<(), PlanError> { + if let Some(rows) = rows { + self.writer.push_pending(table, input, patches, rows)?; } Ok(()) } @@ -280,6 +344,31 @@ impl<'a, V: PlanRouteView> Planner<'a, V> { } } +/// Tuple images of `input`, failing closed on a malformed record or an +/// update whose new image elides a predecessor prefix or suffix +fn images(input: &DescribedInput) -> Result, PlanError> { + let rel = &input.descriptor; + let lsn = input.input.source_lsn; + let images = view_heap_record(input.input.record_bytes(), rel).map_err(|e| { + XactBufferError::OrdinaryFailClosed { + relid: rel.oid, + lsn, + rm: input.input.rm, + info: input.input.info, + reason: FailClosedReason::Malformed(e.to_string()), + } + })?; + if images + .iter() + .any(|h| h.new.as_ref().is_some_and(|t| t.partial)) + { + return Err(PlanError::PartialUpdate { lsn }); + } + Ok(images) +} + +type ImageViews<'a> = smallvec::SmallVec<[HeapImages>; 1]>; + #[cfg(test)] mod tests { use super::*; @@ -557,7 +646,7 @@ mod tests { match item { PlanItem::Control { .. } => order.push("e".to_string()), PlanItem::Segment(s) => { - for c in &s.chunks { + for c in &s.chunks() { order.extend(lsns(c, 0).iter().map(|lsn| format!("h{lsn}r"))); } } @@ -746,11 +835,10 @@ mod tests { let PlanItem::Segment(s) = item else { panic!("segments only"); }; - assert_eq!( - s.rows, - s.chunks.iter().map(|c| c.rows() as u64).sum::() - ); rows += s.rows; + let want = s.rows; + let encoded = s.chunks().iter().map(|c| c.rows() as u64).sum::(); + assert_eq!(want, encoded); } drop(rd); assert_eq!(rows, ROWS as u64); @@ -1011,7 +1099,7 @@ mod tests { let mut out = Vec::new(); while let Some(item) = rd.next_item().unwrap() { let PlanItem::Segment(s) = item else { continue }; - for c in &s.chunks { + for c in &s.chunks() { let crate::emit::ch_emitter::Wire::NullableString { offsets, data, diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index 37e929ca..9522c441 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -48,7 +48,10 @@ use crate::config::ResolvedConfig; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::{AckHandle, BufferedRowSender, Publish, SeqAlloc}; use crate::emit::pipeline::batcher::{BatcherMsg, PlacedSegment}; -use crate::emit::pipeline::plan_spool::{EncoderCache, PlanItem, PlanPath, SealedPlan, Segment}; +use crate::emit::pipeline::encode::{self, Encoders, PendingSegment}; +use crate::emit::pipeline::plan_spool::{ + EncoderCache, PlanItem, PlanPath, SealedPlan, Segment, SegmentBody, +}; use crate::emit::pipeline::planner::{PlanRouteView, Planner, drain_reason}; use crate::emit::route::{RouteSnapshot, RowPolicy, route_via}; use crate::mapping::{MappingSnapshot, TableMapping}; @@ -159,8 +162,15 @@ struct Outbox { /// with [`Self::staged`] seqs: SeqAlloc, /// Placed rows not yet on the batcher channel, shipped per worker batch - /// so small commits share one channel hop and wakeup + /// so small commits share one channel hop and wakeup. Encoded and + /// pending segments never stage together, keeping seq order staged: Vec, + /// Pending segments, shipped as one encode job + pending: Vec, + /// Admission of `pending`, riding their job + pending_permit: Option, + encoders: Encoders, + stats: Arc, staged_rows: usize, staged_bytes: usize, /// Global resident-payload pool; slice and chunk generation admission @@ -169,6 +179,43 @@ struct Outbox { } impl Outbox { + /// Stage `segment` under `seq`, first shipping staged segments of the + /// other kind so the batcher sees seqs in order + async fn stage( + &mut self, + seq: u64, + segment: Segment, + permit: Option, + ) -> Result<(), SinkError> { + let other = match segment.body { + SegmentBody::Pending(_) => !self.staged.is_empty(), + SegmentBody::Encoded(_) => !self.pending.is_empty(), + }; + if other { + Box::pin(self.flush_pending()).await?; + } + match segment.body { + SegmentBody::Pending(records) => { + self.pending.push(PendingSegment { + seq, + rows: segment.rows, + records, + }); + if let Some(p) = permit { + crate::budget::absorb_into(&mut self.pending_permit, p); + } + } + SegmentBody::Encoded(chunks) => self.staged.push(PlacedSegment { + seq, + chunks, + permit: permit.map(Arc::new), + }), + } + self.staged_rows += segment.rows as usize; + self.staged_bytes += segment.bytes; + Ok(()) + } + /// Admission permit for `bytes`. Staged rows hold permits only shipping /// releases, so ship before waiting on the pool async fn admit( @@ -181,12 +228,16 @@ impl Outbox { if let Some(p) = budget.try_admit(bytes) { return Ok(Some(p)); } - self.flush_pending().await?; - tokio::select! { - biased; - msg = self.fatal.wait() => Err(SinkError::Other(msg)), - p = budget.admit(bytes) => Ok(Some(p)), - } + // Boxed: waiting is rare, inline it would bloat every commit future + Box::pin(async { + self.flush_pending().await?; + tokio::select! { + biased; + msg = self.fatal.wait() => Err(SinkError::Other(msg)), + p = budget.admit(bytes) => Ok(Some(p)), + } + }) + .await } } @@ -196,16 +247,24 @@ impl BufferedRowSender for Outbox { } async fn send_rows(&mut self) -> Result<(), SinkError> { - if self.staged.is_empty() { + let msg = if !self.pending.is_empty() { + BatcherMsg::Encoded(encode::spawn( + std::mem::take(&mut self.pending), + self.pending_permit.take(), + self.encoders.clone(), + self.stats.clone(), + )) + } else if !self.staged.is_empty() { + BatcherMsg::Segments(std::mem::take(&mut self.staged)) + } else { return Ok(()); - } - let chunks = std::mem::take(&mut self.staged); + }; self.staged_rows = 0; self.staged_bytes = 0; let sent = tokio::select! { biased; - msg = self.fatal.wait() => return Err(SinkError::Other(msg)), - r = self.msg_tx.send(BatcherMsg::Segments(chunks)) => r, + why = self.fatal.wait() => return Err(SinkError::Other(why)), + r = self.msg_tx.send(msg) => r, }; sent.map_err(|_| SinkError::Other("batcher channel closed".into())) } @@ -290,6 +349,10 @@ impl ReorderSink { fatal, seqs, staged: Vec::new(), + pending: Vec::new(), + pending_permit: None, + encoders: Encoders::default(), + stats: stats.clone(), staged_rows: 0, staged_bytes: 0, budget, @@ -697,18 +760,22 @@ impl ReorderSink { if !self.resolver.stores_chunks() || self.retires.is_empty() { return Ok(()); } - let cut = self.resume_floor.get(); - for entry in self.retires.due(cut) { - self.resolver - .retire_mirror(entry.db_oid, entry.toast_relid) - .await - .map_err(SinkError::ctx("toast mirror retire"))?; - self.retires - .remove(entry) - .await - .map_err(SinkError::ctx("toast retire ledger"))?; - } - Ok(()) + // Boxed: rare, and deep enough to bloat every commit future + Box::pin(async { + let cut = self.resume_floor.get(); + for entry in self.retires.due(cut) { + self.resolver + .retire_mirror(entry.db_oid, entry.toast_relid) + .await + .map_err(SinkError::ctx("toast mirror retire"))?; + self.retires + .remove(entry) + .await + .map_err(SinkError::ctx("toast retire ledger"))?; + } + Ok(()) + }) + .await } /// Settle pending rows on commit or abort, including subtransactions @@ -886,27 +953,22 @@ impl ReorderSink { commit_lsn: u64, publish: Publish, ) -> Result<(), SinkError> { - let permit = self.out.admit(segment.bytes).await?.map(Arc::new); + let permit = self.out.admit(segment.bytes).await?; let seq = self.out.seqs.open(commit_lsn, publish); self.stats.queue_jobs_out.fetch_add(1, Ordering::Relaxed); self.stats.decode_jobs_in.fetch_add(1, Ordering::Relaxed); self.stats .decode_rows_out .fetch_add(segment.rows, Ordering::Relaxed); - let chunk_rows = self.dest.current().decode_chunk_rows; + let rows = segment.rows; + self.out.stage(seq.seq(), segment, permit).await?; let out = &mut self.out; - out.staged.push(PlacedSegment { - seq: seq.seq(), - chunks: segment.chunks, - permit, - }); - out.seqs.place(seq, segment.rows); - out.staged_rows += segment.rows as usize; - out.staged_bytes += segment.bytes; - if out.staged_rows >= chunk_rows + out.seqs.place(seq, rows); + if out.staged_rows >= self.dest.current().decode_chunk_rows || out.staged_bytes >= crate::emit::pipeline::batcher::DECODE_CHUNK_BYTES { - out.flush_pending().await?; + // Boxed: once per many commits, inline it would bloat each + Box::pin(out.flush_pending()).await?; } Ok(()) } @@ -1476,29 +1538,6 @@ impl PlanRouteView for ReorderRouteView<'_> { } } -impl ReorderSink { - async fn handle(&mut self, record: &Record<'_>) -> Result<(), SinkError> { - if !Self::wants(record) { - return Ok(()); - } - let info = record.parsed.header.info; - let op = info & XLOG_XACT_OPMASK; - let xid = record.parsed.header.xact_id; - match op { - XLOG_XACT_COMMIT | XLOG_XACT_COMMIT_PREPARED => self.on_commit(xid, info, record).await, - XLOG_XACT_ABORT | XLOG_XACT_ABORT_PREPARED => self.on_abort(xid, info, record).await, - XLOG_XACT_ASSIGNMENT => { - if let Some((xtop, subs)) = parse_xact_assignment(&record.parsed.main_data) { - self.subxact_tracker.lock().await.assign(xtop, &subs); - } - Ok(()) - } - // PREPARE allocates no seq; COMMIT_PREPARED drains it later - _ => Ok(()), - } - } -} - impl RecordSink for ReorderSink { fn wants(record: &Record<'_>) -> bool { record.parsed.header.resource_manager_id == RmId::Xact as u8 @@ -1520,10 +1559,7 @@ impl RecordSink for ReorderSink { &'a mut self, record: &'a Record<'a>, ) -> Pin> + Send + 'a>> { - Box::pin(async move { - self.handle(record).await?; - self.out.flush_pending().await - }) + self.on_records(std::slice::from_ref(record)) } fn on_records<'a>( @@ -1531,8 +1567,27 @@ impl RecordSink for ReorderSink { batch: &'a [Record<'a>], ) -> Pin> + Send + 'a>> { Box::pin(async move { + // Dispatch inline: a per-record helper future would copy the + // commit future once more per commit for record in batch.iter().filter(|r| Self::wants(r)) { - self.handle(record).await?; + let info = record.parsed.header.info; + let xid = record.parsed.header.xact_id; + match info & XLOG_XACT_OPMASK { + XLOG_XACT_COMMIT | XLOG_XACT_COMMIT_PREPARED => { + self.on_commit(xid, info, record).await?; + } + XLOG_XACT_ABORT | XLOG_XACT_ABORT_PREPARED => { + self.on_abort(xid, info, record).await?; + } + XLOG_XACT_ASSIGNMENT => { + if let Some((xtop, subs)) = parse_xact_assignment(&record.parsed.main_data) + { + self.subxact_tracker.lock().await.assign(xtop, &subs); + } + } + // PREPARE allocates no seq; COMMIT_PREPARED drains it later + _ => {} + } } self.out.flush_pending().await }) @@ -1747,6 +1802,10 @@ mod tests { fatal: fatal.clone(), seqs: ack.seqs(0).buffer_events(), staged: Vec::new(), + pending: Vec::new(), + pending_permit: None, + encoders: Encoders::default(), + stats: Arc::default(), staged_rows: 0, staged_bytes: 0, budget: Some(budget.clone()), @@ -1754,6 +1813,61 @@ mod tests { (out, tail) } + /// Encoded and pending segments ship in staging order, each run of one + /// kind as one message + #[tokio::test] + async fn staged_kinds_ship_in_seq_order() { + use crate::emit::pipeline::plan_spool::PendingRows; + let fatal = Fatal::new(); + let (ack, _collector) = + crate::emit::pipeline::ack::spawn(Arc::new(Monotone::default()), fatal.clone()); + let (msg_tx, mut msg_rx) = mpsc::channel(8); + let mut out = Outbox { + msg_tx, + fatal, + seqs: ack.seqs(0).buffer_events(), + staged: Vec::new(), + pending: Vec::new(), + pending_permit: None, + encoders: Encoders::default(), + stats: Arc::default(), + staged_rows: 0, + staged_bytes: 0, + budget: None, + }; + let segment = |body| Segment { + body, + rows: 0, + bytes: 0, + }; + let encoded = || segment(SegmentBody::Encoded(Vec::new())); + let pending = || segment(SegmentBody::Pending(PendingRows::default())); + for (seq, s) in [ + (1, encoded()), + (2, encoded()), + (3, pending()), + (4, pending()), + (5, encoded()), + ] { + out.stage(seq, s, None).await.unwrap(); + } + out.flush_pending().await.unwrap(); + let mut shipped = Vec::new(); + while let Ok(msg) = msg_rx.try_recv() { + shipped.push(match msg { + BatcherMsg::Segments(s) => { + format!("{:?}", s.iter().map(|s| s.seq).collect::>()) + } + BatcherMsg::Encoded(job) => { + job.await.unwrap().unwrap(); + "job".into() + } + BatcherMsg::FlushAll(_) => unreachable!(), + }); + } + assert_eq!(shipped, ["[1, 2]", "job", "[5]"]); + } + /// Pool short only by permits riding staged rows: admission ships them /// rather than waiting on itself #[tokio::test] diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index f753070f..03a01609 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -56,8 +56,6 @@ pub(crate) struct ResolvedBatch { pub blocks: Vec, pub n_rows: usize, pub per_seq: Vec<(u64, u64)>, - /// Admission permit shares covering these rows, dropped post-insert-ack - _permits: Vec>, _encoded: crate::budget::MemoryPermit, } @@ -237,7 +235,6 @@ async fn run( blocks, n_rows, per_seq, - permits, mut encoded, } = batch; let mut sealed = Vec::with_capacity(blocks.len()); @@ -287,7 +284,6 @@ async fn run( blocks: sealed, n_rows, per_seq, - _permits: permits, _encoded: encoded, }; if tx.send(ready).await.is_err() { diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index fac415bf..ea918c91 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -22,6 +22,7 @@ use crate::config::ResolvedConfig; use crate::emit::ch_emitter::EmitterStats; use crate::emit::pipeline::ack::{self, AckHandle}; use crate::emit::pipeline::batcher::{self, BatcherConfig, BatcherMsg, InsertBatch}; +use crate::emit::pipeline::encode::EncodedRun; use crate::emit::pipeline::inserter; use crate::emit::pipeline::resolver::{self, ResolvedBatch}; use crate::emit::pipeline::{DEFAULT_PIPELINE_FLUSH, Fatal}; @@ -217,6 +218,15 @@ fn swallow_segments(ack: &AckHandle, segments: Vec) { ack.swallowed(counts.into_iter().collect()); } +/// Ack every row an encode job produced +fn swallow_run(ack: &AckHandle, run: EncodedRun) { + let mut counts: HashMap = HashMap::new(); + for (seq, rows) in run.chunks.into_iter().flat_map(|(_, seqs)| seqs) { + *counts.entry(seq).or_insert(0) += rows; + } + ack.swallowed(counts.into_iter().collect()); +} + /// Metrics-only tail: ack collector + one swallow task, zero CH /// connections. Every routed row acks at swallow (permits release on /// drop) so nothing can pin the watermark; `FlushAll` replies @@ -226,7 +236,7 @@ pub fn spawn_null( emitter_ack: Arc>, fatal: Fatal, ) -> (mpsc::Sender, AckHandle, TailParts) { - let (ack, collector) = ack::spawn(emitter_ack, fatal); + let (ack, collector) = ack::spawn(emitter_ack, fatal.clone()); let collector = AbortOnDropHandle::new(collector); let (msg_tx, mut msg_rx) = mpsc::channel::(256); let swallow_ack = ack.clone(); @@ -234,6 +244,11 @@ pub fn spawn_null( while let Some(msg) = msg_rx.recv().await { match msg { BatcherMsg::Segments(segments) => swallow_segments(&swallow_ack, segments), + BatcherMsg::Encoded(job) => match job.await { + Ok(Ok(run)) => swallow_run(&swallow_ack, run), + Ok(Err(e)) => return fatal.set(format!("encode: {e}")), + Err(e) => return fatal.set(format!("encode job: {e}")), + }, BatcherMsg::FlushAll(reply) => { let _ = reply.send(()); } diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index dc68ca89..458c0e93 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -134,6 +134,16 @@ impl OracleColumnBuf { } } + /// Empty buffer for the same column + pub fn empty_like(&self) -> Self { + Self::with_target( + self.source_type_oid, + self.source_typmod, + self.string_target, + self.nullable_target, + ) + } + pub fn push(&mut self, cell: OracleCell) { self.payload_capacity += match &cell { OracleCell::Default => 0, diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index 78a06fee..ac1c3d6c 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -1455,16 +1455,16 @@ impl XactBuffer { }); } let allowed_xids = xids.iter().copied().collect(); - let merged = self - .open_drain( - states, - collect_rows, - stash, - top_xid, - commit_lsn, - allowed_xids, - ) - .await?; + // Boxed: merge setup would bloat every commit future + let merged = Box::pin(self.open_drain( + states, + collect_rows, + stash, + top_xid, + commit_lsn, + allowed_xids, + )) + .await?; self.stats.committed_xacts_total += 1; Ok(CommittedDrain { commit_ts, @@ -2329,6 +2329,57 @@ pub struct CommittedDrain { generations: Vec>, } +/// Next slice of a merged drain, see [`CommittedDrain::next_batch_admitting`] +async fn next_merged>( + m: &mut MergedDrain, + generations: &mut Vec>, + max_rows: usize, + max_bytes: usize, + admit: impl AsyncFnOnce(usize) -> std::result::Result, E>, +) -> std::result::Result, E> { + let mut heaps: Vec = Vec::new(); + let mut ordered_events: Vec = Vec::new(); + let mut truncate_rows: Vec = Vec::new(); + let mut bytes = 0usize; + while heaps.len() < max_rows.max(1) && bytes < max_bytes.max(1) { + match m.next().await? { + None => break, + Some(MergeItem::Event(event)) => ordered_events.push(OrderedEvent { + heap_idx: heaps.len(), + row_idx: m.rows.len(), + event: *event, + }), + Some(MergeItem::Truncate(t)) => { + truncate_rows.push(m.rows.len()); + heaps.push(DrainHeap::Truncate(t)); + } + Some(MergeItem::Input(i)) => { + bytes += i.approx_bytes(); + heaps.push(DrainHeap::Input(i)); + } + } + } + let is_final = m.is_exhausted(); + let mut sealed = m.take_chunks()?; + let sealed_empty = sealed.map.is_empty(); + let new_rows = m.take_rows(); + if !sealed_empty { + sealed._permit = admit(sealed.resident_bytes()).await?; + generations.push(Arc::new(sealed)); + } + if heaps.is_empty() && ordered_events.is_empty() && sealed_empty && new_rows.is_empty() { + return Ok(None); + } + Ok(Some(DrainedBatch { + heaps, + ordered_events, + chunks: generations.clone(), + new_rows, + truncate_rows, + is_final, + })) +} + enum Drained { /// Boxed so the commit path moves a pointer, not the merge state Merged(Box), @@ -2382,47 +2433,15 @@ impl CommittedDrain { })); } }; - let mut heaps: Vec = Vec::new(); - let mut ordered_events: Vec = Vec::new(); - let mut truncate_rows: Vec = Vec::new(); - let mut bytes = 0usize; - while heaps.len() < max_rows.max(1) && bytes < max_bytes.max(1) { - match m.next().await? { - None => break, - Some(MergeItem::Event(event)) => ordered_events.push(OrderedEvent { - heap_idx: heaps.len(), - row_idx: m.rows.len(), - event: *event, - }), - Some(MergeItem::Truncate(t)) => { - truncate_rows.push(m.rows.len()); - heaps.push(DrainHeap::Truncate(t)); - } - Some(MergeItem::Input(i)) => { - bytes += i.approx_bytes(); - heaps.push(DrainHeap::Input(i)); - } - } - } - let is_final = m.is_exhausted(); - let mut sealed = m.take_chunks()?; - let sealed_empty = sealed.map.is_empty(); - let new_rows = m.take_rows(); - if !sealed_empty { - sealed._permit = admit(sealed.resident_bytes()).await?; - self.generations.push(Arc::new(sealed)); - } - if heaps.is_empty() && ordered_events.is_empty() && sealed_empty && new_rows.is_empty() { - return Ok(None); - } - Ok(Some(DrainedBatch { - heaps, - ordered_events, - chunks: self.generations.clone(), - new_rows, - truncate_rows, - is_final, - })) + // Boxed: merge state would bloat every commit future + Box::pin(next_merged( + m, + &mut self.generations, + max_rows, + max_bytes, + admit, + )) + .await } /// Unlink spill files; call after the final slice dispatches. On an From 9ff568da67fa7342bc198cda63358891940a2025 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:19:22 +0000 Subject: [PATCH 18/20] Don't handle compression mismatch, update with more clickhouse-c-rs changes --- Cargo.lock | 2 +- docs/configuration.md | 10 +- docs/destination-tables.md | 5 +- plans/performance.md | 4 +- src/bin/stream/metrics_publish.rs | 3 - src/bin/stream/session.rs | 2 +- src/bin/stream/source_db.rs | 2 +- src/catalog/type_bridge.rs | 18 ++- src/ch.rs | 26 ++++- src/ch/test_support.rs | 2 +- src/config.rs | 14 ++- src/decode/heap_decoder.rs | 2 +- src/emit/ch_emitter.rs | 22 ++-- src/emit/pipeline/inserter.rs | 127 ++++++-------------- src/emit/pipeline/resolver.rs | 21 +++- src/emit/pipeline/sealed.rs | 186 +----------------------------- src/emit/pipeline/tail.rs | 2 +- src/ops/metrics.rs | 6 - src/toast/resolver.rs | 39 ++++--- tests/bootstrap_pipeline_ch.rs | 2 +- tests/bootstrap_window_leg_ch.rs | 2 +- tests/ch_ddl_apply_ch.rs | 2 +- tests/common/inproc_harness.rs | 2 +- tests/emitter_budget_flush.rs | 4 +- tests/emitter_native_types.rs | 2 +- tests/emitter_tls.rs | 2 +- tests/pipeline_tail.rs | 2 +- tests/tee_cdc.rs | 10 +- 28 files changed, 153 insertions(+), 368 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f800be2c..afb789fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2415,7 +2415,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/docs/configuration.md b/docs/configuration.md index 80f2baa9..04ec4e80 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -349,16 +349,14 @@ reserves workspace on top of `byte_budget` for one block's compressed output and frame scratch, plus 64 MiB for oracle request and response when the tail has an oracle; resolvers count oracle bridge concurrency, or one without an oracle. Blocks needing more workspace than that reserve seal one at a time -Inserters send prepared bytes unchanged. A connection unable to accept them, -such as a destination configured with other compression, gets each block -decoded and rebuilt into one block of scratch per inserter, outside the pool. +Inserters send prepared bytes unchanged to every destination. An INSERT carries several blocks, sealed at 1,048,576 rows or 16 MiB of column bytes and when oracle request size fills, while batch limits still bound each INSERT. `walshadow_resident_raw_bytes_total` over `walshadow_resident_stored_bytes_total` reports how far sealing shrinks -resident blocks. `walshadow_output_resident_bytes`, -`walshadow_batcher_open_bytes`, and `walshadow_inserter_scratch_bytes` report -allocated bytes held by output pools, open batches, and rebuild scratch +resident blocks. `walshadow_output_resident_bytes` and +`walshadow_batcher_open_bytes` report allocated bytes held by output pools and +open batches This pool is separate from decoded-payload admission to avoid circular waits. Live byte-budget reductions lower flush thresholds; increasing sealed-buffer capacity requires restarting walshadow diff --git a/docs/destination-tables.md b/docs/destination-tables.md index 61861445..dbb1b5a1 100644 --- a/docs/destination-tables.md +++ b/docs/destination-tables.md @@ -198,9 +198,10 @@ By default, a tee uses its main destination's instance and database. Set use `archive`, and another copy of orders goes to `[ch]` Named connections accept `host`, `port`, `database`, `user`, `password`, `secure`, -`tls_server_name`, and `compression`, with the same defaults as `[ch]`. +and `tls_server_name`, with the same defaults as `[ch]`. Credentials from `[ch]` are not reused for named connections. All destinations -share pipeline limits, pool sizes, retry settings, and column layout. Reload +share pipeline limits, pool sizes, retry settings, `[ch] compression`, and column +layout. Reload TOML to update connection names and table mappings. `config_table` has no instance column. `default` is reserved, and unknown names cause configuration errors diff --git a/plans/performance.md b/plans/performance.md index abca4c74..e9948261 100644 --- a/plans/performance.md +++ b/plans/performance.md @@ -60,8 +60,8 @@ payloads, healthy and stalled destinations Record rows/s, source WAL bytes/s, CPU, allocation count, peak/steady RSS, spill volume, durable lag, blocks per INSERT, and ClickHouse part creation. Record retained bytes by owner from `walshadow_` metrics `xact_bytes_in_memory`, -`resident_payload_bytes`, `output_resident_bytes`, `batcher_open_bytes`, -`inserter_scratch_bytes`, and `resident_raw_bytes_total` over +`resident_payload_bytes`, `output_resident_bytes`, `batcher_open_bytes`, and +`resident_raw_bytes_total` over `resident_stored_bytes_total`, then compare their sum against RSS to size allocator and client overhead. Attribute savings separately to compact transaction storage, direct encoding, and compression diff --git a/src/bin/stream/metrics_publish.rs b/src/bin/stream/metrics_publish.rs index 62146820..1197b6f0 100644 --- a/src/bin/stream/metrics_publish.rs +++ b/src/bin/stream/metrics_publish.rs @@ -608,14 +608,11 @@ pub(crate) fn stage_gauges_on(v: &StageCounters<'_>, base: MetricsSnapshot) -> M insertbatch_batches_out_total: emitter(|s| &s.insertbatch_batches_out), inserter_batches_in_total: emitter(|s| &s.inserter_batches_in), inserter_ch_seconds_total: emitter_seconds(|s| &s.inserter_ch_nanos), - inserter_rebuilds_total: emitter(|s| &s.inserter_rebuilds), - inserter_rebuild_seconds_total: emitter_seconds(|s| &s.inserter_rebuild_nanos), resident_seal_seconds_total: emitter_seconds(|s| &s.resident_seal_nanos), resident_raw_bytes_total: emitter(|s| &s.resident_raw_bytes), resident_stored_bytes_total: emitter(|s| &s.resident_stored_bytes), output_resident_bytes: emitter(|s| &s.output_resident_bytes), batcher_open_bytes: emitter(|s| &s.batcher_open_bytes), - inserter_scratch_bytes: emitter(|s| &s.inserter_scratch_bytes), oracle_resolve_seconds_total: emitter_seconds(|s| &s.oracle_resolve_nanos), process_cpu_seconds_total: proc_cpu, process_resident_memory_bytes: proc_rss, diff --git a/src/bin/stream/session.rs b/src/bin/stream/session.rs index 0052f8fa..fc1ec7ce 100644 --- a/src/bin/stream/session.rs +++ b/src/bin/stream/session.rs @@ -1235,7 +1235,7 @@ async fn session_body( emitter_cfg.row_budget = rc.row_budget; emitter_cfg.byte_budget = rc.byte_budget; emitter_cfg.flush_timeout = rc.flush_timeout; - emitter_cfg.conn.compression = rc.conn.compression; + emitter_cfg.compression = rc.compression; emitter_cfg.retry.max_attempts = rc.retry_max_attempts; } applicators.insert(conn.oid, built.applicator); diff --git a/src/bin/stream/source_db.rs b/src/bin/stream/source_db.rs index 6f3977b6..b419333b 100644 --- a/src/bin/stream/source_db.rs +++ b/src/bin/stream/source_db.rs @@ -124,7 +124,7 @@ pub(crate) async fn build_source_db(input: SourceDbInputs<'_>) -> anyhow::Result cfg.row_budget = rc.row_budget; cfg.byte_budget = rc.byte_budget; cfg.flush_timeout = rc.flush_timeout; - cfg.conn.compression = rc.conn.compression; + cfg.compression = rc.compression; cfg.retry.max_attempts = rc.retry_max_attempts; } // DDL applicator owned by the reorder coordinator so ALTER / diff --git a/src/catalog/type_bridge.rs b/src/catalog/type_bridge.rs index a77e9427..ee266087 100644 --- a/src/catalog/type_bridge.rs +++ b/src/catalog/type_bridge.rs @@ -225,9 +225,9 @@ pub(crate) fn column_literal(buf: &ColumnBuf, ch_inner: &str) -> Option }; let unsigned = |n: usize| le(n).map(|w| u128::from_le_bytes(w) as i128); // Ticks at `scale` digits to microseconds - let micros = |ticks: i64, scale: i32| match 6 - scale { - up @ 0.. => ticks.saturating_mul(10i64.pow(up as u32)), - down => ticks / 10i64.pow(down.unsigned_abs()), + let micros = |ticks: i64, scale: u32| match scale.checked_sub(6) { + Some(down) => ticks / 10i64.pow(down), + None => ticks.saturating_mul(10i64.pow(6 - scale)), }; match view.kind()? { Kind::Bool => Some(if bytes[0] != 0 { "true" } else { "false" }.into()), @@ -253,11 +253,11 @@ pub(crate) fn column_literal(buf: &ColumnBuf, ch_inner: &str) -> Option )), Kind::Time64 => { let ticks = i64::from_le_bytes(bytes.try_into().ok()?); - let txt = format_time_us(micros(ticks, view.datetime64_scale()? as i32)); + let txt = format_time_us(micros(ticks, view.datetime64_scale()?)); Some(sql_string_literal(&txt)) } Kind::DateTime64 => { - let prec = view.datetime64_scale()? as i32; + let prec = view.datetime64_scale()?; let unix = micros(i64::from_le_bytes(bytes.try_into().ok()?), prec); let txt = render_pg_timestamp(unix - walrus::pg::replication::PG_EPOCH_USEC); Some(format!( @@ -916,11 +916,7 @@ mod tests { for type_name in ["vector", "halfvec"] { let r = map(&array_attr(type_name, "[1,2,3]"), false); assert_eq!(r.ch_type, "Array(Float32)"); - assert_eq!( - r.default_sql.as_deref(), - Some("unhex('5b312c322c335d')"), - "{type_name}" - ); + assert_eq!(r.default_sql.as_deref(), Some("'[1,2,3]'"), "{type_name}"); } } @@ -934,7 +930,7 @@ mod tests { #[test] fn other_pending_text_defaults_cast_from_their_text() { let r = map(&attr(NUMERICOID, -1, true, Some("3.14")), false); - assert_eq!(r.default_sql.as_deref(), Some("unhex('332e3134')")); + assert_eq!(r.default_sql.as_deref(), Some("'3.14'")); } #[test] diff --git a/src/ch.rs b/src/ch.rs index 1cea8371..258bb628 100644 --- a/src/ch.rs +++ b/src/ch.rs @@ -302,6 +302,7 @@ pub async fn connect_creating_database( struct Connection { client: Option, dialed: Option>, + compression: CompressionChoice, last_used: Instant, } @@ -310,6 +311,7 @@ impl Default for Connection { Self { client: None, dialed: None, + compression: CompressionChoice::None, last_used: Instant::now(), } } @@ -319,6 +321,8 @@ impl Default for Connection { pub struct ChConn { instance: Option, selected: Option>, + /// Dial with this instead of configured compression + compression: Option, active: Connection, parked: foldhash::HashMap, Connection>, dest: Arc, @@ -332,6 +336,7 @@ impl ChConn { dest, instance: None, selected: None, + compression: None, active: Connection::default(), parked: Default::default(), dials: 0, @@ -366,6 +371,12 @@ impl ChConn { Ok(()) } + /// Dial every destination with `compression`, so prepared blocks sealed + /// in it send as is + pub fn force_compression(&mut self, compression: CompressionChoice) { + self.compression = Some(compression); + } + /// Redial now, for settings fixed at connect (codec, host, credentials) pub async fn dial(&mut self) -> Result<(), EmitterError> { self.active.client = None; @@ -446,23 +457,32 @@ impl ChConn { self.selected = Some(self.dest.instance(name)?); } let config = self.config(); + let compression = self.compression.unwrap_or(config.compression); let moved = self .active .dialed .as_ref() .is_none_or(|dialed| !Arc::ptr_eq(dialed, &config)); - if moved || self.active.last_used.elapsed() >= config.idle_reconnect() { + if moved + || compression != self.active.compression + || self.active.last_used.elapsed() >= config.idle_reconnect() + { self.active.client = None; } if let Some(client) = self.active.client.take() { return Ok(client); } + let mut dial = config.clone(); + if compression != config.compression { + Arc::make_mut(&mut dial).compression = compression; + } let client = if self.instance.is_some() { - connect_creating_database(&config).await?.0 + connect_creating_database(&dial).await?.0 } else { - connect_client(&*config).await? + connect_client(&*dial).await? }; self.active.dialed = Some(config); + self.active.compression = compression; self.active.last_used = Instant::now(); self.dials += 1; Ok(client) diff --git a/src/ch/test_support.rs b/src/ch/test_support.rs index 931e7fbd..e1119bc9 100644 --- a/src/ch/test_support.rs +++ b/src/ch/test_support.rs @@ -21,9 +21,9 @@ pub(crate) async fn retry_server( conn: InstanceConfig { host: "127.0.0.1".into(), port: listener.local_addr().unwrap().port(), - compression: CompressionChoice::None, ..Default::default() }, + compression: CompressionChoice::None, retry: RetryConfig { max_attempts, initial_backoff: Duration::from_millis(1), diff --git a/src/config.rs b/src/config.rs index e382f4f1..949120bc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -158,9 +158,10 @@ pub struct ResolvedConfig { pub flush_timeout: Duration, /// CH client retry budget (live: inserter reads per attempt loop) pub retry_max_attempts: u32, - /// CH connection (live: inserter + DDL applicator reconnect on change, - /// inserter rebuilds its codec on compression change) + /// CH connection (live: inserter + DDL applicator reconnect on change) pub conn: InstanceConfig, + /// Wire compression (live: resolvers seal later batches in it) + pub compression: CompressionChoice, /// Source PostgreSQL connection and slot, live via config reload pub source: SourceConn, /// Columns-less `[table.*]` opt-in intents (live: reorder coordinator @@ -183,6 +184,7 @@ impl ResolvedConfig { pub fn dest_conn_eq(&self, cfg: &EmitterConfig) -> bool { cfg.instances == self.instances && cfg.conn == self.conn + && cfg.compression == self.compression && cfg.retry.max_attempts == self.retry_max_attempts } @@ -190,6 +192,7 @@ impl ResolvedConfig { let mut out = EmitterConfig { instances: self.instances.clone(), conn: self.conn.clone(), + compression: self.compression, ..base.clone() }; out.retry.max_attempts = self.retry_max_attempts; @@ -786,6 +789,7 @@ impl ConfigResolver { flush_timeout: base.flush_timeout, retry_max_attempts: base.retry.max_attempts, conn: base.conn.clone(), + compression: base.compression, source: base.source.clone(), table_opt_ins: base.table_opt_ins.clone(), paused: base.paused, @@ -863,7 +867,7 @@ impl ConfigResolver { // codec (e.g. zstd with the feature off) is rejected here, never // surfaced as a fatal when the inserter reconnects. match v.parse().map(|c: CompressionChoice| (c, c.build_codec())) { - Ok((c, Ok(_))) => rc.conn.compression = c, + Ok((c, Ok(_))) => rc.compression = c, _ => { rejections += 1; tracing::warn!(target: "walshadow::config", value = %v, "config_global.compression rejected"); @@ -1160,7 +1164,7 @@ mod tests { assert_eq!(r.row_budget, 1000); assert_eq!(r.byte_budget, 4096); assert_eq!(r.flush_timeout, Duration::from_millis(250)); - assert_eq!(r.conn.compression, CompressionChoice::None); + assert_eq!(r.compression, CompressionChoice::None); assert_eq!(r.retry_max_attempts, 9); } @@ -1206,7 +1210,7 @@ mod tests { // Prior (TOML/base) values survive each rejection. assert_eq!(r.drop_table_strategy, DropTableStrategy::Retain); assert_eq!(r.detach_strategy, DropTableStrategy::Retain); - assert_eq!(r.conn.compression, base.conn.compression); + assert_eq!(r.compression, base.compression); assert_eq!(r.row_budget, base.row_budget); assert_eq!(r.byte_budget, base.byte_budget); assert_eq!(r.flush_timeout, base.flush_timeout); diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index 3e0f4355..24df8134 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -293,7 +293,7 @@ pub struct CommittedTuple { /// (decoder unbuffered path). pub commit_ts: i64, /// Source LSN of matching `XLOG_XACT_COMMIT`. CH emitter snapshots onto - /// its ack-LSN gauge once the xact's `send_data(None)` finishes; daemon + /// its ack-LSN gauge once the xact's `send_data_end()` finishes; daemon /// status loop writes it to cursor file `emitter_ack_lsn`. 0 for /// decoder unbuffered path. pub commit_lsn: u64, diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index a214e01d..a890b9db 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -128,6 +128,8 @@ pub fn default_database<'a>( pub struct EmitterConfig { pub instances: BTreeMap, pub conn: InstanceConfig, + /// Wire compression for every destination, so blocks seal once + pub compression: CompressionChoice, /// Custom rustls roots/config for `secure` path: private CA, pinned /// self-signed cert, or mTLS. `None` uses public webpki roots via /// [`clickhouse_c::tls::default_config`]. Not parsed from TOML; @@ -620,7 +622,7 @@ impl ConnectionConfig for EmitterConfig { } fn compression(&self) -> CompressionChoice { - self.conn.compression + self.compression } fn idle_reconnect(&self) -> Duration { @@ -791,8 +793,6 @@ pub struct InstanceConfig { /// handshake still names what the certificate was issued for. Empty /// uses `host`. pub tls_server_name: String, - #[serde(deserialize_with = "crate::toml_de::de_parse")] - pub compression: CompressionChoice, } impl Default for InstanceConfig { @@ -806,7 +806,6 @@ impl Default for InstanceConfig { password: ch.password, secure: ch.secure, tls_server_name: ch.tls_server_name, - compression: ch.compression, } } } @@ -1018,8 +1017,8 @@ impl EmitterConfig { password: ch.password, secure: ch.secure, tls_server_name: ch.tls_server_name, - compression: ch.compression, }, + compression: ch.compression, tls_config: None, row_budget: ch.row_budget, byte_budget: ch.byte_budget, @@ -1146,7 +1145,6 @@ impl EmitterConfig { "ch.instances.{name}: host and port must be nonempty" ))); } - instance.compression.build_codec()?; } out.auto_create_name_all = prefix_all .as_deref() @@ -3126,8 +3124,7 @@ fn micros_to_ticks(us: i128, scale: u32) -> Result { return Err(EmitterError::Type("invalid temporal precision".into())); } let ticks = if scale < 6 { - let divisor = 10i128.pow(6 - scale); - us / divisor + us / 10i128.pow(6 - scale) } else { us * 10i128.pow(scale - 6) }; @@ -3508,10 +3505,6 @@ crate::atomic_stats! { /// `EndOfStream`), retries included. Against `inserter_pool_size × /// elapsed` this is CH-side utilization pub inserter_ch_nanos, - /// Blocks decoded and rebuilt for a connection refusing their - /// prepared bytes, and inserter time doing so - pub inserter_rebuilds, - pub inserter_rebuild_nanos, /// Resolver time inside the oracle round trip, retries included. /// Overlaps the inserters' ClickHouse time, so against /// `inserter_ch_nanos` it says which stage is the limiter @@ -3526,10 +3519,9 @@ crate::atomic_stats! { /// Resolver time serializing and compressing sealed blocks pub resident_seal_nanos, /// Gauges: every tail's output pool (queued slabs, sealed blocks, - /// seal workspace), open batch slabs, inserter rebuild scratch + /// seal workspace), open batch slabs pub output_resident_bytes: Arc, pub batcher_open_bytes, - pub inserter_scratch_bytes, } } @@ -5485,7 +5477,7 @@ mod tests { assert_eq!(c.conn.password, "secret"); assert!(c.conn.secure); assert_eq!(c.conn.tls_server_name, "svc.clickhouse.cloud"); - assert_eq!(c.conn.compression, CompressionChoice::Lz4); + assert_eq!(c.compression, CompressionChoice::Lz4); // Omitting `secure` defaults to plaintext, and SNI falls back to `host` let plain = EmitterConfig::from_toml_str("[ch]\nhost = \"h\"\n").unwrap(); assert!(!plain.conn.secure); diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index ce32ee90..1f9821a9 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -12,7 +12,7 @@ //! dedups by `_lsn`). Retry-exhaustion is fatal: the watermark can't advance //! without this batch. -use clickhouse_c::{Allocator, ErrorKind, QueryOpts}; +use clickhouse_c::QueryOpts; use tokio::task::JoinHandle; use crate::ch::{ChConn, EmitterError, EndOfStream, drain_to_end_of_stream, with_timeout}; @@ -22,25 +22,19 @@ use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::AckHandle; use crate::emit::pipeline::batcher::BatchMeta; use crate::emit::pipeline::resolver::ResolvedBatch; -use crate::emit::pipeline::sealed::{SealedBlock, builder_of}; +use crate::emit::pipeline::sealed::SealedBlock; use std::sync::Arc; use std::sync::atomic::Ordering; struct Inserter { client: ChConn, - alloc: Allocator, - /// One block's decompressed body for rebuilds, reused across sends; - /// locked only while a block opens - scratch: std::sync::Mutex>, ack: AckHandle, stats: Arc, } impl Inserter { /// Bounded reconnect+retry around one INSERT. Every attempt resends - /// every block in order, so a replay carries unchanged rows. A - /// connection refusing prepared bytes gets blocks opened one at a time - /// into `scratch` and rebuilt + /// every block in order, so a replay carries unchanged rows async fn send_with_retry( &mut self, sql: &str, @@ -48,48 +42,21 @@ impl Inserter { ) -> Result { let insert_timeout = self.client.config().insert_timeout; let started = std::time::Instant::now(); - let (alloc, stats) = (self.alloc, self.stats.clone()); - let scratch = &self.scratch; + let stats = &self.stats; let result = self .client .retry( - |mut client| { - let stats = &stats; - async move { - let result = with_timeout(insert_timeout, async { - client.send_query(sql, &QueryOpts::new()).await?; - for sealed in blocks { - match client.send_prepared(sealed.prepared()).await { - Ok(()) => continue, - Err(e) if e.kind != ErrorKind::Usage => return Err(e.into()), - Err(_) => {} - } - let opened = std::time::Instant::now(); - let block = { - let mut scratch = - scratch.lock().expect("inserter scratch poisoned"); - let held = scratch.capacity(); - let block = sealed.open(alloc, &mut scratch); - stats.inserter_scratch_bytes.fetch_add( - (scratch.capacity() - held) as u64, - Ordering::Relaxed, - ); - block? - }; - let bb = builder_of(&block)?; - stats.inserter_rebuilds.fetch_add(1, Ordering::Relaxed); - stats.inserter_rebuild_nanos.fetch_add( - opened.elapsed().as_nanos() as u64, - Ordering::Relaxed, - ); - client.send_data(&bb).await?; - } - client.send_data_end().await?; - drain_to_end_of_stream(&mut client).await - }) - .await; - (client, result) - } + |mut client| async move { + let result = with_timeout(insert_timeout, async { + client.send_query(sql, &QueryOpts::new()).await?; + for sealed in blocks { + client.send_prepared(sealed.prepared()).await?; + } + client.send_data_end().await?; + drain_to_end_of_stream(&mut client).await + }) + .await; + (client, result) }, |_, _| { stats.retries_attempted.fetch_add(1, Ordering::Relaxed); @@ -126,6 +93,7 @@ impl Inserter { async fn run(mut self, rx: async_channel::Receiver, fatal: Fatal) { while let Ok(batch) = rx.recv().await { + self.client.force_compression(batch.compression); match self.send_all(&batch.meta, &batch.blocks).await { Ok(durable) => { self.stats @@ -156,15 +124,6 @@ impl Inserter { } } -impl Drop for Inserter { - fn drop(&mut self) { - let held = self.scratch.get_mut().map_or(0, |s| s.capacity()); - self.stats - .inserter_scratch_bytes - .fetch_sub(held as u64, Ordering::Relaxed); - } -} - /// Connect `n` inserters and spawn drain loops pub(crate) async fn spawn_pool( n: usize, @@ -178,8 +137,6 @@ pub(crate) async fn spawn_pool( for _ in 0..n.max(1) { let inserter = Inserter { client: ChConn::connect(dest.clone()).await?, - alloc: Allocator::global(&mimalloc::MiMalloc), - scratch: Default::default(), ack: ack.clone(), stats: stats.clone(), }; @@ -195,49 +152,49 @@ mod tests { use super::*; use crate::ch::CompressionChoice; use crate::emit::pipeline::ack; - use clickhouse_c::{BlockBuilder, ColumnBuilder, TypeAst}; + use clickhouse_c::{Allocator, BlockBuilder, ColumnBuilder, TypeAst}; - /// One UInt8 column x holding one row 42 - fn one_row_block(compression: CompressionChoice) -> SealedBlock { + /// One uncompressed UInt8 column x holding one row 42 + fn one_row_block() -> SealedBlock { let alloc = Allocator::global(&mimalloc::MiMalloc); let ast = TypeAst::parse("UInt8", alloc).unwrap(); let column = ColumnBuilder::fixed(&[42], 1, 1).unwrap(); let mut bb = BlockBuilder::new(); bb.append("x", ast.view(), &column).unwrap(); - let codec = compression.build_codec().unwrap(); - SealedBlock::seal(&bb, compression, codec.as_ref().map(|c| c.as_ref()), alloc).unwrap() + SealedBlock::seal(&bb, CompressionChoice::None, None, alloc).unwrap() } /// Every attempt resends every block of the INSERT through failed and - /// idle reconnects, and after a drop between Data blocks; blocks - /// prepared for other compression than the uncompressed server - /// connection rebuild + /// idle reconnects, and after a drop between Data blocks, dialing + /// uncompressed blocks' compression over configured lz4 #[tokio::test] async fn insert_retry_preserves_payload_through_failed_reconnect() { let sql = "INSERT INTO retry_test (x) VALUES"; - for (retries, idle, blocks, codec, cut) in [ - (0, false, 1, CompressionChoice::Lz4, None), - (1, false, 1, CompressionChoice::None, None), - (2, false, 1, CompressionChoice::Zstd, None), - (2, false, 3, CompressionChoice::Lz4, None), - (1, true, 2, CompressionChoice::Zstd, None), - (2, false, 3, CompressionChoice::None, Some(1)), - (2, false, 3, CompressionChoice::Zstd, Some(2)), + for (retries, idle, blocks, cut) in [ + (0, false, 1, None), + (1, false, 1, None), + (2, false, 1, None), + (2, false, 3, None), + (1, true, 2, None), + (2, false, 3, Some(1)), + (2, false, 3, Some(2)), ] { - let (config, server) = + let (mut config, server) = crate::ch::test_support::retry_server(retries, sql, blocks, idle, cut).await; + config.compression = CompressionChoice::Lz4; let dest = DestEmitter::new(Arc::new(config), None); - let client = ChConn::connect(dest).await.unwrap(); + let mut client = ChConn::deferred(dest); + client.force_compression(CompressionChoice::None); + client.dial().await.unwrap(); + client.take_dials(); let (ack, collector) = ack::spawn(Arc::default(), Fatal::new()); let stats = Arc::new(EmitterStats::default()); let mut inserter = Inserter { client, - alloc: Allocator::global(&mimalloc::MiMalloc), - scratch: Default::default(), ack, stats: stats.clone(), }; - let sealed: Vec<_> = (0..blocks).map(|_| one_row_block(codec)).collect(); + let sealed: Vec<_> = (0..blocks).map(|_| one_row_block()).collect(); assert_eq!( inserter.send_with_retry(sql, &sealed).await.is_ok(), retries == 2 || idle @@ -250,18 +207,8 @@ mod tests { stats.reconnects.load(Ordering::Relaxed), u64::from(retries == 2 || idle) ); - assert_eq!( - stats.inserter_rebuilds.load(Ordering::Relaxed) > 0, - codec != CompressionChoice::None, - "{codec:?}" - ); - assert_eq!( - stats.inserter_scratch_bytes.load(Ordering::Relaxed), - inserter.scratch.lock().unwrap().capacity() as u64 - ); server.await.unwrap(); drop(inserter); - assert_eq!(stats.inserter_scratch_bytes.load(Ordering::Relaxed), 0); collector.await.unwrap(); } } diff --git a/src/emit/pipeline/resolver.rs b/src/emit/pipeline/resolver.rs index 03a01609..8b32d5e8 100644 --- a/src/emit/pipeline/resolver.rs +++ b/src/emit/pipeline/resolver.rs @@ -54,6 +54,8 @@ use foldhash::HashMap; pub(crate) struct ResolvedBatch { pub meta: Arc, pub blocks: Vec, + /// Compression every block was sealed in + pub compression: CompressionChoice, pub n_rows: usize, pub per_seq: Vec<(u64, u64)>, _encoded: crate::budget::MemoryPermit, @@ -67,8 +69,7 @@ pub(crate) struct ResolverOptions { pub oracle: Option>, /// Boot budget for oracle transport retries, re-read from `config_rx` pub retry: RetryConfig, - /// Compression of prepared blocks, matching connections that negotiate - /// it; others rebuild + /// Boot compression of prepared blocks, re-read from `config_rx` pub codec: CompressionChoice, pub stats: Arc, pub fatal: Fatal, @@ -146,6 +147,14 @@ impl Sealer { }) } + fn set_compression(&mut self, compression: CompressionChoice) -> Result<(), EmitterError> { + if compression != self.compression { + self.codec = compression.build_codec()?; + self.compression = compression; + } + Ok(()) + } + /// Serialize and compress one block over its slabs and oracle answer pub(crate) fn seal( &mut self, @@ -228,7 +237,12 @@ async fn run( }; while let Ok(batch) = rx.recv().await { if let Some(rx) = config_rx.as_ref() { - retry.max_attempts = rx.borrow().retry_max_attempts; + let rc = rx.borrow().clone(); + retry.max_attempts = rc.retry_max_attempts; + if let Err(e) = sealer.set_compression(rc.compression) { + fatal.set(format!("seal codec: {e}")); + return; + } } let InsertBatch { meta, @@ -282,6 +296,7 @@ async fn run( let ready = ResolvedBatch { meta, blocks: sealed, + compression: sealer.compression, n_rows, per_seq, _encoded: encoded, diff --git a/src/emit/pipeline/sealed.rs b/src/emit/pipeline/sealed.rs index 6ffec5ae..b3f12c50 100644 --- a/src/emit/pipeline/sealed.rs +++ b/src/emit/pipeline/sealed.rs @@ -2,37 +2,18 @@ //! INSERT acknowledgement. //! //! clickhouse-c serializes each block once, compressing wire frames as it -//! goes, and column slabs drop. Inserters write those bytes on connections -//! negotiating the same compression; a connection refusing them, such as a -//! named destination configured otherwise, gets the block decoded and rebuilt +//! goes, and column slabs drop. Inserters dial with compression blocks were +//! sealed in, since clickhouse-c refuses prepared bytes otherwise use std::pin::Pin; -use clickhouse_c::{ - Allocator, Block, BlockBuilder, BlockOpts, BlockReader, Codec, PreparedBlock, SliceIo, -}; +use clickhouse_c::{Allocator, BlockBuilder, Codec, PreparedBlock}; use crate::ch::{CompressionChoice, EmitterError}; -/// TCP Data packet body framing, which prepared bytes carry -const OPTS: BlockOpts = BlockOpts { - has_block_info: true, - has_custom_serialization: true, - read_buffer_bytes: 0, -}; - -/// CityHash128 checksum, then method byte and two little-endian u32 sizes; -/// ClickHouse `src/Compression/CompressionInfo.h` -const CHECKSUM: usize = 16; -const FRAME_HEADER: usize = 9; -const METHOD_NONE: u8 = 0x02; -const METHOD_LZ4: u8 = 0x82; -const METHOD_ZSTD: u8 = 0x90; - /// One prepared block body, owned until its INSERT is durable pub(crate) struct SealedBlock { prepared: PreparedBlock, - compression: CompressionChoice, } impl SealedBlock { @@ -46,7 +27,6 @@ impl SealedBlock { let codec = codec.filter(|_| compression != CompressionChoice::None); Ok(Self { prepared: bb.prepare(compression.to_wire(), codec, alloc)?, - compression, }) } @@ -62,88 +42,6 @@ impl SealedBlock { pub(crate) fn allocated_bytes(&self) -> usize { self.prepared.capacity() } - - /// Decode the block, decompressing into `scratch`; the block owns its - /// columns, so `scratch` is free again once this returns - pub(crate) fn open( - &self, - alloc: Allocator, - scratch: &mut Vec, - ) -> Result { - let bytes = self.prepared.as_bytes(); - let body = match self.compression { - CompressionChoice::None => bytes, - _ => { - unframe(bytes, scratch)?; - &scratch[..] - } - }; - if body.len() != self.raw_len() { - return Err(EmitterError::Type(format!( - "sealed block body {} bytes, sealed {}", - body.len(), - self.raw_len() - ))); - } - let mut io = SliceIo::new(body); - let mut reader = BlockReader::new(io.as_mut(), alloc, OPTS)?; - reader - .read()? - .filter(|b| b.n_rows() == self.prepared.n_rows()) - .ok_or_else(|| EmitterError::Type("sealed block decoded short".into())) - } -} - -/// Concatenate decompressed frames of `bytes` into `out` -fn unframe(mut bytes: &[u8], out: &mut Vec) -> Result<(), EmitterError> { - let corrupt = |what: &str| EmitterError::Type(format!("sealed block frame {what}")); - out.clear(); - while !bytes.is_empty() { - let header = bytes - .get(CHECKSUM..CHECKSUM + FRAME_HEADER) - .ok_or_else(|| corrupt("header short"))?; - let size = |at: usize| u32::from_le_bytes(header[at..at + 4].try_into().unwrap()) as usize; - let (stored, raw) = (size(1), size(5)); - let data = stored - .checked_sub(FRAME_HEADER) - .and_then(|n| bytes.get(CHECKSUM + FRAME_HEADER..CHECKSUM + FRAME_HEADER + n)) - .ok_or_else(|| corrupt("body short"))?; - let at = out.len(); - out.resize(at + raw, 0); - let dst = &mut out[at..]; - let n = match header[0] { - METHOD_LZ4 => lz4::block::decompress_to_buffer(data, Some(raw as i32), dst) - .map_err(|e| corrupt(&format!("lz4: {e}")))?, - METHOD_ZSTD => zstd::bulk::decompress_to_buffer(data, dst) - .map_err(|e| corrupt(&format!("zstd: {e}")))?, - METHOD_NONE if data.len() == raw => { - dst.copy_from_slice(data); - raw - } - m => return Err(corrupt(&format!("method {m:#x}"))), - }; - if n != raw { - return Err(corrupt("size")); - } - bytes = &bytes[CHECKSUM + stored..]; - } - Ok(()) -} - -/// Rebuild a builder over a decoded block's columns, for `send_data` -pub(crate) fn builder_of(block: &Block) -> Result, EmitterError> { - let mut bb = BlockBuilder::new(); - for i in 0..block.n_columns() { - let name = block - .column_name(i) - .and_then(|n| std::str::from_utf8(n).ok()) - .ok_or_else(|| EmitterError::Type(format!("sealed block column {i} name")))?; - let (Some(ty), Some(col)) = (block.column_type(i), block.column(i)) else { - return Err(EmitterError::Type(format!("sealed block column {name}"))); - }; - bb.append_column(name, ty, col)?; - } - Ok(bb) } #[cfg(test)] @@ -186,39 +84,8 @@ mod tests { .unwrap() } - fn check(strings: &[Option<&[u8]>], compression: CompressionChoice) -> SealedBlock { - let sealed = seal_rows(strings, compression); - let mut scratch = Vec::new(); - let block = sealed.open(alloc(), &mut scratch).unwrap(); - assert_eq!(block.n_rows(), strings.len()); - let text = block.column(1).unwrap(); - let nulls = text.null_map().unwrap(); - let (offsets, data) = text.nullable_inner().unwrap().string().unwrap(); - let mut start = 0; - for (i, s) in strings.iter().enumerate() { - let end = offsets[i] as usize; - assert_eq!(nulls[i] == 1, s.is_none()); - assert_eq!(&data[start..end], s.unwrap_or_default()); - start = end; - } - let (width, ids) = block.column(0).unwrap().fixed().unwrap(); - assert_eq!(width, 8); - assert_eq!(ids.len(), 8 * strings.len()); - // Rebuilt for a refusing connection, the block prepares identically - let codec = compression.build_codec().unwrap(); - let again = SealedBlock::seal( - &builder_of(&block).unwrap(), - compression, - codec.as_ref().map(|c| c.as_ref()), - alloc(), - ) - .unwrap(); - assert_eq!(again.prepared().as_bytes(), sealed.prepared().as_bytes()); - sealed - } - #[test] - fn blocks_round_trip_through_every_codec() { + fn blocks_seal_in_every_codec() { let big = vec![b'x'; 1 << 20]; let rows: Vec> = vec![Some(b"a"), None, Some(b""), Some(&big), None]; for compression in [ @@ -226,7 +93,7 @@ mod tests { CompressionChoice::Lz4, CompressionChoice::Zstd, ] { - let sealed = check(&rows, compression); + let sealed = seal_rows(&rows, compression); if compression == CompressionChoice::None { assert_eq!(sealed.prepared().as_bytes().len(), sealed.raw_len()); } else { @@ -237,47 +104,4 @@ mod tests { } } } - - #[test] - fn incompressible_body_round_trips() { - let mut x = 0x9E37_79B9_7F4A_7C15u64; - let noise: Vec = (0..200 << 10) - .map(|_| { - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - x as u8 - }) - .collect(); - for compression in [CompressionChoice::Lz4, CompressionChoice::Zstd] { - check(&[Some(&noise)], compression); - } - } - - #[test] - fn corrupt_frames_fail_to_open() { - let sealed = seal_rows(&[Some(&b"abc"[..]); 100], CompressionChoice::Lz4); - let bytes = sealed.prepared().as_bytes(); - let mut out = Vec::new(); - unframe(bytes, &mut out).unwrap(); - assert_eq!(out.len(), sealed.raw_len()); - assert!(unframe(&bytes[..bytes.len() - 1], &mut out).is_err()); - assert!(unframe(&bytes[..CHECKSUM + 4], &mut out).is_err()); - let mut method = bytes.to_vec(); - method[CHECKSUM] = 0x7f; - assert!(unframe(&method, &mut out).is_err()); - let mut sizes = bytes.to_vec(); - sizes[CHECKSUM + 5] ^= 1; - assert!(unframe(&sizes, &mut out).is_err()); - let none = [ - &[0u8; CHECKSUM][..], - &[METHOD_NONE, 12, 0, 0, 0, 3, 0, 0, 0], - b"xyz", - ] - .concat(); - unframe(&none, &mut out).unwrap(); - assert_eq!(out, b"xyz"); - let short = [&none[..CHECKSUM + 5], &[4, 0, 0, 0], b"xyz"].concat(); - assert!(unframe(&short, &mut out).is_err()); - } } diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index ea918c91..c0e856e3 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -354,7 +354,7 @@ pub async fn spawn_with_config( resolver::ResolverOptions { oracle, retry: emitter.retry.clone(), - codec: emitter.conn.compression, + codec: emitter.compression, stats: stats.clone(), fatal: fatal.clone(), config_rx: config_rx.clone(), diff --git a/src/ops/metrics.rs b/src/ops/metrics.rs index 6a9fea32..4f8c4c78 100644 --- a/src/ops/metrics.rs +++ b/src/ops/metrics.rs @@ -416,10 +416,6 @@ snapshot! { "InsertBatches an inserter finished draining to ClickHouse. insertbatch_batches_out - inserter_batches_in is the live backlog; their rates show inserter-pool saturation.", counter inserter_ch_seconds_total: f64 = "Cumulative inserter time inside the ClickHouse INSERT round trip. Against inserter_pool_size x uptime this is CH-side utilization.", - counter inserter_rebuilds_total: u64 = - "Sealed blocks decoded and rebuilt because a connection negotiated other compression or an older revision than they were prepared for.", - counter inserter_rebuild_seconds_total: f64 = - "Cumulative inserter time decoding and rebuilding sealed blocks.", counter resident_seal_seconds_total: f64 = "Cumulative resolver time serializing and compressing sealed Native blocks.", counter resident_raw_bytes_total: u64 = @@ -430,8 +426,6 @@ snapshot! { "Bytes held in insert tail output pools: queued column slabs, sealed blocks awaiting acknowledgement, and seal workspace.", gauge batcher_open_bytes: u64 = "Allocated column slab bytes in batches still open for more rows.", - gauge inserter_scratch_bytes: u64 = - "Allocated inserter buffers for decoding sealed blocks a connection cannot send as prepared.", counter oracle_resolve_seconds_total: f64 = "Cumulative resolver time inside the oracle round trip. Overlaps inserter_ch_seconds_total, so the larger of the two is the tail's limiter.", counter process_cpu_seconds_total: f64 = diff --git a/src/toast/resolver.rs b/src/toast/resolver.rs index 78813087..3912066b 100644 --- a/src/toast/resolver.rs +++ b/src/toast/resolver.rs @@ -8,7 +8,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use async_trait::async_trait; -use clickhouse_c::{Allocator, Block, BlockBuilder, ColumnBuilder, Event, QueryOpts, TypeAst}; +use clickhouse_c::{ + Allocator, Block, BlockBuilder, ColumnBuilder, Event, QueryOpts, QuerySetting, TypeAst, +}; use futures::stream::StreamExt; use thiserror::Error; use tokio::sync::Mutex; @@ -40,7 +42,7 @@ pub const CHUNK_PUT_BATCH: usize = 65_536; pub const CHUNK_PUT_BYTES: usize = 64 << 20; /// Fetch result rows per block: bounds one block's buffer to ~2 MiB at /// `TOAST_MAX_CHUNK_SIZE`, ordering validated across block boundaries -const FETCH_BLOCK_ROWS: usize = 1024; +const FETCH_BLOCK_ROWS: QuerySetting = QuerySetting::new("max_block_size", "1024"); const FETCH_QUERY_IDS: usize = 1024; const CHUNK_ID_INDEX_FP: f64 = 0.000001; @@ -767,8 +769,7 @@ impl ClickHouseChunkStore { SELECT `blkno`, `offnum`, 0, 0, '', {commit_lsn}, 1\n\ FROM {table}\n\ GROUP BY `blkno`, `offnum`\n\ - HAVING max(`_lsn`) <= {marker_lsn} AND argMax(`_is_deleted`, `_lsn`) = 0\n\ - SETTINGS optimize_aggregation_in_order = 1" + HAVING max(`_lsn`) <= {marker_lsn} AND argMax(`_is_deleted`, `_lsn`) = 0" ) } @@ -801,8 +802,7 @@ impl ClickHouseChunkStore { )\n\ WHERE `chunk_id` IN ({ids}) AND `dead` = 0\n\ GROUP BY `chunk_id`, `chunk_seq`\n\ - ORDER BY `chunk_id`, `chunk_seq`\n\ - SETTINGS max_block_size = {FETCH_BLOCK_ROWS}" + ORDER BY `chunk_id`, `chunk_seq`" ) } @@ -810,6 +810,7 @@ impl ClickHouseChunkStore { &self, state: &mut ChState, sql: &str, + opts: &QueryOpts<'_>, bb: Option<&BlockBuilder<'_>>, ) -> Result<(), EmitterError> { let insert_timeout = self.dest.current().insert_timeout; @@ -819,7 +820,7 @@ impl ClickHouseChunkStore { |e| is_retryable(e) && !is_missing_mirror(e), |mut client| async move { let result = with_timeout(insert_timeout, async { - client.send_query(sql, &QueryOpts::new()).await?; + client.send_query(sql, opts).await?; if let Some(bb) = bb { client.send_data(bb).await?; client.send_data_end().await?; @@ -846,7 +847,8 @@ impl ClickHouseChunkStore { for ((db_oid, relid), group) in by_mirror { if !state.created.contains(&(db_oid, relid)) { let create = self.create_sql(db_oid, relid); - self.exec_write(state, &create, None).await?; + self.exec_write(state, &create, &QueryOpts::new(), None) + .await?; state.created.insert((db_oid, relid)); } let n = group.len(); @@ -894,7 +896,8 @@ impl ClickHouseChunkStore { bb.append("_lsn", u64_ast.view(), &lsn_col)?; bb.append("_is_deleted", u8_ast.view(), &is_deleted_col)?; let insert = self.insert_sql(db_oid, relid); - self.exec_write(state, &insert, Some(&bb)).await?; + self.exec_write(state, &insert, &QueryOpts::new(), Some(&bb)) + .await?; } Ok(()) } @@ -903,6 +906,7 @@ impl ClickHouseChunkStore { &self, state: &mut ChState, sql: &str, + opts: &QueryOpts<'_>, parse: impl Fn(&Block, &mut A) -> Result<(), EmitterError>, ) -> Result { let insert_timeout = self.dest.current().insert_timeout; @@ -912,7 +916,7 @@ impl ClickHouseChunkStore { |e| is_retryable(e) && !is_missing_mirror(e), |mut client| async { let result = with_timeout(insert_timeout, async { - client.send_query(sql, &QueryOpts::new()).await?; + client.send_query(sql, opts).await?; let mut out = A::default(); loop { match client.recv_event().await? { @@ -957,7 +961,8 @@ impl ClickHouseChunkStore { ) -> Result, ChunkStoreError> { let sql = self.fetch_sql(db_oid, toast_relid, ids, max_lsn); let mut state = self.slot().await; - self.query_locked(&mut state, &sql, |block, out| { + let opts = QueryOpts::new().settings(&[FETCH_BLOCK_ROWS]); + self.query_locked(&mut state, &sql, &opts, |block, out| { read_value_block(block, expected, out) }) .await @@ -1090,7 +1095,7 @@ impl ChunkStore for ClickHouseChunkStore { async fn truncate_mirror(&self, db_oid: u32, toast_relid: u32) -> Result<(), ChunkStoreError> { let sql = self.truncate_sql(db_oid, toast_relid); let mut state = self.slot_for(toast_relid).await; - self.exec_write(&mut state, &sql, None) + self.exec_write(&mut state, &sql, &QueryOpts::new(), None) .await .map_err(|e| ChunkStoreError::Clickhouse(e.to_string())) } @@ -1104,7 +1109,9 @@ impl ChunkStore for ClickHouseChunkStore { ) -> Result<(), ChunkStoreError> { let sql = self.rewrite_barrier_sql(db_oid, toast_relid, marker_lsn, commit_lsn); let mut state = self.slot_for(toast_relid).await; - match self.exec_write(&mut state, &sql, None).await { + let in_order = [QuerySetting::new("optimize_aggregation_in_order", "1")]; + let opts = QueryOpts::new().settings(&in_order); + match self.exec_write(&mut state, &sql, &opts, None).await { Ok(()) => Ok(()), // Never-populated mirror: no table, nothing lived, nothing to // tombstone @@ -2981,8 +2988,7 @@ mod tests { SELECT `blkno`, `offnum`, 0, 0, '', 16384, 1\n\ FROM `wh`.`pg_toast_5_16500`\n\ GROUP BY `blkno`, `offnum`\n\ - HAVING max(`_lsn`) <= 8192 AND argMax(`_is_deleted`, `_lsn`) = 0\n\ - SETTINGS optimize_aggregation_in_order = 1" + HAVING max(`_lsn`) <= 8192 AND argMax(`_is_deleted`, `_lsn`) = 0" ); assert_eq!( @@ -3005,8 +3011,7 @@ mod tests { )\n\ WHERE `chunk_id` IN (7, 9) AND `dead` = 0\n\ GROUP BY `chunk_id`, `chunk_seq`\n\ - ORDER BY `chunk_id`, `chunk_seq`\n\ - SETTINGS max_block_size = 1024" + ORDER BY `chunk_id`, `chunk_seq`" ); } diff --git a/tests/bootstrap_pipeline_ch.rs b/tests/bootstrap_pipeline_ch.rs index 35252d5f..e6b812b0 100644 --- a/tests/bootstrap_pipeline_ch.rs +++ b/tests/bootstrap_pipeline_ch.rs @@ -135,9 +135,9 @@ async fn bootstrap_tail_fans_out_n2() { host: "127.0.0.1".into(), port: slot.ch_tcp, database: "walshadow_test".into(), - compression: CompressionChoice::None, ..Default::default() }, + compression: CompressionChoice::None, row_budget: 4, flush_timeout: Duration::from_millis(50), ..Default::default() diff --git a/tests/bootstrap_window_leg_ch.rs b/tests/bootstrap_window_leg_ch.rs index e802ec71..d822a95e 100644 --- a/tests/bootstrap_window_leg_ch.rs +++ b/tests/bootstrap_window_leg_ch.rs @@ -35,9 +35,9 @@ fn emitter(port: u16) -> EmitterConfig { host: "127.0.0.1".into(), port, database: "walshadow_test".into(), - compression: CompressionChoice::None, ..Default::default() }, + compression: CompressionChoice::None, flush_timeout: Duration::from_millis(50), ..Default::default() }; diff --git a/tests/ch_ddl_apply_ch.rs b/tests/ch_ddl_apply_ch.rs index 01328c9e..8178ab78 100644 --- a/tests/ch_ddl_apply_ch.rs +++ b/tests/ch_ddl_apply_ch.rs @@ -78,9 +78,9 @@ async fn ddl_applies_and_refuses_against_clickhouse() { host: "127.0.0.1".into(), port: slot.ch_tcp, database: DB.into(), - compression: CompressionChoice::None, ..Default::default() }, + compression: CompressionChoice::None, ..Default::default() }; let resolved = Arc::new(ResolvedConfig::default()); diff --git a/tests/common/inproc_harness.rs b/tests/common/inproc_harness.rs index 4fbc958d..dc62902f 100644 --- a/tests/common/inproc_harness.rs +++ b/tests/common/inproc_harness.rs @@ -763,9 +763,9 @@ async fn build_pipeline_inner( host: "127.0.0.1".into(), port: ch_tcp_port, database: ch_database.into(), - compression: CompressionChoice::Lz4, ..Default::default() }, + compression: CompressionChoice::Lz4, ..Default::default() }; for spec in mappings { diff --git a/tests/emitter_budget_flush.rs b/tests/emitter_budget_flush.rs index 8caaf5a9..d6b01293 100644 --- a/tests/emitter_budget_flush.rs +++ b/tests/emitter_budget_flush.rs @@ -155,9 +155,9 @@ async fn budget_trips_seal_complete_inserts() { host: "127.0.0.1".into(), port: slot.ch_tcp, database: "walshadow_test".into(), - compression: CompressionChoice::Lz4, ..Default::default() }, + compression: CompressionChoice::Lz4, row_budget: 2, ..Default::default() }; @@ -295,9 +295,9 @@ async fn stalled_inserters_backpressure_many_tables_then_drain() { host: "127.0.0.1".into(), port, database: "walshadow_test".into(), - compression: CompressionChoice::Lz4, ..Default::default() }, + compression: CompressionChoice::Lz4, row_budget: 16, byte_budget: 16 << 10, ..Default::default() diff --git a/tests/emitter_native_types.rs b/tests/emitter_native_types.rs index 44689ed7..fd353745 100644 --- a/tests/emitter_native_types.rs +++ b/tests/emitter_native_types.rs @@ -113,9 +113,9 @@ async fn native_numeric_time_timetz_round_trip() { host: "127.0.0.1".into(), port: slot.ch_tcp, database: "walshadow_test".into(), - compression: CompressionChoice::Lz4, ..Default::default() }, + compression: CompressionChoice::Lz4, ..Default::default() }; diff --git a/tests/emitter_tls.rs b/tests/emitter_tls.rs index 08b7ffd7..44e1ff11 100644 --- a/tests/emitter_tls.rs +++ b/tests/emitter_tls.rs @@ -337,9 +337,9 @@ async fn emitter_tls_round_trip() { port: ch.secure_port, database: "walshadow_test".into(), secure: true, - compression: CompressionChoice::Lz4, ..Default::default() }, + compression: CompressionChoice::Lz4, tls_config: Some(ch.pinned_config()), ..Default::default() }; diff --git a/tests/pipeline_tail.rs b/tests/pipeline_tail.rs index e3162431..f25c6f03 100644 --- a/tests/pipeline_tail.rs +++ b/tests/pipeline_tail.rs @@ -18,9 +18,9 @@ fn emitter(port: u16) -> EmitterConfig { host: "127.0.0.1".into(), port, database: "walshadow_test".into(), - compression: CompressionChoice::Lz4, ..Default::default() }, + compression: CompressionChoice::Lz4, ..Default::default() } } diff --git a/tests/tee_cdc.rs b/tests/tee_cdc.rs index 7a203c2f..aae43252 100644 --- a/tests/tee_cdc.rs +++ b/tests/tee_cdc.rs @@ -1,8 +1,6 @@ //! Tee end-to-end: one source table feeds a `ReplacingMergeTree` primary, a //! `MergeTree` audit log, and a replica in another database. Rows, deletes -//! and `ADD COLUMN` must reach all three. Named instances differ in -//! compression, so one INSERT sends prepared blocks to one and rebuilds -//! them for other +//! and `ADD COLUMN` must reach all three #![cfg(target_os = "linux")] @@ -105,7 +103,6 @@ async fn run(named: bool) { host: "127.0.0.1".into(), port: remote_ports.ch_tcp, database: "walshadow_test".into(), - compression: walshadow::ch::CompressionChoice::None, ..Default::default() }, ); @@ -177,11 +174,6 @@ async fn run(named: bool) { .wait_for_replay(target, Duration::from_secs(30)) .expect("shadow replay"); assert!(observed >= target); - let rebuilds = pipeline - .stats - .inserter_rebuilds - .load(std::sync::atomic::Ordering::Relaxed); - assert_eq!(rebuilds > 0, named, "rebuilds {rebuilds}"); pipeline.shutdown().await.expect("pipeline drains clean"); let live = "SELECT arrayStringConcat(groupArray(concat(toString(id), ':', body, ':', \ From 32b3c4de6cec7c774ac534f0db0e3dd2931eb44e Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:59:28 +0000 Subject: [PATCH 19/20] remove instrumentation, use perf. move bench functions to bench --- bench/README.md | 6 +- benches/bootstrap_pump.rs | 25 +------ benches/common/alloc_meter.rs | 119 ---------------------------------- benches/spool_read.rs | 87 ++----------------------- benches/xact_spill.rs | 22 +------ src/decode/datum.rs | 21 ------ src/emit/ch_emitter.rs | 21 ------ 7 files changed, 11 insertions(+), 290 deletions(-) delete mode 100644 benches/common/alloc_meter.rs diff --git a/bench/README.md b/bench/README.md index dfe582c5..7b1b6237 100644 --- a/bench/README.md +++ b/bench/README.md @@ -16,8 +16,6 @@ xid, LSN order, descriptor, and file cleanup. Write timing includes allocation and admission; read timing includes reader setup, verified drain, and unlink. Report spill bytes alongside throughput: final memory tails can avoid disk. Compare equal spill bytes when isolating I/O improvements. -`buffer` and `drain` report allocation count, bytes still live, and peak live -bytes for largest transaction, counted by a wrapping global allocator Files follow disposable-spill semantics: no fsync, immediate replay, then unlink. Results include page cache and codec costs; they do not measure sustained device @@ -48,9 +46,7 @@ Read `plan_total` for combined write and replay time per sample. For plans in memory, byte counts report allocated chunk bytes; for spilled plans, encoded file size. Route caching measures descriptor identity checks and reuse of cached references. Measurements exclude mapping resolution, WAL draining, TOAST I/O, -control execution, and PostgreSQL/ClickHouse I/O. `plan` reports allocations and -bytes retained by sealed plan; `replay` reports peak bytes while replaying it. -With `--mapped`, replayed rows must match a reference plan encoded from cells +control execution, and PostgreSQL/ClickHouse I/O. ## pgbench workload diff --git a/benches/bootstrap_pump.rs b/benches/bootstrap_pump.rs index 30deeac0..c7925e4d 100644 --- a/benches/bootstrap_pump.rs +++ b/benches/bootstrap_pump.rs @@ -32,11 +32,7 @@ // serializes on that pattern. A bench on a different allocator measures a // different program #[global_allocator] -static GLOBAL: alloc_meter::Metered = - alloc_meter::Metered::new(mimalloc::MiMalloc); - -#[path = "common/alloc_meter.rs"] -mod alloc_meter; +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -376,7 +372,6 @@ async fn bench_pump(shape: Shape) -> Report { parallelism: shape.parallelism, }); - let base = GLOBAL.reset_peak(); let started = Instant::now(); let (mut rx, pump) = spawn_greenfield_bootstrap(cfg, source, catalog.clone(), false); let drained = tokio::spawn(async move { @@ -399,19 +394,15 @@ async fn bench_pump(shape: Shape) -> Report { pump.await.unwrap().unwrap(); let (walked, digest) = drained.await.unwrap(); let elapsed = started.elapsed().as_secs_f64(); - let metered = GLOBAL.snapshot().since(base); assert_eq!(digest, expected, "walked values differ from written pages"); assert_eq!(walked, tuples); - let mut detail = pump_detail(&progress); - detail.push(("allocs", metered.allocs as f64)); - detail.push(("peak_mib", metered.peak as f64 / (1 << 20) as f64)); Report { label: "pump".into(), elapsed_secs: elapsed, tuples, bytes: progress.pump.bytes_tapped.load(Ordering::Relaxed), - detail, + detail: pump_detail(&progress), } } @@ -470,7 +461,6 @@ async fn bench_drain(shape: Shape) -> Report { }); let stats = Arc::new(walshadow::ch_emitter::EmitterStats::default()); - let base = GLOBAL.reset_peak(); let started = Instant::now(); let outcome = bootstrap::drain( tup_rx, @@ -489,7 +479,6 @@ async fn bench_drain(shape: Shape) -> Report { let elapsed = started.elapsed().as_secs_f64(); feeder.await.unwrap(); tail.close(msg_tx, ack).await; - let metered = GLOBAL.snapshot().since(base); assert_eq!(outcome.rows_routed, shape.tuples()); Report { @@ -498,11 +487,7 @@ async fn bench_drain(shape: Shape) -> Report { tuples: outcome.rows_routed, // Row payload only; the interesting rate here is rows, not bytes bytes: outcome.rows_routed * (4 * shape.columns + shape.text) as u64, - detail: vec![ - ("seqs", outcome.next_seq as f64), - ("allocs", metered.allocs as f64), - ("peak_mib", metered.peak as f64 / (1 << 20) as f64), - ], + detail: vec![("seqs", outcome.next_seq as f64)], } } @@ -532,7 +517,6 @@ async fn bench_load(shape: Shape) -> Report { flush_timeout: std::time::Duration::from_secs(1), }; - let base = GLOBAL.reset_peak(); let started = Instant::now(); let (msg_tx, ack, tail) = walshadow::pipeline::tail::spawn_discard(batcher, stats.clone(), fatal.clone()); @@ -556,7 +540,6 @@ async fn bench_load(shape: Shape) -> Report { .await .unwrap(); let elapsed = started.elapsed().as_secs_f64(); - let metered = GLOBAL.snapshot().since(base); assert_eq!(outcome.rows_routed, tuples); assert_eq!( stats.insertbatch_rows_in.load(Ordering::Relaxed), @@ -569,8 +552,6 @@ async fn bench_load(shape: Shape) -> Report { "batches", stats.insertbatch_batches_out.load(Ordering::Relaxed) as f64, )); - detail.push(("allocs", metered.allocs as f64)); - detail.push(("peak_mib", metered.peak as f64 / (1 << 20) as f64)); Report { label: "load".into(), elapsed_secs: elapsed, diff --git a/benches/common/alloc_meter.rs b/benches/common/alloc_meter.rs deleted file mode 100644 index 4b89062b..00000000 --- a/benches/common/alloc_meter.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Global allocator wrapper counting allocations and live bytes, so benches -//! report retained memory by stage instead of payload estimates - -use std::alloc::{GlobalAlloc, Layout}; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; - -pub struct Metered { - inner: A, - allocs: AtomicU64, - live: AtomicUsize, - peak: AtomicUsize, -} - -impl Metered { - pub const fn new(inner: A) -> Self { - Self { - inner, - allocs: AtomicU64::new(0), - live: AtomicUsize::new(0), - peak: AtomicUsize::new(0), - } - } - - fn grow(&self, n: usize) { - let live = self.live.fetch_add(n, Ordering::Relaxed) + n; - self.peak.fetch_max(live, Ordering::Relaxed); - } - - fn shrink(&self, n: usize) { - self.live.fetch_sub(n, Ordering::Relaxed); - } - - pub fn snapshot(&self) -> Meter { - Meter { - allocs: self.allocs.load(Ordering::Relaxed), - live: self.live.load(Ordering::Relaxed), - peak: self.peak.load(Ordering::Relaxed), - } - } - - /// Restart peak tracking from current live bytes - pub fn reset_peak(&self) -> Meter { - self.peak - .store(self.live.load(Ordering::Relaxed), Ordering::Relaxed); - self.snapshot() - } -} - -/// Requested sizes, not allocator size classes or RSS -#[derive(Clone, Copy, Debug)] -pub struct Meter { - pub allocs: u64, - pub live: usize, - pub peak: usize, -} - -impl Meter { - /// Allocations, bytes still live, and peak growth since `base` - pub fn since(self, base: Meter) -> Delta { - Delta { - allocs: self.allocs - base.allocs, - retained: self.live as i64 - base.live as i64, - peak: self.peak.saturating_sub(base.live), - } - } -} - -#[derive(Clone, Copy, Debug)] -pub struct Delta { - pub allocs: u64, - pub retained: i64, - pub peak: usize, -} - -impl std::fmt::Display for Delta { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "allocs={} retained_bytes={} peak_bytes={}", - self.allocs, self.retained, self.peak - ) - } -} - -// SAFETY: forwards to `inner`; counters never touch the returned memory -unsafe impl GlobalAlloc for Metered { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - let p = unsafe { self.inner.alloc(layout) }; - if !p.is_null() { - self.allocs.fetch_add(1, Ordering::Relaxed); - self.grow(layout.size()); - } - p - } - - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - let p = unsafe { self.inner.alloc_zeroed(layout) }; - if !p.is_null() { - self.allocs.fetch_add(1, Ordering::Relaxed); - self.grow(layout.size()); - } - p - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { self.inner.dealloc(ptr, layout) }; - self.shrink(layout.size()); - } - - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - let p = unsafe { self.inner.realloc(ptr, layout, new_size) }; - if !p.is_null() { - self.allocs.fetch_add(1, Ordering::Relaxed); - self.shrink(layout.size()); - self.grow(new_size); - } - p - } -} diff --git a/benches/spool_read.rs b/benches/spool_read.rs index 38f4a7dc..d60b91e2 100644 --- a/benches/spool_read.rs +++ b/benches/spool_read.rs @@ -15,10 +15,8 @@ use walrus::pg::walparser::{ BlockLocation, RmId, XLogRecord, XLogRecordBlock, XLogRecordBlockHeader, XLogRecordHeader, }; use walshadow::backup_page_walk::{BackfillTuple, TupleData}; -use walshadow::ch_emitter::EncodedChunk; use walshadow::datum::build; use walshadow::emit::route::RouteSnapshot; -use walshadow::heap_decoder::{DecodedHeap, DecodedTuple, DescribedHeap, HeapOp}; use walshadow::mapping::{ColumnMapping, TableMapping, TableTarget}; use walshadow::pipeline::plan_spool::{PlanItem, PlanWriter, ViewChange}; use walshadow::pipeline::planner::{PlanRouteView, Planner}; @@ -30,12 +28,8 @@ use walshadow::spool::DeferredSpool; use walshadow::toast::ToastResolver; use walshadow::xact_buffer::{DrainEntry, DrainHeap, DrainedBatch, OrderedEvent}; -#[path = "common/alloc_meter.rs"] -mod alloc_meter; - #[global_allocator] -static GLOBAL: alloc_meter::Metered = - alloc_meter::Metered::new(mimalloc::MiMalloc); +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; #[derive(Debug, Parser)] struct Args { @@ -101,7 +95,6 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let dir = tempfile::tempdir_in(&args.dir)?; let payload = "x".repeat(args.payload_bytes); let mut spool = DeferredSpool::new(dir.path().join("deferred.spool"), 1 << 20); - let base = GLOBAL.reset_peak(); let start = Instant::now(); let data = |i: usize| { let cells = [Some(build::int4(i as i32)), Some(build::text(&payload))]; @@ -127,8 +120,6 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { let bytes = spool.spooled_bytes(); let mut reader = spool.into_reader().await?; let write = start.elapsed(); - let spooled = GLOBAL.snapshot().since(base); - let base = GLOBAL.reset_peak(); let start = Instant::now(); let mut n = 0; while let Some(t) = reader.next().await? { @@ -138,10 +129,8 @@ async fn deferred(args: &Args, repeat: usize) -> anyhow::Result<()> { } reader.finish().await?; let read = start.elapsed(); - let replayed = GLOBAL.snapshot().since(base); assert_eq!(n, args.rows); report("deferred", repeat, n, bytes, write, read); - println!("deferred_alloc repeat={repeat} spool {spooled} replay {replayed}"); Ok(()) } @@ -279,24 +268,6 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { descriptor_valid_from: 1, } }; - let make_heap = |i: usize| { - let descriptor = &descriptors[i % descriptors.len()]; - DescribedHeap { - decoded: DecodedHeap { - rfn: descriptor.rfn, - xid: xid_of(i), - source_lsn: i as u64, - op: HeapOp::Insert, - new: Some(DecodedTuple { - columns: vec![Some(build::int4(i as i32)), Some(build::text(&payload))], - partial: false, - }), - old: None, - }, - descriptor: descriptor.clone(), - descriptor_valid_from: 1, - } - }; let control = |i: usize| { (args.control_every != 0 && i != 0 && i.is_multiple_of(args.control_every)).then(|| { DrainEntry::Catalog(SchemaEvent::Dropped { @@ -343,7 +314,6 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { } else { Vec::new() }; - let base = GLOBAL.reset_peak(); let start = Instant::now(); let sealed = if args.planner { let mut planner = Planner::create( @@ -389,71 +359,24 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { writer.seal(u64::MAX, 0)? }; let write = start.elapsed(); - let planned = GLOBAL.snapshot().since(base); let (size_bytes, spilled) = (sealed.size_bytes, sealed.path().is_some()); let routed = sealed.routed_rows; - let base = GLOBAL.reset_peak(); let start = Instant::now(); let mut rd = sealed.replay()?; let mut n = 0; let mut controls = 0; - let mut segments = Vec::new(); + let mut n_segments = 0; while let Some(item) = rd.next_item()? { match item { PlanItem::Segment(s) => { - let chunks = s.into_chunks()?; - n += chunks.iter().map(|c| c.rows()).sum::(); - segments.push(chunks); + n += s.into_chunks()?.iter().map(|c| c.rows()).sum::(); + n_segments += 1; } PlanItem::Control { .. } => controls += 1, PlanItem::Truncate { .. } => unreachable!(), } } let replay = start.elapsed(); - let replayed = GLOBAL.snapshot().since(base); - let n_segments = segments.len(); - // Encoded chunks match a reference plan encoding decoded rows - if args.mapped { - let mut reference = PlanWriter::create(dir.path().join("ref.spool"), u64::MAX, u64::MAX) - .with_segment_limits(usize::MAX, usize::MAX); - let mut tables = HashMap::new(); - for i in 0..args.rows { - if control(i).is_some() { - tables.clear(); - } - let rel = &descriptors[i % descriptors.len()]; - let table = *tables - .entry(rel.oid) - .or_insert_with(|| reference.table(rel, 1, || view.route.clone())); - reference.push_cells(make_heap(i).decoded, table)?; - } - // Chunk boundaries differ between plans; compare rows per relation - let merged = |chunks: Vec| { - let mut by_rel: HashMap = HashMap::new(); - for c in chunks { - match by_rel.entry(c.rel_oid()) { - std::collections::hash_map::Entry::Occupied(mut e) => { - assert!(e.get_mut().merge(c), "one layout per relation"); - } - std::collections::hash_map::Entry::Vacant(e) => { - e.insert(c); - } - } - } - let mut out: Vec<_> = by_rel.into_iter().map(|(k, c)| (k, c.to_bytes())).collect(); - out.sort(); - out - }; - let mut rrd = reference.seal(u64::MAX, 0)?.replay()?; - let mut want = Vec::new(); - while let Some(PlanItem::Segment(s)) = rrd.next_item()? { - want.extend(s.into_chunks()?); - } - assert!( - merged(want) == merged(segments.into_iter().flatten().collect()), - "encoded rows differ from reference" - ); - } assert_eq!(n, if args.mapped { args.rows } else { 0 }); assert_eq!(routed, n as u64); assert_eq!( @@ -484,7 +407,7 @@ async fn plan(args: &Args, repeat: usize) -> anyhow::Result<()> { + (args.rows % interval).min(descriptors.len()); assert_eq!(view.calls, expected_calls); println!( - "plan_total repeat={repeat} total_s={:.6} route_calls={} tables={} routed={routed} segments={} spilled={} plan {planned} replay {replayed}", + "plan_total repeat={repeat} total_s={:.6} route_calls={} tables={} routed={routed} segments={} spilled={}", (write + replay).as_secs_f64(), view.calls, rd.tables.len(), diff --git a/benches/xact_spill.rs b/benches/xact_spill.rs index 32a0856c..25adad1f 100644 --- a/benches/xact_spill.rs +++ b/benches/xact_spill.rs @@ -22,12 +22,8 @@ use walshadow::spill::{DescribedInput, HeapInput, RawRecord}; use walshadow::toast::ToastResolver; use walshadow::xact_buffer::{DrainEntry, DrainHeap, StashResolved, XactBuffer, XactBufferConfig}; -#[path = "common/alloc_meter.rs"] -mod alloc_meter; - #[global_allocator] -static GLOBAL: alloc_meter::Metered = - alloc_meter::Metered::new(mimalloc::MiMalloc); +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; #[derive(Debug, Parser)] struct Args { @@ -158,9 +154,7 @@ async fn run(args: Args) -> anyhow::Result<()> { let mut read_time = Duration::ZERO; let mut spill_bytes = 0; let mut peak_memory = 0; - let (mut buffered, mut drained) = (Vec::new(), Vec::new()); for xid in 1..=args.transactions { - let base = GLOBAL.reset_peak(); let start = Instant::now(); let record = record(xid); for row in 0..args.rows { @@ -181,8 +175,6 @@ async fn run(args: Args) -> anyhow::Result<()> { } write_time += start.elapsed(); spill_bytes += buffer.stats().spill_bytes_active; - buffered.push(GLOBAL.snapshot().since(base)); - let base = GLOBAL.reset_peak(); let start = Instant::now(); let commit_lsn = u64::from(xid + 1) * (args.rows as u64 + 1); let resolved = if args.stashed { @@ -237,18 +229,10 @@ async fn run(args: Args) -> anyhow::Result<()> { drain.finish().await?; buffer.resume_safe_lsn(walshadow::pos::Pos::new(commit_lsn)); read_time += start.elapsed(); - drained.push(GLOBAL.snapshot().since(base)); } - // Largest transaction footprint, buffered rows still owned by buffer - let worst = |d: &[alloc_meter::Delta]| { - d.iter() - .copied() - .max_by_key(|d| d.peak) - .expect("one transaction") - }; let rows = args.rows as f64 * f64::from(args.transactions); println!( - "repeat={} rows={} spill_bytes={} evictions={} peak_buffer_bytes={} write_s={:.6} read_s={:.6} rows_per_s={:.0} spill_write_mib_s={:.2} spill_read_mib_s={:.2} buffer {} drain {}", + "repeat={} rows={} spill_bytes={} evictions={} peak_buffer_bytes={} write_s={:.6} read_s={:.6} rows_per_s={:.0} spill_write_mib_s={:.2} spill_read_mib_s={:.2}", repeat + 1, rows as u64, spill_bytes, @@ -259,8 +243,6 @@ async fn run(args: Args) -> anyhow::Result<()> { rows / (write_time + read_time).as_secs_f64(), spill_bytes as f64 / (1 << 20) as f64 / write_time.as_secs_f64(), spill_bytes as f64 / (1 << 20) as f64 / read_time.as_secs_f64(), - worst(&buffered), - worst(&drained), ); assert_eq!(buffer.stats().bytes_in_memory, 0); assert_eq!(buffer.stats().spill_bytes_active, 0); diff --git a/src/decode/datum.rs b/src/decode/datum.rs index 82365ffe..a31b1c80 100644 --- a/src/decode/datum.rs +++ b/src/decode/datum.rs @@ -82,20 +82,11 @@ impl Datum { #[cfg(any(test, feature = "test-support"))] pub mod build { use super::Datum; - use crate::decode::heap_decoder::ToastPointer; - - pub fn null() -> Datum { - Datum::Null - } pub fn bool_(v: bool) -> Datum { Datum::fixed(&[u8::from(v)]) } - pub fn char_(v: i8) -> Datum { - Datum::fixed(&v.to_le_bytes()) - } - pub fn int2(v: i16) -> Datum { Datum::fixed(&v.to_le_bytes()) } @@ -108,14 +99,6 @@ pub mod build { Datum::fixed(&v.to_le_bytes()) } - pub fn float4(v: f32) -> Datum { - Datum::fixed(&v.to_le_bytes()) - } - - pub fn float8(v: f64) -> Datum { - Datum::fixed(&v.to_le_bytes()) - } - pub fn oid(v: u32) -> Datum { Datum::fixed(&v.to_le_bytes()) } @@ -162,10 +145,6 @@ pub mod build { Datum::Varlena(crate::decode::codecs::numeric_from_text(v.as_ref()).expect("numeric text")) } - pub fn external(p: ToastPointer) -> Datum { - Datum::External(p) - } - /// `typoutput` text pub fn out_text(v: impl AsRef) -> Datum { Datum::Text(v.as_ref().as_bytes().to_vec()) diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index a890b9db..779023a5 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -2642,27 +2642,6 @@ impl EncodedChunk { .sum::() } - pub fn rel_oid(&self) -> u32 { - self.rel.oid - } - - /// Append `other`'s rows; `false` leaves both untouched when layouts differ - pub fn merge(&mut self, other: Self) -> bool { - let same = self.buffers.len() == other.buffers.len() - && self - .buffers - .iter() - .zip(&other.buffers) - .all(|(a, b)| a.same_shape(b)); - if same { - for (buf, more) in self.buffers.iter_mut().zip(other.buffers) { - buf.extend(more); - } - self.rows += other.rows; - } - same - } - /// Wire content as a plan spool frame body pub fn to_bytes(&self) -> Vec { let mut out = Vec::new(); From 9f77ac5d12711a2884d71fa5f6b4218217d19d3f Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:12:57 +0000 Subject: [PATCH 20/20] coverage --- src/backfill/copy_backfill.rs | 26 ++++++++++ src/catalog/type_bridge.rs | 46 ++++++++++++++++-- src/decode/heap_decoder.rs | 84 +++++++++++++++++++++++++++++++-- src/emit/ch_emitter.rs | 9 +++- src/emit/pipeline/plan_spool.rs | 57 ++++++++++++++++++++++ src/emit/pipeline/planner.rs | 7 +-- src/emit/pipeline/tail.rs | 44 +++++++++++++++++ src/ops/oracle.rs | 10 ---- src/runtime_config.rs | 30 ++++++++++++ src/xact/xact_buffer.rs | 34 ++++--------- 10 files changed, 300 insertions(+), 47 deletions(-) diff --git a/src/backfill/copy_backfill.rs b/src/backfill/copy_backfill.rs index 86bd2f06..6de32154 100644 --- a/src/backfill/copy_backfill.rs +++ b/src/backfill/copy_backfill.rs @@ -2499,6 +2499,12 @@ mod tests { const TSVECTOROID: u32 = 3614; let tsvector = hex("000000010000000000000e1e0000000100000001000000080000000178000000"); assert_eq!(missing_from_send(TSVECTOROID, &tsvector), None); + // Two elements, then a length word disagreeing with the bytes + let text = hex("0000000100000000000000190000000100000001000000026869"); + let mut two = text.clone(); + two[15] = 2; + assert_eq!(missing_from_send(TEXTOID, &two), None); + assert_eq!(missing_from_send(TEXTOID, &text[..text.len() - 1]), None); } #[test] @@ -2550,6 +2556,26 @@ mod tests { assert!(store(WireKind::Numeric, &[0, 1, 0]).is_err()); // Cast columns store text output verbatim, invalid UTF-8 included assert_eq!(store(WireKind::Cast, &[0xFF]).unwrap(), [0xFF]); + assert!(store(WireKind::Numeric, &[0, 1, 0, 0, 0, 0, 0, 0]).is_err()); + assert!(store(WireKind::Numeric, &[0, 0, 0, 0, 0x12, 0x34, 0, 0]).is_err()); + use WireKind::*; + for (kind, width) in [ + (Char, 1), + (Int2, 2), + (Oid, 4), + (Float4, 4), + (Oid8, 8), + (Time, 8), + ] { + let raw: Vec = (1..=width).collect(); + let want: Vec = raw.iter().rev().copied().collect(); + assert_eq!(store(kind, &raw).unwrap(), want, "{kind:?}"); + assert!(store(kind, &raw[1..]).is_err(), "{kind:?}"); + } + let uuid: Vec = (0..16).collect(); + assert_eq!(store(Uuid, &uuid).unwrap(), uuid); + assert!(store(Uuid, &uuid[1..]).is_err()); + assert!(store(Name, &[b'n'; NAMEDATALEN]).is_err()); } /// PostgreSQL binary COPY uses each type's `typsend` format diff --git a/src/catalog/type_bridge.rs b/src/catalog/type_bridge.rs index ee266087..3de45b33 100644 --- a/src/catalog/type_bridge.rs +++ b/src/catalog/type_bridge.rs @@ -959,7 +959,13 @@ mod tests { OID8OID, OIDOID, TIDOID, }; let cases = [ - (CHAROID, 1, build::char_(-3), "Int8", "-3"), + ( + CHAROID, + 1, + Datum::fixed(&(-3i8).to_le_bytes()), + "Int8", + "-3", + ), (INT2OID, 2, build::int2(7), "Int16", "7"), (INT8OID, 8, build::int8(-1 << 40), "Int64", "-1099511627776"), (OIDOID, 4, build::oid(4_000_000_000), "UInt32", "4000000000"), @@ -970,11 +976,17 @@ mod tests { "UInt64", "18446744073709551615", ), - (FLOAT4OID, 4, build::float4(2.0), "Float32", "2.0"), + ( + FLOAT4OID, + 4, + Datum::fixed(&2.0f32.to_le_bytes()), + "Float32", + "2.0", + ), ( FLOAT8OID, 8, - build::float8(f64::NEG_INFINITY), + Datum::fixed(&f64::NEG_INFINITY.to_le_bytes()), "Float64", "-inf", ), @@ -1036,4 +1048,32 @@ mod tests { assert_eq!(pg_array_items(r#"{"a"x}"#), None); assert_eq!(pending_text_literal(r#"{"a"x}"#, "Array(String)"), None); } + + #[test] + fn fixed_cells_render_by_target_width() { + let fixed = |bytes: &[u8]| ColumnBuf::Fixed { + width: bytes.len(), + bytes: bytes.to_vec(), + }; + let cases: [(&[u8], &str, &str); 10] = [ + (&[0xFE], "Int8", "-2"), + (&(-3i16).to_le_bytes(), "Int16", "-3"), + (&(-5i64).to_le_bytes(), "Int64", "-5"), + (&[0xFE], "UInt8", "254"), + (&u16::MAX.to_le_bytes(), "UInt16", "65535"), + (&u32::MAX.to_le_bytes(), "UInt32", "4294967295"), + (&u64::MAX.to_le_bytes(), "UInt64", "18446744073709551615"), + (&1.5f32.to_le_bytes(), "Float32", "1.5"), + (&(-0.25f64).to_le_bytes(), "Float64", "-0.25"), + (&7i64.to_le_bytes(), "Time64(3)", "'00:00:00.007'"), + ]; + for (bytes, ch, want) in cases { + assert_eq!( + column_literal(&fixed(bytes), ch).as_deref(), + Some(want), + "{ch}" + ); + } + assert_eq!(column_literal(&fixed(&[1]), "IPv4"), None); + } } diff --git a/src/decode/heap_decoder.rs b/src/decode/heap_decoder.rs index 24df8134..d188f571 100644 --- a/src/decode/heap_decoder.rs +++ b/src/decode/heap_decoder.rs @@ -2497,6 +2497,80 @@ mod tests { assert!(missing_defaults_equivalent(&raw_ts, &text_ts)); } + #[test] + fn fast_default_equivalent_across_raw_and_text_name() { + let mut raw = rel_attr(1, "n", NAMEOID, 64, 'c'); + let name = build::name("nm"); + let Datum::Fixed(stored) = &name else { + panic!("name is fixed width"); + }; + raw.missing_default = Some(MissingDefault::Raw(raw_missing_array(NAMEOID, stored))); + let mut text = rel_attr(1, "n", NAMEOID, 64, 'c'); + text.missing_default = Some("nm".into()); + assert!(missing_defaults_equivalent(&raw, &text)); + + // Text the parser rejects reads as NULL on both sides + let mut bad = rel_attr(1, "x", INT4OID, 4, 'i'); + bad.missing_default = Some("x".into()); + assert!(missing_defaults_equivalent(&bad, &bad.clone())); + } + + /// Owned datums view and resolve back to themselves, holding their bytes + #[test] + fn owned_datums_round_trip_through_views() { + let att = rel_attr(1, "x", INT4OID, 4, 'i'); + let pointer = ToastPointer { + va_rawsize: 8, + va_extinfo: 4, + va_valueid: 1, + va_toastrelid: 9, + }; + let cases = [ + (Datum::Null, 0), + (build::int4(3), 4), + (build::text("ab"), 2), + ( + Datum::External(pointer), + std::mem::size_of::(), + ), + (Datum::Cstring(b"c".to_vec()), 1), + (Datum::BadTag(b"de".to_vec()), 2), + (Datum::Text(b"f".to_vec()), 1), + (Datum::Send(b"gh".to_vec()), 2), + ]; + for (d, bytes) in cases { + assert_eq!(resolve(&att, d.view()).as_ref(), Some(&d)); + assert_eq!(d.approx_bytes(), bytes, "{d:?}"); + } + } + + #[test] + fn text_datum_parses_local_types() { + let fixed = |b: &[u8]| TextDatum::Datum(Datum::fixed(b)); + let cases: [(u32, &str, TextDatum); 8] = [ + (CHAROID, "q", fixed(b"q")), + (INT2OID, "-2", fixed(&(-2i16).to_le_bytes())), + (FLOAT4OID, "1.5", fixed(&1.5f32.to_le_bytes())), + (FLOAT8OID, "-0.25", fixed(&(-0.25f64).to_le_bytes())), + (OIDOID, "16384", fixed(&16384u32.to_le_bytes())), + ( + MACADDR8OID, + "08:00:2b:01:02:03:04:05", + fixed(&[8, 0, 0x2b, 1, 2, 3, 4, 5]), + ), + ( + TIDOID, + "(1,2)", + fixed(&crate::decode::codecs::encode_tid(1, 2)), + ), + (INT2OID, "70000", TextDatum::Null), + ]; + for (oid, text, want) in cases { + assert_eq!(text_datum(oid, text.as_bytes()), want, "{oid} {text}"); + } + assert_eq!(text_datum(INT4OID, &[0xFF]), TextDatum::Opaque); + } + /// Raw and text both render, so jsonb equivalence is by document #[test] fn fast_default_equivalent_across_raw_and_text_jsonb() { @@ -2968,13 +3042,17 @@ mod tests { fn text_datum_scalar_matrix() { let d = TextDatum::Datum; let cases = [ - (CHAROID, "a", d(build::char_(b'a' as i8))), + (CHAROID, "a", d(Datum::fixed(b"a"))), (CHAROID, "", TextDatum::Null), (INT2OID, "-7", d(build::int2(-7))), (INT2OID, "70000", TextDatum::Null), - (FLOAT4OID, "1.5", d(build::float4(1.5))), + (FLOAT4OID, "1.5", d(Datum::fixed(&1.5f32.to_le_bytes()))), (FLOAT4OID, "x", TextDatum::Null), - (FLOAT8OID, "-2.25", d(build::float8(-2.25))), + ( + FLOAT8OID, + "-2.25", + d(Datum::fixed(&(-2.25f64).to_le_bytes())), + ), (FLOAT8OID, "x", TextDatum::Null), (OIDOID, "4294967295", d(build::oid(u32::MAX))), (OIDOID, "-1", TextDatum::Null), diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 779023a5..73431ae7 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -4472,7 +4472,14 @@ mod tests { #[test] fn column_buffers_reject_mismatched_shapes() { let mut fixed = buf("Int8"); - encode_value(&mut fixed, Codec::Le, &build::char_(-3), None, 0).unwrap(); + encode_value( + &mut fixed, + Codec::Le, + &Datum::fixed(&(-3i8).to_le_bytes()), + None, + 0, + ) + .unwrap(); fixed.append_default(); assert_eq!(format!("{fixed:?}"), "Fixed { width: 1, bytes_len: 2 }"); assert!(matches!( diff --git a/src/emit/pipeline/plan_spool.rs b/src/emit/pipeline/plan_spool.rs index 5c225d14..8ba1aa46 100644 --- a/src/emit/pipeline/plan_spool.rs +++ b/src/emit/pipeline/plan_spool.rs @@ -1300,6 +1300,63 @@ mod tests { }) } + /// Index-key equality as `datumIsEqual` judges stored bytes + #[test] + fn datums_differ_matrix() { + use crate::decode::heap_decoder::ToastPointer; + use crate::schema::FLOAT4OID; + let mut att = descriptor(1).attributes[0].clone(); + let one = 1i32.to_le_bytes(); + let two = 2i32.to_le_bytes(); + let toast = |va_valueid| ToastPointer { + va_rawsize: 8, + va_extinfo: 4, + va_valueid, + va_toastrelid: 9, + }; + let cases = [ + (DatumRef::Absent, DatumRef::Fixed(&one), false), + (DatumRef::Missing, DatumRef::Null, false), + (DatumRef::Fixed(&one), DatumRef::Missing, true), + (DatumRef::Null, DatumRef::Dropped, false), + (DatumRef::Fixed(&one), DatumRef::Fixed(&two), true), + (DatumRef::Text(b"a"), DatumRef::Varlena(b"a"), false), + (DatumRef::Cstring(b"a"), DatumRef::Varlena(b"b"), true), + ( + DatumRef::External(toast(1)), + DatumRef::External(toast(1)), + false, + ), + ( + DatumRef::External(toast(1)), + DatumRef::External(toast(2)), + true, + ), + (DatumRef::Null, DatumRef::Fixed(&one), true), + ]; + for (o, n, want) in cases { + assert_eq!(datums_differ(&att, o, n), want, "{o:?} vs {n:?}"); + } + att.type_oid = FLOAT4OID; + let f = |v: f32| v.to_le_bytes(); + let (zero, neg_zero, nan) = (f(0.0), f(-0.0), f(f32::NAN)); + assert!(!datums_differ( + &att, + DatumRef::Fixed(&zero), + DatumRef::Fixed(&neg_zero) + )); + assert!(datums_differ( + &att, + DatumRef::Fixed(&nan), + DatumRef::Fixed(&nan) + )); + assert!(datums_differ( + &att, + DatumRef::Fixed(&zero), + DatumRef::Fixed(&[0]) + )); + } + /// Delete marker splits off when UPDATE moves a key, or any column under /// `REPLICA IDENTITY FULL` without a primary key #[test] diff --git a/src/emit/pipeline/planner.rs b/src/emit/pipeline/planner.rs index 6db16017..c330694d 100644 --- a/src/emit/pipeline/planner.rs +++ b/src/emit/pipeline/planner.rs @@ -379,9 +379,7 @@ mod tests { use crate::emit::pipeline::plan_spool::PlanItem; use crate::mapping::{TableMapping, TableTarget}; use crate::schema::{INT4OID, RelAttr, RelDescriptor, RelName, SchemaEvent, TEXTOID}; - use crate::xact::xact_buffer::raw_fixtures::{ - inject_ordinary, int4_descriptor, multi_insert_raw, - }; + use crate::xact::xact_buffer::raw_fixtures::{int4_descriptor, multi_insert_raw}; use crate::xact::xact_buffer::{FailClosedReason, StashResolved, XactBuffer, XactBufferConfig}; use foldhash::{HashMap, HashMapExt, HashSet, HashSetExt}; use walrus::pg::walparser::RelFileNode; @@ -738,14 +736,13 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16610); - let rfn = rel.rfn; b.stash_raw(1, multi_insert_raw(1, 100, 16610, &[1, 2])) .await .unwrap(); let mut bad = multi_insert_raw(1, 120, 16610, &[3]); bad.main_data[0] = 0; // strip CONTAINS_NEW_TUPLE: ImageOnly on fold b.stash_raw(1, bad).await.unwrap(); - let stash = inject_ordinary(rfn, rel.clone()); + let stash = StashResolved::ordinary(1, rel.clone(), 0x50); let mut view = MapView { routes: HashMap::from_iter([(rel.rel_name.clone(), route())]), diff --git a/src/emit/pipeline/tail.rs b/src/emit/pipeline/tail.rs index c0e856e3..a65c4745 100644 --- a/src/emit/pipeline/tail.rs +++ b/src/emit/pipeline/tail.rs @@ -447,4 +447,48 @@ mod tests { let e = spawned.err().expect("dead port fails spawn"); assert!(e.starts_with("leg: spawn insert tail: "), "{e}"); } + + #[tokio::test] + async fn null_tail_swallows_encoded_runs() { + let fatal = Fatal::new(); + let (tx, ack, parts) = spawn_null(Arc::new(Monotone::default()), fatal.clone()); + let run = tokio::spawn(async { + Ok(EncodedRun { + chunks: Vec::new(), + permit: None, + }) + }); + tx.send(BatcherMsg::Encoded(run)).await.unwrap(); + parts.finish(tx, ack, 0, &fatal).await.unwrap(); + } + + /// A failed encode job stops the null tail, so a later flush reports the + /// job's error rather than a closed channel + #[tokio::test] + async fn null_tail_fails_on_encode_errors() { + let failed = || tokio::spawn(async { Err("boom".to_string()) }); + let panicked = || tokio::spawn(async { panic!("encoder panicked") }); + for (job, want) in [(failed(), "encode: boom"), (panicked(), "encode job: ")] { + let fatal = Fatal::new(); + let (tx, ack, parts) = spawn_null(Arc::new(Monotone::default()), fatal.clone()); + tx.send(BatcherMsg::Encoded(job)).await.unwrap(); + tx.closed().await; + let err = parts.finish(tx, ack, 0, &fatal).await.unwrap_err(); + assert!(err.starts_with(want), "{err}"); + } + } + + #[tokio::test] + async fn discard_tail_finishes_without_clickhouse() { + let fatal = Fatal::new(); + let cfg = BatcherConfig { + row_budget: 1024, + byte_budget: 1 << 20, + inserters: 1, + flush_timeout: std::time::Duration::from_secs(1), + }; + let (tx, ack, parts) = spawn_discard(cfg, Arc::default(), fatal.clone()); + parts.finish(tx, ack, 0, &fatal).await.unwrap(); + assert!(!fatal.is_set()); + } } diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 458c0e93..7bc629fe 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -200,16 +200,6 @@ impl OracleColumnBuf { self.wire_bytes } - /// Request bytes this column adds: none while it resolves locally, - /// every cell once one needs conversion - pub fn request_bytes(&self) -> usize { - if self.remote_cells == 0 { - 0 - } else { - self.wire_bytes - } - } - /// Request bytes once `other`'s cells follow these pub fn request_bytes_with(&self, other: &Self) -> usize { if self.remote_cells + other.remote_cells == 0 { diff --git a/src/runtime_config.rs b/src/runtime_config.rs index a6ba2d51..6deb9003 100644 --- a/src/runtime_config.rs +++ b/src/runtime_config.rs @@ -985,4 +985,34 @@ mod tests { assert_eq!(mode.as_str().parse(), Ok(mode)); } } + + /// Typed and `typoutput` text cells read alike + #[test] + fn field_readers_take_text_and_stored_forms() { + use crate::schema::{BOOLOID, INT2OID, INT4OID, INT8OID, NAMEOID}; + let r = rel( + "t", + vec![ + attr(1, "n", INT8OID), + attr(2, "i2", INT2OID), + attr(3, "on", BOOLOID), + attr(4, "nm", NAMEOID), + attr(5, "i4", INT4OID), + ], + ); + let text = |s: &str| Some(build::out_text(s)); + let cols = [ + text("42"), + Some(build::int2(-2)), + text("on"), + Some(build::name("ab")), + Some(build::int4(7)), + ]; + assert_eq!(field_i64(&r, &cols, "n"), Some(42)); + assert_eq!(field_i64(&r, &cols, "i2"), Some(-2)); + assert_eq!(field_i64(&r, &cols, "nm"), None); + assert_eq!(field_bool(&r, &cols, "on"), Some(true)); + assert_eq!(field_bool(&r, &cols, "i4"), None); + assert_eq!(field_string(&r, &cols, "nm").as_deref(), Some("ab")); + } } diff --git a/src/xact/xact_buffer.rs b/src/xact/xact_buffer.rs index ac1c3d6c..1f012ad2 100644 --- a/src/xact/xact_buffer.rs +++ b/src/xact/xact_buffer.rs @@ -3759,15 +3759,6 @@ pub(crate) mod raw_fixtures { } } - /// `Ordinary` verdict for top xid 1, injected directly (no descriptor - /// log in these fixtures) - pub(crate) fn inject_ordinary( - rfn: RelFileNode, - rel: Arc, - ) -> StashResolved { - inject_ordinary_with_stats(rfn, rel, None) - } - pub(crate) fn inject_ordinary_with_stats( rfn: RelFileNode, rel: Arc, @@ -5306,13 +5297,12 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16410); - let rfn = rel.rfn; b.stash_raw(8, multi_insert_raw(8, 100, 16410, &[100, 200, 300])) .await .unwrap(); b.on_heap(heap_with_value(1, 200, 16)).await.unwrap(); b.on_schema_event(1, 100, dropped_event(7)); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[8], false) @@ -5387,11 +5377,10 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16411); - let rfn = rel.rfn; let mut raw = multi_insert_raw(1, 100, 16411, &[1]); raw.main_data[0] = 0; b.stash_raw(1, raw).await.unwrap(); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -5815,7 +5804,7 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); b.stash_raw(1, raw).await.unwrap(); - let stash = inject_ordinary(rel.rfn, rel.clone()); + let stash = StashResolved::ordinary(1, rel.clone(), 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -5873,11 +5862,10 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16417); - let rfn = rel.rfn; b.stash_raw(1, multi_insert_raw(99, 100, 16417, &[5])) .await .unwrap(); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -5900,7 +5888,6 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16413); - let rfn = rel.rfn; let mut prune = multi_insert_raw(1, 90, 16413, &[1]); prune.info = 0x10; // XLOG_HEAP2_PRUNE* let mut lock = multi_insert_raw(1, 95, 16413, &[1]); @@ -5911,7 +5898,7 @@ mod tests { b.stash_raw(1, multi_insert_raw(1, 100, 16413, &[7])) .await .unwrap(); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -5944,7 +5931,6 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16414); - let rfn = rel.rfn; let mut main_data = vec![0u8; SIZE_OF_HEAP_UPDATE]; main_data[7] = XLH_UPDATE_PREFIX_FROM_OLD; // block 0: [prefixlen=4 covers the whole int4][xl_heap_header][pad] @@ -5978,7 +5964,7 @@ mod tests { }], }; b.stash_raw(1, raw).await.unwrap(); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -6007,12 +5993,11 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16415); - let rfn = rel.rfn; b.stash_raw(1, multi_insert_raw(1, 100, 16415, &[1, 2, 3])) .await .unwrap(); b.stash_raw(1, update_raw(1, 150, 16415, 9)).await.unwrap(); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[], false) .await @@ -6158,7 +6143,6 @@ mod tests { let tmp = tempdir().unwrap(); let mut b = XactBuffer::new(cfg(tmp.path().to_path_buf())).unwrap(); let rel = int4_descriptor(16416); - let rfn = rel.rfn; // ~3.6 KiB of tuple data blows the 1 KiB budget: subxact 8 spills let big: Vec = (0..300).collect(); b.stash_raw(8, multi_insert_raw(8, 100, 16416, &big)) @@ -6174,7 +6158,7 @@ mod tests { .collect(); assert!(spilled.contains(&(8, true)), "{spilled:?}"); assert!(spilled.contains(&(9, false)), "{spilled:?}"); - let stash = inject_ordinary(rfn, rel); + let stash = StashResolved::ordinary(1, rel, 0x50); let mut drain = b .drain_committed(stash, 42, 0x2000, &[8, 9], false) .await @@ -6803,7 +6787,7 @@ mod tests { assert_eq!(live.load(Ordering::Relaxed), 3, "top abort"); let mut drain = b - .drain_committed(inject_ordinary(rel_rfn, rel), 0, 0x300, &[], false) + .drain_committed(StashResolved::ordinary(1, rel, 0x50), 0, 0x300, &[], false) .await .unwrap(); let batch = drain