diff --git a/pgext/overlay.c b/pgext/overlay.c index 1bfe65cb..e994491c 100644 --- a/pgext/overlay.c +++ b/pgext/overlay.c @@ -334,6 +334,10 @@ ws_emit_type(StringInfo out, HeapTuple tup, TupleDesc desc) { ws_put_oid(out, f->oid); ws_put_name(out, &f->typname); + ws_put_oid(out, f->typbasetype); + ws_put_int(out, f->typtypmod); + ws_put_oid(out, f->typelem); + ws_put_oid(out, f->typarray); } typedef void (*WsEmitRow)(StringInfo out, HeapTuple tup, TupleDesc desc); @@ -396,7 +400,7 @@ ws_catalog_plan(WsCatalog cat, WsCatalogPlan* plan) { plan->indexid = InvalidOid; plan->keyattno = InvalidAttrNumber; plan->min_attnum = 0; - plan->ncols = 2; + plan->ncols = 6; plan->emit = ws_emit_type; return true; } diff --git a/pgext/walshadow.h b/pgext/walshadow.h index bae035e6..3cf4c465 100644 --- a/pgext/walshadow.h +++ b/pgext/walshadow.h @@ -10,7 +10,7 @@ * of an unchanged layout changes */ #define WS_PROTO_VERSION 7 /* Bumped when any catalog projection changes shape */ -#define WS_PROJECTION_VERSION 2 +#define WS_PROJECTION_VERSION 3 /* request opcodes */ #define WS_OP_HELLO 0x01 diff --git a/src/backfill/backfill_bootstrap.rs b/src/backfill/backfill_bootstrap.rs index f0a4b12d..7b91fae1 100644 --- a/src/backfill/backfill_bootstrap.rs +++ b/src/backfill/backfill_bootstrap.rs @@ -325,6 +325,12 @@ pub async fn seed_catalog_from_source(client: &Client) -> Result { ) .await .context("bootstrap: enumerate user relations on source")?; + let types = crate::pg::type_rows( + &client + .query(crate::pg::TYPE_SQL, &[]) + .await + .context("bootstrap: read pg_type on source")?, + ); let mut by_oid: ahash::HashMap, Option)> = ahash::HashMap::default(); @@ -349,7 +355,7 @@ pub async fn seed_catalog_from_source(client: &Client) -> Result { rel_node, }; let replident = fetch_replident(client, replident_char, oid).await?; - let attributes = fetch_attributes(client, oid).await?; + let attributes = fetch_attributes(client, oid, &types).await?; let desc = RelDescriptor { rfn, oid, @@ -432,12 +438,16 @@ async fn fetch_replident(client: &Client, c: char, rel_oid: Oid) -> Result Result> { +async fn fetch_attributes( + client: &Client, + rel_oid: Oid, + types: &ahash::HashMap, +) -> Result> { let rows = client.query(crate::pg::ATTR_SQL, &[&rel_oid]).await?; rows.iter() .map(|row| { crate::pg::RawAttr::from_row(row) - .build() + .build(types) .map_err(|e| anyhow::anyhow!("bootstrap: {e}")) }) .collect() diff --git a/src/catalog/shadow_catalog.rs b/src/catalog/shadow_catalog.rs index bac23202..74523dce 100644 --- a/src/catalog/shadow_catalog.rs +++ b/src/catalog/shadow_catalog.rs @@ -507,19 +507,14 @@ impl ShadowCatalog { }, ) .await?; - // DROP COLUMN zeroes atttypid, and no pg_type row for it is what leaves - // the slot's type_name empty let types = self - .resolve_names( + .resolve_types( bridge, - "SELECT oid::oid, typname::text FROM pg_type \ - WHERE oid = ANY($1::oid[])", &attrs .iter() .map(|a| a.atttypid) .filter(|oid| *oid != 0) .collect(), - |r: TypeRow| (r.oid, r.typname), top_xid, NamePin { lsn: pinned, @@ -602,6 +597,34 @@ impl ShadowCatalog { Ok(names) } + /// Type rows for `wanted` and their domain bases, read in + /// [`Self::resolve_names`] order + async fn resolve_types( + &mut self, + bridge: &Bridge, + wanted: &BTreeSet, + top_xid: u32, + pin: NamePin, + ) -> Result> { + if wanted.is_empty() { + return Ok(HashMap::new()); + } + if !pin.by_caller { + let list: Vec = wanted.iter().copied().collect(); + let rows = self.query_retry(&TYPE_CLOSURE_SQL, &[&list]).await?; + let types = crate::pg::type_rows(&rows); + if wanted.iter().all(|oid| types.contains_key(oid)) { + return Ok(types); + } + } + let scan = bridge.scan_at(Catalog::Type, top_xid, &[], pin.lsn).await?; + Ok(scan + .parse::()? + .into_iter() + .map(|row| (row.oid, row)) + .collect()) + } + pub async fn current_database_oid(&mut self) -> Result { let row = self .query_one_retry( @@ -724,7 +747,7 @@ fn mirror_sql(scope_pred: &str) -> String { ), att AS MATERIALIZED (\ SELECT a.* FROM pg_attribute a JOIN scoped s ON s.oid = a.attrelid \ WHERE a.attnum >= 1\ - ) \ + ), {types} \ SELECT {position}, ARRAY[coalesce(pg_last_wal_replay_lsn()::text, '{no_replay}')] \ UNION ALL SELECT {class}, ARRAY[\ format('%s', c.oid), format('%s', c.relnamespace), format('%s', c.relname), \ @@ -750,8 +773,9 @@ fn mirror_sql(scope_pred: &str) -> String { FROM pg_inherits i JOIN scoped s ON s.oid = i.inhrelid AND s.relispartition \ UNION ALL SELECT {namespace}, ARRAY[format('%s', n.oid), format('%s', n.nspname)] \ FROM pg_namespace n WHERE n.oid IN (SELECT relnamespace FROM scoped) \ - UNION ALL SELECT {typ}, ARRAY[format('%s', t.oid), format('%s', t.typname)] \ - FROM pg_type t WHERE t.oid IN (SELECT atttypid FROM att)", + UNION ALL SELECT {typ}, ARRAY[format('%s', t.oid), format('%s', t.typname), \ + format('%s', t.typbasetype), format('%s', t.typtypmod), \ + format('%s', t.typelem), format('%s', t.typarray)] FROM types t", position = MIRROR_POSITION, no_replay = NO_REPLAY, class = Catalog::Class as u8, @@ -760,9 +784,32 @@ fn mirror_sql(scope_pred: &str) -> String { inherits = Catalog::Inherits as u8, namespace = Catalog::Namespace as u8, typ = Catalog::Type as u8, + types = type_closure("SELECT atttypid FROM att"), + ) +} + +/// `types` CTE over `seed` oids plus their domain bases, array elements and +/// array types, enough for [`crate::pg::RawAttr::build`] +fn type_closure(seed: &str) -> String { + format!( + "types AS (\ + SELECT oid, typname, typbasetype, typtypmod, typelem, typarray \ + FROM pg_type WHERE oid IN ({seed}) \ + UNION \ + SELECT t.oid, t.typname, t.typbasetype, t.typtypmod, t.typelem, t.typarray \ + FROM types d JOIN pg_type t ON t.oid IN (d.typbasetype, d.typelem, d.typarray)\ + )" ) } +static TYPE_CLOSURE_SQL: LazyLock = LazyLock::new(|| { + format!( + "WITH RECURSIVE {} \ + SELECT oid, typname::text, typbasetype, typtypmod, typelem, typarray FROM types", + type_closure("SELECT unnest($1::oid[])") + ) +}); + static MIRROR_BATCH_SQL: LazyLock = LazyLock::new(|| mirror_sql("c.oid = ANY($1::oid[])")); /// [`MIRROR_BATCH_SQL`] over every eligible user relation. Predicate is @@ -901,7 +948,7 @@ struct DescriptorRows { indexes: Vec, inherits: Vec, namespaces: HashMap, - types: HashMap, + types: HashMap, /// Requested relations. Other `class` rows supply ancestor information. /// Use `None` to keep every row requested: Option>, @@ -940,7 +987,7 @@ impl DescriptorRows { } Catalog::Type => { let row = TypeRow::parse(&cols)?; - out.types.insert(row.oid, row.typname); + out.types.insert(row.oid, row); } } } @@ -1037,7 +1084,7 @@ fn descriptor_from_rows( attrs: &[AttributeRow], indexes: &[IndexRow], namespaces: &HashMap, - types: &HashMap, + types: &HashMap, db_node: Oid, default_tablespace: Oid, ) -> Result { @@ -1080,14 +1127,13 @@ fn descriptor_from_rows( typmod: attr.atttypmod, not_null: attr.attnotnull, dropped: attr.attisdropped, - type_name: types.get(&attr.atttypid).cloned(), type_byval: attr.attbyval, type_len: attr.attlen, type_align: attr.attalign.to_string(), type_storage: attr.attstorage.to_string(), missing: None, }; - let mut rel_attr = raw.build().map_err(CatalogError::Parse)?; + let mut rel_attr = raw.build(types).map_err(CatalogError::Parse)?; if let Some(hex) = attr.attmissingval.as_deref() && let Some(bytes) = decode_hex(hex) { @@ -1203,6 +1249,25 @@ mod tests { pairs.iter().map(|(o, n)| (*o, (*n).to_owned())).collect() } + fn types(pairs: &[(Oid, &str)]) -> HashMap { + pairs + .iter() + .map(|(oid, name)| { + ( + *oid, + TypeRow { + oid: *oid, + typname: (*name).into(), + typbasetype: 0, + typtypmod: -1, + typelem: 0, + typarray: 0, + }, + ) + }) + .collect() + } + fn overlay( class: &ClassRow, attrs: &[AttributeRow], @@ -1214,7 +1279,7 @@ mod tests { attrs, indexes, namespaces, - &names(&[(23, "int4"), (25, "text")]), + &types(&[(23, "int4"), (25, "text")]), 5, DEFAULT_TABLESPACE, ) @@ -1346,7 +1411,7 @@ mod tests { }, ], namespaces: names(&[(2200, "public")]), - types: names(&[(23, "int4")]), + types: types(&[(23, "int4")]), requested: Some([16500, 16510].into_iter().collect()), ..DescriptorRows::default() }; diff --git a/src/ops/bridge.rs b/src/ops/bridge.rs index 3db09405..4a7fc477 100644 --- a/src/ops/bridge.rs +++ b/src/ops/bridge.rs @@ -25,7 +25,7 @@ use crate::toast::FetchedValue; /// Frame and op layouts. Must equal `WS_PROTO_VERSION` in `pgext/walshadow.h` pub const PROTO_VERSION: u32 = 7; /// Catalog column plans. Must equal `WS_PROJECTION_VERSION` -pub const PROJECTION_VERSION: u32 = 2; +pub const PROJECTION_VERSION: u32 = 3; /// Match `WS_MAX_REQUEST_BYTES` pub const MAX_REQUEST_BYTES: usize = 256 * 1024 * 1024; @@ -112,7 +112,8 @@ impl Catalog { Catalog::Class => 10, Catalog::Attribute => 12, Catalog::Index => 5, - Catalog::Namespace | Catalog::Type | Catalog::Inherits => 2, + Catalog::Namespace | Catalog::Inherits => 2, + Catalog::Type => 6, } } } @@ -1049,6 +1050,10 @@ impl ScanRow for NamespaceRow { pub struct TypeRow { pub oid: u32, pub typname: String, + pub typbasetype: u32, + pub typtypmod: i32, + pub typelem: u32, + pub typarray: u32, } impl ScanRow for TypeRow { @@ -1058,6 +1063,10 @@ impl ScanRow for TypeRow { Ok(Self { oid: field(row, 0)?.parse().map_err(|_| bad(row, 0))?, typname: field(row, 1)?.to_owned(), + typbasetype: field(row, 2)?.parse().map_err(|_| bad(row, 2))?, + typtypmod: field(row, 3)?.parse().map_err(|_| bad(row, 3))?, + typelem: field(row, 4)?.parse().map_err(|_| bad(row, 4))?, + typarray: field(row, 5)?.parse().map_err(|_| bad(row, 5))?, }) } } diff --git a/src/pg.rs b/src/pg.rs index bec0977c..63c8abf7 100644 --- a/src/pg.rs +++ b/src/pg.rs @@ -1,8 +1,10 @@ //! Source-PG sidecar SQL helpers shared across sweep/backfill/config paths. +use ahash::HashMap; use tokio_postgres::types::{Oid, PgLsn}; use tokio_postgres::{Client, Row}; +use crate::ops::bridge::TypeRow; use crate::schema::RelAttr; pub use walrus::pg::backup::parse_pg_lsn; @@ -29,36 +31,53 @@ pub fn parse_array_one_element(raw: &str) -> Option { /// Attribute rows shared by catalog fetchers. Physical layout comes straight /// off pg_attribute: DROP COLUMN zeroes atttypid but preserves /// attlen/attalign/attbyval/attstorage (PG `src/backend/catalog/heap.c` -/// `RemoveAttributeById`), so pg_type joins LEFT and supplies typname only — -/// an INNER join loses dropped slots and misaligns attnum-1 indexed decode. +/// `RemoveAttributeById`). Type names come from [`TYPE_SQL`] rows at +/// [`RawAttr::build`] pub const ATTR_SQL: &str = "SELECT \ - a.attnum::int2, \ - a.attname::text, \ - a.atttypid::oid, \ - a.atttypmod::int4, \ - a.attnotnull::bool, \ - a.attisdropped::bool, \ - t.typname::text, \ - a.attbyval::bool, \ - a.attlen::int2, \ - a.attalign::text, \ - a.attstorage::text, \ - CASE WHEN a.atthasmissing THEN a.attmissingval::text END \ - FROM pg_attribute a \ - LEFT JOIN pg_type t ON t.oid = a.atttypid \ - WHERE a.attrelid = $1 AND a.attnum >= 1 \ - ORDER BY a.attnum"; - -/// One [`ATTR_SQL`] row before char-field validation. + attnum::int2, \ + attname::text, \ + atttypid::oid, \ + atttypmod::int4, \ + attnotnull::bool, \ + attisdropped::bool, \ + attbyval::bool, \ + attlen::int2, \ + attalign::text, \ + attstorage::text, \ + CASE WHEN atthasmissing THEN attmissingval::text END \ + FROM pg_attribute \ + WHERE attrelid = $1 AND attnum >= 1 \ + ORDER BY attnum"; + +pub const TYPE_SQL: &str = + "SELECT oid, typname::text, typbasetype, typtypmod, typelem, typarray FROM pg_type"; + +/// Key [`TYPE_SQL`] shaped rows by oid +pub fn type_rows(rows: &[Row]) -> HashMap { + rows.iter() + .map(|r| { + let row = TypeRow { + oid: r.get(0), + typname: r.get(1), + typbasetype: r.get(2), + typtypmod: r.get(3), + typelem: r.get(4), + typarray: r.get(5), + }; + (row.oid, row) + }) + .collect() +} + +/// One [`ATTR_SQL`] row before type resolution and char-field validation. pub struct RawAttr { pub attnum: i16, pub name: String, + /// atttypid, 0 for dropped slots pub type_oid: Oid, pub typmod: i32, pub not_null: bool, pub dropped: bool, - /// `None` for dropped slots (atttypid = 0) - pub type_name: Option, pub type_byval: bool, pub type_len: i16, pub type_align: String, @@ -76,16 +95,35 @@ impl RawAttr { typmod: row.get(3), not_null: row.get(4), dropped: row.get(5), - type_name: row.get(6), - type_byval: row.get(7), - type_len: row.get(8), - type_align: row.get(9), - type_storage: row.get(10), - missing: row.get(11), + type_byval: row.get(6), + type_len: row.get(7), + type_align: row.get(8), + type_storage: row.get(9), + missing: row.get(10), } } - pub fn build(self) -> Result { + /// `types` must hold atttypid, its domain bases, and for arrays the + /// element's domain bases and their array types + pub fn build(mut self, types: &HashMap) -> Result { + let mut type_name = String::new(); + if self.type_oid != 0 { + let (mut typ, mut typmod) = base_type(types, self.type_oid, self.typmod)?; + // Array of domain becomes array of base, unless base is itself an + // array and so has no array type + if let Some(elem) = types.get(&typ.typelem) + && elem.typbasetype != 0 + { + let (elem_base, elem_typmod) = base_type(types, elem.oid, typmod)?; + if let Some(array) = types.get(&elem_base.typarray) { + typ = array; + typmod = elem_typmod; + } + } + self.type_oid = typ.oid; + self.typmod = typmod; + type_name.clone_from(&typ.typname); + } Ok(RelAttr { attnum: self.attnum, name: self.name, @@ -93,7 +131,7 @@ impl RawAttr { typmod: self.typmod, not_null: self.not_null, dropped: self.dropped, - type_name: self.type_name.unwrap_or_default(), + type_name, type_byval: self.type_byval, type_len: self.type_len, type_align: single_char(&self.type_align, "attalign")?, @@ -107,6 +145,24 @@ impl RawAttr { } } +/// Domain to base type and typmod, as PG `getBaseTypeAndTypmod` +fn base_type( + types: &HashMap, + mut oid: Oid, + mut typmod: i32, +) -> Result<(&TypeRow, i32), String> { + loop { + let typ = types + .get(&oid) + .ok_or_else(|| format!("no pg_type row for type {oid}"))?; + if typ.typbasetype == 0 { + return Ok((typ, typmod)); + } + oid = typ.typbasetype; + typmod = typ.typtypmod; + } +} + fn single_char(s: &str, what: &str) -> Result { let mut it = s.chars(); match (it.next(), it.next()) { @@ -184,14 +240,13 @@ mod tests { typmod: -1, not_null: false, dropped: true, - type_name: None, type_byval: false, type_len: -1, type_align: "i".into(), type_storage: "x".into(), missing: None, }; - let attr = raw.build().unwrap(); + let attr = raw.build(&HashMap::default()).unwrap(); assert!(attr.dropped); assert_eq!(attr.type_name, ""); assert_eq!(attr.type_len, -1); @@ -208,13 +263,84 @@ mod tests { typmod: -1, not_null: true, dropped: false, - type_name: Some("int4".into()), type_byval: true, type_len: 4, type_align: "ii".into(), type_storage: "p".into(), missing: None, }; - assert!(raw.build().is_err()); + assert!(raw.build(&domain_types()).is_err()); + } + + /// (oid, typname, typbasetype, typtypmod, typelem, typarray) + fn types(rows: &[(Oid, &str, Oid, i32, Oid, Oid)]) -> HashMap { + rows.iter() + .map( + |&(oid, typname, typbasetype, typtypmod, typelem, typarray)| { + let row = TypeRow { + oid, + typname: typname.into(), + typbasetype, + typtypmod, + typelem, + typarray, + }; + (oid, row) + }, + ) + .collect() + } + + const NUMERIC_10_2: i32 = (10 << 16) + 2 + 4; + + /// `cents` over `price` over numeric(10,2), `numbers` over int4[], and + /// `cents_array` over cents[] + fn domain_types() -> HashMap { + types(&[ + (23, "int4", 0, -1, 0, 1007), + (1007, "_int4", 0, -1, 23, 0), + (1700, "numeric", 0, -1, 0, 1231), + (1231, "_numeric", 0, -1, 1700, 0), + (16400, "price", 1700, NUMERIC_10_2, 0, 16401), + (16401, "_price", 0, -1, 16400, 0), + (16402, "cents", 16400, -1, 0, 16403), + (16403, "_cents", 0, -1, 16402, 0), + (16404, "numbers", 1007, -1, 0, 16405), + (16405, "_numbers", 0, -1, 16404, 0), + (16406, "cents_array", 16403, -1, 0, 16407), + ]) + } + + fn resolve(type_oid: Oid) -> (Oid, i32, String) { + let raw = RawAttr { + attnum: 1, + name: "c".into(), + type_oid, + typmod: -1, + not_null: false, + dropped: false, + type_byval: false, + type_len: -1, + type_align: "i".into(), + type_storage: "x".into(), + missing: None, + }; + let attr = raw.build(&domain_types()).unwrap(); + (attr.type_oid, attr.typmod, attr.type_name) + } + + #[test] + fn raw_attr_build_resolves_domains() { + let numeric = (1700, NUMERIC_10_2, "numeric".into()); + let numeric_array = (1231, NUMERIC_10_2, "_numeric".into()); + assert_eq!(resolve(16402), numeric); + assert_eq!(resolve(16403), numeric_array, "array of domain"); + assert_eq!(resolve(16406), numeric_array, "domain over array of domain"); + assert_eq!(resolve(16404), (1007, -1, "_int4".into())); + assert_eq!( + resolve(16405), + (16405, -1, "_numbers".into()), + "array of domain over array has no base array type" + ); } } diff --git a/src/schema.rs b/src/schema.rs index 569820f8..b6d7460b 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -189,6 +189,7 @@ pub fn replident_key_attnums(desc: &RelDescriptor) -> &[i16] { pub struct RelAttr { pub attnum: i16, pub name: String, + /// Domain base type and typmod, as PG `getBaseTypeAndTypmod` pub type_oid: Oid, pub typmod: i32, pub not_null: bool, diff --git a/tests/bridge.rs b/tests/bridge.rs index f3f03b4a..30a2fbdd 100644 --- a/tests/bridge.rs +++ b/tests/bridge.rs @@ -798,6 +798,149 @@ async fn bridge_drops_bad_frames_per_connection() { assert_eq!(bridge.stats.reconnects.load(Ordering::Relaxed), 0); } +#[tokio::test(flavor = "current_thread")] +async fn bridge_domains_resolve_across_catalog_paths_and_copy() { + if !pg_available() { + return; + } + use walshadow::decode::heap_decoder::{ + ColumnValue, missing_defaults_equivalent, missing_value_for, + }; + use walshadow::schema::{INT4OID, JSONBOID, TEXTOID, TIMESTAMPOID}; + + let tmp = tempfile::tempdir().unwrap(); + let guard = start_pg(&tmp, ports::PG_SHADOW_PORT); + let bridge = Arc::new(dial(&guard.sh).await); + let mut cat = open_catalog(&guard.sh, bridge).await; + let (_, mut mirror) = open_mirror_catalog(&guard.sh, &tmp.path().join("moving.sock")).await; + let setup = connect_sql(&guard.sh).await; + setup + .batch_execute( + "CREATE SCHEMA app; + CREATE DOMAIN app.counter AS int; + CREATE DOMAIN app.money AS numeric(10,2); + CREATE DOMAIN app.cents AS app.money CHECK (VALUE >= 0); + CREATE DOMAIN app.moment AS timestamp(3); + CREATE DOMAIN app._int4 AS text; + CREATE DOMAIN app.document AS jsonb; + CREATE DOMAIN app.numbers AS int[]; + CREATE TABLE app.domains ( + id app.counter PRIMARY KEY, amount app.cents, at app.moment, + label app._int4, doc app.document, nums app.numbers, gone app.counter); + INSERT INTO app.domains VALUES + (1, 12.34, '2024-01-02 03:04:05.678', 'hello', '{\"k\": 1}', '{1,NULL,3}', 9); + ALTER TABLE app.domains DROP COLUMN gone; + ALTER TABLE app.domains ADD COLUMN counter app.counter DEFAULT 7; + ALTER TABLE app.domains ADD COLUMN counts app.counter[]; + UPDATE app.domains SET counts = '{4,NULL}';", + ) + .await + .unwrap(); + let oid = oid_of(&setup, "app.domains").await; + let (_, committed) = cat.fetch_descriptors_batch(&[oid]).await.unwrap(); + let (_, stated) = mirror.fetch_descriptors_batch(&[oid]).await.unwrap(); + let pinned = cat + .fetch_overlay_descriptors(&[oid], 0, &ParkedAt::assume_for_test(0)) + .await + .unwrap(); + assert_eq!(committed, stated); + assert_eq!(committed, pinned); + let desc = &committed[0]; + let expected = [ + (INT4OID, "int4", -1, "Int32"), + ( + NUMERICOID, + "numeric", + (10 << 16) + 2 + 4, + "Nullable(Decimal(10, 2))", + ), + ( + TIMESTAMPOID, + "timestamp", + 3, + "Nullable(DateTime64(3, 'UTC'))", + ), + (TEXTOID, "text", -1, "Nullable(String)"), + (JSONBOID, "jsonb", -1, "Nullable(String)"), + (1007, "_int4", -1, "Array(Nullable(Int32))"), + ]; + for (attr, (oid, name, typmod, ch_type)) in desc.attributes.iter().zip(expected) { + assert_eq!( + (attr.type_oid, attr.type_name.as_str(), attr.typmod), + (oid, name, typmod) + ); + assert_eq!( + walshadow::type_bridge::map(attr, attr.attnum == 1) + .unwrap() + .ch_type, + ch_type + ); + } + assert!(desc.attributes[6].dropped); + let counts = &desc.attributes[8]; + assert_eq!( + (counts.type_oid, counts.type_name.as_str()), + (1007, "_int4"), + "array of domain" + ); + assert_eq!( + walshadow::type_bridge::map(counts, false).unwrap().ch_type, + "Array(Nullable(Int32))" + ); + assert_eq!(missing_value_for(&desc.attributes[7]), ColumnValue::Int4(7)); + assert_eq!( + walshadow::type_bridge::map(&desc.attributes[7], false) + .unwrap() + .default_sql + .as_deref(), + Some("7") + ); + + let types = walshadow::pg::type_rows(&setup.query(walshadow::pg::TYPE_SQL, &[]).await.unwrap()); + let sql_attrs = setup.query(walshadow::pg::ATTR_SQL, &[&oid]).await.unwrap(); + for (row, attr) in sql_attrs.iter().zip(&desc.attributes) { + let mut sql_attr = walshadow::pg::RawAttr::from_row(row).build(&types).unwrap(); + assert!(missing_defaults_equivalent(&sql_attr, attr)); + sql_attr.missing_default = attr.missing_default.clone(); + assert_eq!(&sql_attr, attr); + } + + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + assert_eq!( + walshadow::copy_backfill::copy_rows_into( + &setup, + desc, + 42, + &tx, + &walshadow::ch_emitter::EmitterStats::default(), + walshadow::copy_backfill::BlockRange { + start: 0, + end: None + }, + ) + .await + .unwrap(), + 1 + ); + let rows = rx.recv().await.unwrap(); + let values = &rows[0].columns; + assert_eq!(values[0], Some(ColumnValue::Int4(1))); + assert!(matches!(&values[1], Some(ColumnValue::Numeric(_)))); + assert!(matches!(&values[2], Some(ColumnValue::Timestamp(_)))); + assert_eq!(values[3], Some(ColumnValue::Text("hello".into()))); + assert_eq!(values[4], Some(ColumnValue::Json("{\"k\": 1}".into()))); + assert!(matches!( + &values[5], + Some(ColumnValue::PgPendingText { type_oid: 1007, .. }) + )); + assert_eq!(values[6], None); + assert_eq!(values[7], Some(ColumnValue::Int4(7))); + assert!(matches!( + &values[8], + Some(ColumnValue::PgPendingText { type_oid: 1007, .. }) + )); +} + /// Three row sources — the mirroring statement, `SCAN` at top xid 0, and /// `SCAN` under a transaction that wrote nothing — must build the same /// `RelDescriptor`; overlay then tracks open transaction, and the statement @@ -891,8 +1034,9 @@ async fn bridge_overlay_descriptors_track_open_ddl() { "BEGIN; ALTER TABLE app.t ADD COLUMN c int DEFAULT 7; CREATE SCHEMA fresh; - CREATE DOMAIN fresh.cents AS int; - CREATE TABLE fresh.u (id int PRIMARY KEY, amount fresh.cents);", + CREATE DOMAIN fresh.money AS numeric(10,2); + CREATE DOMAIN fresh.cents AS fresh.money; + CREATE TABLE fresh.u (id int PRIMARY KEY, amount fresh.cents, amounts fresh.cents[]);", ) .await .expect("open ddl"); @@ -923,10 +1067,17 @@ async fn bridge_overlay_descriptors_track_open_ddl() { assert_eq!(u.rel_name.to_string(), "fresh.u", "in-xact schema name"); assert_eq!( - u.attributes[1].type_name, "cents", - "in-xact domain name: {:#?}", + u.attributes[1].type_name, "numeric", + "in-xact domain base type: {:#?}", u.attributes ); + assert_eq!(u.attributes[1].type_oid, NUMERICOID); + assert_eq!(u.attributes[1].typmod, (10 << 16) + 2 + 4); + assert_eq!( + (u.attributes[2].type_oid, u.attributes[2].typmod), + (1231, (10 << 16) + 2 + 4), + "in-xact array of in-xact domain" + ); assert_eq!( u.replident, ReplIdent::Default { diff --git a/tests/oracle_types_e2e.rs b/tests/oracle_types_e2e.rs index b4c69e00..601ec47e 100644 --- a/tests/oracle_types_e2e.rs +++ b/tests/oracle_types_e2e.rs @@ -133,6 +133,79 @@ async fn run_oracle_stats( (source, ch, tmp, oracle) } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn domains_over_local_types_skip_oracle() { + if skip_gate() { + return; + } + let (source, ch, _tmp, oracle) = run_oracle_stats( + fx::Ports::alloc(), + "walshadow-domains", + "CREATE DOMAIN counter AS int; + CREATE DOMAIN price AS numeric(10,2); + CREATE DOMAIN cents AS price CHECK (VALUE >= 0); + CREATE DOMAIN label AS text; + CREATE DOMAIN document AS jsonb; + CREATE DOMAIN moment AS timestamp(3); + CREATE TABLE public.domains ( + id counter PRIMARY KEY, amount cents, label label, doc document, at moment); + ALTER TABLE public.domains ALTER COLUMN label SET STORAGE EXTERNAL;", + "CREATE OR REPLACE TABLE walshadow_test.domains ( + id Int32, amount Nullable(Decimal(10,2)), label Nullable(String), + doc Nullable(String), at Nullable(DateTime64(3, 'UTC')), + _lsn UInt64, _xid UInt32, _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool + ) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + vec![fx::TableMappingSpec { + source_table: RelName::new("public", "domains"), + target_table: TableTarget::new("walshadow_test", "domains"), + columns: vec![ + col(1, "id", "Int32"), + col(2, "amount", "Nullable(Decimal(10,2))"), + col(3, "label", "Nullable(String)"), + col(4, "doc", "Nullable(String)"), + col(5, "at", "Nullable(DateTime64(3, 'UTC'))"), + ], + }], + "INSERT INTO public.domains VALUES + (1, 12.34, repeat('x', 12000), '{\"k\": 1}', '2024-01-02 03:04:05.678'), + (2, NULL, NULL, NULL, NULL), + (3, 1.00, 'gone', '{}', NULL); + UPDATE public.domains SET amount = 56.78 WHERE id = 1; + DELETE FROM public.domains WHERE id = 3; + SELECT pg_switch_wal();", + ) + .await; + let _src = fx::StopOnDrop { sh: &source }; + assert_eq!( + ch.query( + "SELECT id, amount, length(label), doc, toString(at) + FROM walshadow_test.domains FINAL WHERE id = 1 AND _is_deleted = 0" + ) + .unwrap(), + "1\t56.78\t12000\t{\"k\": 1}\t2024-01-02 03:04:05.678", + ); + assert_eq!( + ch.query( + "SELECT isNull(amount), isNull(label), isNull(doc), isNull(at) + FROM walshadow_test.domains FINAL WHERE id = 2 AND _is_deleted = 0" + ) + .unwrap(), + "1\t1\t1\t1", + ); + assert_eq!( + ch.query("SELECT count() FROM walshadow_test.domains FINAL WHERE _is_deleted = 0") + .unwrap(), + "2", + ); + assert_eq!( + oracle + .stats + .blocks + .load(std::sync::atomic::Ordering::Relaxed), + 0 + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn arrays_resolve_via_oracle() { if skip_gate() { @@ -141,7 +214,8 @@ async fn arrays_resolve_via_oracle() { let (source, ch, _tmp) = run_oracle( fx::Ports::alloc(), "walshadow-oracle-arrays", - "CREATE TABLE public.arr (id int PRIMARY KEY, ints int[], texts text[], nums numeric[]);\n", + "CREATE DOMAIN int_array AS int[];\n\ + CREATE TABLE public.arr (id int PRIMARY KEY, ints int_array, texts text[], nums numeric[]);\n", "CREATE OR REPLACE TABLE walshadow_test.arr (\ id Int32, ints Nullable(String), texts Nullable(String), nums Nullable(String),\ _lsn UInt64, _xid UInt32, _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool\