diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3aeee098..3a5f78c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,7 +150,8 @@ jobs: postgresql-${{ matrix.pg-major }} \ postgresql-client-${{ matrix.pg-major }} \ postgresql-server-dev-${{ matrix.pg-major }} \ - postgresql-${{ matrix.pg-major }}-pgvector + postgresql-${{ matrix.pg-major }}-pgvector \ + postgresql-${{ matrix.pg-major }}-postgis-3 sudo systemctl stop postgresql 2>/dev/null || true PG_BIN=/usr/lib/postgresql/${{ matrix.pg-major }}/bin "$PG_BIN/postgres" -V diff --git a/docs/destination-tables.md b/docs/destination-tables.md index 884006f6..f4dcac80 100644 --- a/docs/destination-tables.md +++ b/docs/destination-tables.md @@ -338,7 +338,7 @@ Common mappings include: | `json`, `jsonb` | `String` | | `hstore` | `Map(String, Nullable(String))` | | `vector`, `halfvec` (pgvector) | `Array(Float32)` | -| `geography`, `geometry` (PostGIS) | `String`, WKT for 2-D points, PostgreSQL's own hex form otherwise | +| `geography`, `geometry` (PostGIS) | 2-D `Point`, `LineString`, `Polygon`, `MultiLineString`, `MultiPolygon` subtypes map to the same CH type, SRID dropped; NULL is empty except for `Point`. Other subtypes, Z/M and unconstrained columns map to `String` holding `ST_AsText` WKT | | `[]` arrays | `Array(Nullable())`; unknown elem → `Array(Nullable(String))`. One layer, so a multidimensional value needs an explicit nested override | | `inet`, `cidr`, `interval`, unknown types | `String` | diff --git a/pgext/native.c b/pgext/native.c index 799b77e0..a412576d 100644 --- a/pgext/native.c +++ b/pgext/native.c @@ -10,6 +10,7 @@ #include "libpq/pqformat.h" #include "utils/builtins.h" #include "utils/catcache.h" +#include "utils/geo_decls.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -23,6 +24,16 @@ /* hstore_to_matrix returns Nx2 text arrays accepted as Map entries */ #define WS_HSTORE_EXPANDER "hstore_to_matrix" +/* PostGIS renders WKT for String targets, ISO WKB for ClickHouse geo types */ +#define WS_WKT_EXPANDER "st_astext" +#define WS_WKB_EXPANDER "st_asbinary" + +/* ISO WKB type codes, 2-D only */ +#define WS_WKB_POINT 1 +#define WS_WKB_LINESTRING 2 +#define WS_WKB_POLYGON 3 +#define WS_WKB_MULTILINESTRING 5 +#define WS_WKB_MULTIPOLYGON 6 /* Minimum per column: source OID, typmod, two length prefixes */ #define WS_COL_META_BYTES 16 @@ -41,6 +52,9 @@ typedef struct WsNativeCol { bool infn_ready; FmgrInfo expander; + Oid expander_type; + /* ISO WKB type a geo target takes, zero for other targets */ + uint32 wkb_type; Oid native_oid; } WsNativeCol; @@ -168,7 +182,7 @@ ws_input_datum(WsNativeCol* col, const char* body, uint32 len) { /* Require one expander owned by source type's extension */ static bool -ws_find_hstore_expander(Oid source_oid, Oid* out) { +ws_find_expander(Oid source_oid, const char* name, Oid* out) { Oid ext = getExtensionOfObject(TypeRelationId, source_oid); CatCList* list; Oid found = InvalidOid; @@ -177,7 +191,7 @@ ws_find_hstore_expander(Oid source_oid, Oid* out) { if (!OidIsValid(ext)) { return false; } - list = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(WS_HSTORE_EXPANDER)); + list = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(name)); for (int i = 0; i < list->n_members; i++) { HeapTuple tup = &list->members[i]->tuple; Form_pg_proc form = (Form_pg_proc)GETSTRUCT(tup); @@ -203,15 +217,45 @@ static void ws_resolve_expander(WsNativeCol* col) { const chc_type* root = col->type; chc_kind kind; + const char* name = WS_WKB_EXPANDER; + Oid result = BYTEAOID; Oid funcid; while ((kind = chc_type_kind(root)) == CHC_NULLABLE || kind == CHC_LOW_CARDINALITY) { root = chc_type_child(root, 0); } - - if (kind == CHC_MAP && ws_find_hstore_expander(col->source_oid, &funcid)) { + switch (kind) { + case CHC_MAP: + name = WS_HSTORE_EXPANDER; + result = TEXTARRAYOID; + break; + case CHC_STRING: + name = WS_WKT_EXPANDER; + result = TEXTOID; + break; + case CHC_POINT: + col->wkb_type = WS_WKB_POINT; + break; + case CHC_RING: + case CHC_LINE_STRING: + col->wkb_type = WS_WKB_LINESTRING; + break; + case CHC_POLYGON: + col->wkb_type = WS_WKB_POLYGON; + break; + case CHC_MULTI_LINE_STRING: + col->wkb_type = WS_WKB_MULTILINESTRING; + break; + case CHC_MULTI_POLYGON: + col->wkb_type = WS_WKB_MULTIPOLYGON; + break; + default: + return; + } + if (ws_find_expander(col->source_oid, name, &funcid)) { fmgr_info_cxt(funcid, &col->expander, CurrentMemoryContext); + col->expander_type = result; } } @@ -220,10 +264,128 @@ ws_expand(WsNativeCol* col, Datum val, Oid* valtype) { if (!OidIsValid(col->expander.fn_oid)) { return val; } - *valtype = TEXTARRAYOID; + *valtype = col->expander_type; return FunctionCall1(&col->expander, val); } +/* PostGIS writes the WKB, so its bounds and byte order hold */ +typedef struct WsWkb { + const uint8* p; + const uint8* end; + bool little; +} WsWkb; + +static uint64 +ws_wkb_word(WsWkb* r, int n) { + uint64 v = 0; + + Assert(r->end - r->p >= n); + for (int i = 0; i < n; i++) { + v |= (uint64)r->p[r->little ? i : n - 1 - i] << (8 * i); + } + r->p += n; + return v; +} + +/* Byte order then type, which must be `want` */ +static void +ws_wkb_header(WsWkb* r, uint32 want) { + uint32 type; + + Assert(r->p < r->end && *r->p <= 1); + r->little = *r->p++ == 1; + type = (uint32)ws_wkb_word(r, 4); + if (type != want) { + ereport( + ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("WKB geometry type %u, target column takes type %u", type, want), + errhint("ClickHouse geo types hold one 2-D geometry type each")) + ); + } +} + +static void +ws_wkb_point(pgch_writer* w, size_t col, WsWkb* r) { + Point pt; + uint64 x = ws_wkb_word(r, 8); + uint64 y = ws_wkb_word(r, 8); + + memcpy(&pt.x, &x, sizeof pt.x); + memcpy(&pt.y, &y, sizeof pt.y); + pgch_append_datum(w, col, PointPGetDatum(&pt), POINTOID, false); +} + +static void +ws_wkb_points(pgch_writer* w, size_t col, WsWkb* r) { + uint32 n = (uint32)ws_wkb_word(r, 4); + + pgch_array_begin(w, col); + for (uint32 i = 0; i < n; i++) { + ws_wkb_point(w, 0, r); + } + pgch_array_end(w); +} + +static void +ws_wkb_rings(pgch_writer* w, size_t col, WsWkb* r) { + uint32 n = (uint32)ws_wkb_word(r, 4); + + pgch_array_begin(w, col); + for (uint32 i = 0; i < n; i++) { + ws_wkb_points(w, 0, r); + } + pgch_array_end(w); +} + +/* Multi-geometry of `part` geometries, each with its own header */ +static void +ws_wkb_parts(pgch_writer* w, size_t col, WsWkb* r, uint32 part) { + uint32 n = (uint32)ws_wkb_word(r, 4); + + pgch_array_begin(w, col); + for (uint32 i = 0; i < n; i++) { + ws_wkb_header(r, part); + if (part == WS_WKB_LINESTRING) { + ws_wkb_points(w, 0, r); + } else { + ws_wkb_rings(w, 0, r); + } + } + pgch_array_end(w); +} + +/* ISO WKB into a geo column of WKB `type` */ +static void +ws_append_wkb(pgch_writer* w, size_t col, uint32 type, const bytea* wkb) { + WsWkb r = { + .p = (const uint8*)VARDATA_ANY(wkb), + .end = (const uint8*)VARDATA_ANY(wkb) + VARSIZE_ANY_EXHDR(wkb), + }; + + ws_wkb_header(&r, type); + switch (type) { + case WS_WKB_POINT: + ws_wkb_point(w, col, &r); + break; + case WS_WKB_LINESTRING: + ws_wkb_points(w, col, &r); + break; + case WS_WKB_POLYGON: + ws_wkb_rings(w, col, &r); + break; + default: + ws_wkb_parts( + w, + col, + &r, + type == WS_WKB_MULTILINESTRING ? WS_WKB_LINESTRING : WS_WKB_POLYGON + ); + break; + } + Assert(r.p == r.end); +} + /* Default absent values to NULL or empty non-nullable shapes */ static void ws_append_default(pgch_writer* w, size_t col, const WsNativeCol* c) { @@ -250,6 +412,21 @@ ws_append_default(pgch_writer* w, size_t col, const WsNativeCol* c) { w, col, DirectFunctionCall1(json_in, CStringGetDatum("{}")), JSONOID, false ); return; + case CHC_POINT: { + Point origin = { 0 }; + + pgch_append_datum(w, col, PointPGetDatum(&origin), POINTOID, false); + return; + } + /* Only Point nests in Nullable, so other geo shapes default empty */ + case CHC_RING: + case CHC_LINE_STRING: + case CHC_POLYGON: + case CHC_MULTI_LINE_STRING: + case CHC_MULTI_POLYGON: + pgch_array_begin(w, col); + pgch_array_end(w); + return; default: ereport( ERROR, @@ -309,6 +486,10 @@ ws_append_cell(pgch_writer* w, size_t col, WsNativeCol* c, StringInfo req) { val = tag == WS_CELL_DISK_RAW ? ws_reconstruct_datum(c, body, len) : ws_input_datum(c, body, len); val = ws_expand(c, val, &valtype); + if (c->expander_type == BYTEAOID) { + ws_append_wkb(w, col, c->wkb_type, DatumGetByteaPP(val)); + return; + } pgch_append_datum(w, col, val, valtype, false); } diff --git a/pgext/walshadow.h b/pgext/walshadow.h index bae035e6..458448f5 100644 --- a/pgext/walshadow.h +++ b/pgext/walshadow.h @@ -8,7 +8,7 @@ /* Bumped when a request or response layout changes, or when an op's reading * of an unchanged layout changes */ -#define WS_PROTO_VERSION 7 +#define WS_PROTO_VERSION 8 /* Bumped when any catalog projection changes shape */ #define WS_PROJECTION_VERSION 2 diff --git a/src/catalog/type_bridge.rs b/src/catalog/type_bridge.rs index 29b53f49..d8d52f67 100644 --- a/src/catalog/type_bridge.rs +++ b/src/catalog/type_bridge.rs @@ -27,13 +27,14 @@ //! | `json` / `jsonb` | `String` | document text | //! | `hstore` | `Map(String, Nullable(String))` | by type name | //! | `vector` / `halfvec` | `Array(Float32)` | pgvector, by type name | -//! | `geography` / `geometry` | `String` | WKT `POINT(x y)`, rendered at decode | +//! | `geography` / `geometry` | `Point`, `LineString`, `Polygon`, `MultiLineString`, `MultiPolygon` | 2-D typmod subtype; else `String` WKT | //! | `[]` | `Array(Nullable())` | 1-D supported elems, else String | //! | unknown | `String` | oracle converts, never raw bytes | //! //! Nullability: `not_null = false` wraps inner in `Nullable(_)` unless //! column is in CH `ORDER BY` (PK columns must stay non-nullable; caller -//! enforces). +//! enforces). CH nests only scalars and `Point` in `Nullable`, so arrays, +//! maps and other geo types store NULL as empty. //! //! ## Default expression //! @@ -80,7 +81,18 @@ pub fn map(att: &RelAttr, pk_member: bool) -> Result Result { JSONOID | JSONBOID => "String".into(), // Extension / array types carry dynamic OIDs, so match on the type // name: `hstore` → Map, `vector` (pgvector) → Array(Float32), - // `_` (PG array convention) → Array of a supported element, else - // the String fallback (geography stays String, rendered WKT at decode). + // PostGIS → its typmod's geo type, `_` (PG array convention) → + // Array of a supported element, else the String fallback. _ => { - if att.type_name == "hstore" { + if matches!(att.type_name.as_str(), "geometry" | "geography") { + postgis_ch_type(att.typmod).into() + } else if att.type_name == "hstore" { "Map(String, Nullable(String))".into() } else if att.type_name == "vector" || att.type_name == "halfvec" { "Array(Float32)".into() @@ -133,6 +147,23 @@ pub fn base_type_for(att: &RelAttr) -> Result { }) } +/// CH geo type of a PostGIS typmod: subtype in bits 2..8, Z and M in bits 1 +/// and 0. CH geo types are 2-D and have no MultiPoint or collection, so the +/// rest render WKT +fn postgis_ch_type(typmod: i32) -> &'static str { + if typmod < 0 || typmod & 0b11 != 0 { + return "String"; + } + match (typmod >> 2) & 0x3f { + 1 => "Point", + 2 => "LineString", + 3 => "Polygon", + 5 => "MultiLineString", + 6 => "MultiPolygon", + _ => "String", + } +} + /// PG array type name (`_int4`, `_text`, …) → `Array(Nullable())` for /// supported element types; `None` (→ String fallback) for anything else, /// including multi-dim-only element names walshadow can't render natively. @@ -495,6 +526,34 @@ mod tests { } } + /// Typmods as PostGIS 3.7 stores them; only Point stays Nullable + #[test] + fn postgis_typmods_pick_geo_types() { + let cases = [ + (-1, "Nullable(String)"), + (4, "Nullable(Point)"), + (1_107_460, "Nullable(Point)"), + (1_107_462, "Nullable(String)"), + (5, "Nullable(String)"), + (7, "Nullable(String)"), + (8, "LineString"), + (1_107_468, "Polygon"), + (16, "Nullable(String)"), + (20, "MultiLineString"), + (24, "MultiPolygon"), + (28, "Nullable(String)"), + ]; + for name in ["geometry", "geography"] { + for (typmod, want) in cases { + let att = RelAttr { + typmod, + ..named_attr(99_997, name, false) + }; + assert_eq!(map(&att, false).unwrap().ch_type, want, "{name} {typmod}"); + } + } + } + #[test] fn vector_maps_to_array_float32() { assert_eq!( diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index 10b106de..050e3be7 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -2631,7 +2631,7 @@ fn oracle_cell(v: Option<&ColumnValue>, source_type_oid: u32) -> Result OracleCell::TextInput(s.as_bytes().to_vec()), - // PostGIS WKT must bypass HEXEWKB typoutput + // Decoded text is already its own rendering Some(ColumnValue::Text(s)) | Some(ColumnValue::Name(s)) => { OracleCell::Literal(s.as_bytes().to_vec()) } diff --git a/src/emit/pipeline/bootstrap.rs b/src/emit/pipeline/bootstrap.rs index 557a9d31..3c288aa4 100644 --- a/src/emit/pipeline/bootstrap.rs +++ b/src/emit/pipeline/bootstrap.rs @@ -21,7 +21,6 @@ use crate::emit::pipeline::ack::{AckHandle, OpenSeq, Publish, SeqAlloc}; use crate::emit::pipeline::batcher::{BatcherClosed, BatcherMsg, RowBuffer, RowTable}; use crate::emit::route::{FrozenRoutes, RouteSnapshot, RowPolicy, freeze_routes}; use crate::mapping::{MappingSnapshot, TableMapping}; -use crate::ops::oracle::render_ext_columns; use crate::schema::{RelDescriptor, RelName}; use crate::ticker::Ticker; use crate::toast::{ @@ -259,7 +258,6 @@ pub async fn drain( .await? .map(Arc::new); } - render_ext_columns(&rel.attributes, &mut tuple.columns); let committed = tuple.into_committed_insert(); row_buffer .push(&msg_tx, run.seq.seq(), (rel, route), committed, permit) @@ -629,12 +627,11 @@ async fn route_batch( if row.replay_covered { continue; } - let mut tuple = row.tuple; + let tuple = row.tuple; let at = seq .get_or_insert_with(|| seqs.open(tuple.source_lsn, Publish::Commit)) .seq(); let RowTable { rel, route } = &tables[row.table as usize]; - render_ext_columns(&rel.attributes, &mut tuple.columns); row_buffer .push( msg_tx, diff --git a/src/emit/pipeline/decode.rs b/src/emit/pipeline/decode.rs index a8f94229..82ab705e 100644 --- a/src/emit/pipeline/decode.rs +++ b/src/emit/pipeline/decode.rs @@ -77,13 +77,6 @@ pub fn place_rows( commit_ts, commit_lsn, }; - // PostGIS WKT differs from typoutput HEXEWKB - if let Some(t) = committed.decoded.new.as_mut() { - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); - } - if let Some(t) = committed.decoded.old.as_mut() { - crate::ops::oracle::render_ext_columns(&rel.attributes, &mut t.columns); - } if !route.drops_deletes() && let Some(tomb) = take_key_change_tombstone(&mut committed.decoded, rel) { diff --git a/src/ops/bridge.rs b/src/ops/bridge.rs index 3db09405..74a2b01a 100644 --- a/src/ops/bridge.rs +++ b/src/ops/bridge.rs @@ -23,7 +23,7 @@ use tokio::sync::Mutex; use crate::toast::FetchedValue; /// Frame and op layouts. Must equal `WS_PROTO_VERSION` in `pgext/walshadow.h` -pub const PROTO_VERSION: u32 = 7; +pub const PROTO_VERSION: u32 = 8; /// Catalog column plans. Must equal `WS_PROJECTION_VERSION` pub const PROJECTION_VERSION: u32 = 2; diff --git a/src/ops/oracle.rs b/src/ops/oracle.rs index 7d786a84..c80a5b8d 100644 --- a/src/ops/oracle.rs +++ b/src/ops/oracle.rs @@ -6,9 +6,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use clickhouse_c::{Allocator, Block, BlockOpts, BlockReader, Column, Kind, SliceIo, TypeRef}; use crate::ch::types; -use crate::decode::heap_decoder::ColumnValue; use crate::ops::bridge::{Bridge, BridgeError, MAX_REQUEST_BYTES, request_frame}; -use crate::schema::RelAttr; use tokio_postgres::types::Oid; /// Cell tags, matching `WS_CELL_*` in `pgext/walshadow.h` @@ -477,41 +475,6 @@ fn decode_block( Ok(block) } -/// Render PostGIS 2-D points as WKT, typoutput yields HEXEWKB -pub fn render_ext_columns(attrs: &[RelAttr], columns: &mut [Option]) { - // Scan cells first: attribute type names sit behind a pointer per column - for (idx, cell) in columns.iter_mut().enumerate() { - let Some(ColumnValue::PgPending { raw, .. }) = cell.as_ref() else { - continue; - }; - let is_geo = attrs.iter().any(|att| { - !att.dropped - && usize::try_from(att.attnum - 1) == Ok(idx) - && matches!(att.type_name.as_str(), "geography" | "geometry") - }); - if is_geo && let Some(text) = gserialized_point_to_wkt(raw) { - *cell = Some(ColumnValue::Text(text)); - } - } -} - -/// PostGIS on-disk GSERIALIZED → `POINT(x y)` for 2-D points. Layout: -/// `[srid(3) + gflags(1)][geomtype u32-le][…]`; POINT has geomtype 1 and its -/// two `f64`-LE coordinates are the trailing 16 bytes. `None` for non-points. -fn gserialized_point_to_wkt(raw: &[u8]) -> Option { - if raw.len() < 16 { - return None; - } - let geomtype = u32::from_le_bytes(raw[4..8].try_into().ok()?); - if geomtype != 1 { - return None; - } - let n = raw.len(); - let x = f64::from_le_bytes(raw[n - 16..n - 8].try_into().ok()?); - let y = f64::from_le_bytes(raw[n - 8..n].try_into().ok()?); - Some(format!("POINT({x} {y})")) -} - impl OracleStats { pub fn summary(&self) -> String { use std::fmt::Write as _; @@ -639,21 +602,6 @@ mod tests { ); } - #[test] - fn gserialized_2d_point_renders_wkt() { - // [srid+gflags:4][geomtype=1:4][X f64le][Y f64le] - let mut raw = vec![0u8, 0, 0, 0, 1, 0, 0, 0]; - raw.extend_from_slice(&30.5_f64.to_le_bytes()); - raw.extend_from_slice(&81.25_f64.to_le_bytes()); - assert_eq!( - gserialized_point_to_wkt(&raw).as_deref(), - Some("POINT(30.5 81.25)") - ); - // non-point geomtype → None - raw[4] = 2; - assert_eq!(gserialized_point_to_wkt(&raw), None); - } - #[test] fn only_broken_sockets_retry() { let cases = [ diff --git a/tests/bootstrap_types_e2e.rs b/tests/bootstrap_types_e2e.rs index 59f0864f..c66743ad 100644 --- a/tests/bootstrap_types_e2e.rs +++ b/tests/bootstrap_types_e2e.rs @@ -211,8 +211,8 @@ async fn direct_bootstrap_all_types_end_to_end() { ("round(c_vector[1], 2)", "0.1".into()), ]; if has_postgis { - checks.push(("c_geog", "POINT(30.5 50.25)".into())); - checks.push(("c_geom", "POINT(1 2)".into())); + checks.push(("c_geog", "(30.5,50.25)".into())); + checks.push(("c_geom", "(1,2)".into())); } let mut fails = Vec::new(); diff --git a/tests/oracle.rs b/tests/oracle.rs index a15550c3..1e3c51fe 100644 --- a/tests/oracle.rs +++ b/tests/oracle.rs @@ -405,6 +405,92 @@ async fn oracle_encodes_tier3_disk_bytes() { assert_eq!(oracle.stats.errors.load(Ordering::Relaxed), 0); } +/// PostGIS values walk WKB into CH geo columns: `POINT EMPTY` keeps its NaN +/// axes, absent cells take the origin or an empty shape, and a subtype the +/// column cannot hold fails the request +#[tokio::test(flavor = "current_thread")] +async fn oracle_maps_postgis_into_geo_types() { + if !tools::extension("postgis") { + return; + } + let tmp = tempfile::tempdir().unwrap(); + let Some(guard) = start_pg(&tmp, ports::PG_SHADOW_PORT) else { + return; + }; + guard.sh.psql_one("CREATE EXTENSION postgis").unwrap(); + let geometry: u32 = guard + .sh + .psql_one("SELECT 'geometry'::regtype::oid") + .unwrap() + .parse() + .unwrap(); + let oracle = oracle_on(&guard.sh).await; + let wkt = |s: &str| OracleCell::TextInput(s.as_bytes().to_vec()); + let encode = async |ty, cells: Vec| { + let n = cells.len(); + let b = buf(geometry, cells); + let columns = [OracleRequestColumn { + ordinal: 0, + name: "geom", + target_type: ty, + buf: &b, + }]; + oracle + .encode_batch(Oracle::ANY_DATABASE, &columns, n, alloc()) + .await + }; + + let block = encode( + "Point", + vec![wkt("POINT EMPTY"), OracleCell::Default, wkt("POINT(1 2)")], + ) + .await + .expect("points encode"); + let axis = |i| { + let (_, bytes) = block + .column(0) + .and_then(|c| c.tuple_child(i)) + .and_then(|c| c.fixed()) + .expect("Float64 axis"); + bytes + .as_chunks::<8>() + .0 + .iter() + .map(|b| f64::from_le_bytes(*b)) + .collect::>() + }; + let (x, y) = (axis(0), axis(1)); + assert!(x[0].is_nan() && y[0].is_nan(), "{x:?} {y:?}"); + assert_eq!((&x[1..], &y[1..]), (&[0.0, 1.0][..], &[0.0, 2.0][..])); + + let block = encode( + "Polygon", + vec![OracleCell::Default, wkt("POLYGON((0 0,1 0,1 1,0 0))")], + ) + .await + .expect("polygons encode"); + let polygon = block.column(0).unwrap(); + assert_eq!(polygon.array_offsets().unwrap(), [0, 1]); + assert_eq!( + polygon.array_values().unwrap().array_offsets().unwrap(), + [4] + ); + + let cases = [ + ("Point", "POLYGON((0 0,1 0,1 1,0 0))", "WKB geometry type 3"), + ("Point", "POINT Z (1 2 3)", "WKB geometry type 1001"), + ("MultiPolygon", "MULTIPOINT(1 2)", "WKB geometry type 4"), + ]; + for (ty, value, expected) in cases { + let err = encode(ty, vec![wkt(value)]) + .await + .expect_err("subtype mismatch fails"); + let msg = err.to_string(); + assert!(msg.contains(expected), "{msg}"); + assert!(!err.retryable()); + } +} + #[tokio::test(flavor = "current_thread")] async fn oracle_defaults_fill_absent_cells() { let tmp = tempfile::tempdir().unwrap(); diff --git a/tests/runtime_config_e2e.rs b/tests/runtime_config_e2e.rs index 0f612367..69d6953f 100644 --- a/tests/runtime_config_e2e.rs +++ b/tests/runtime_config_e2e.rs @@ -82,6 +82,11 @@ //! * `ALTER DATABASE … SET intervalstyle = iso_8601` before a COPY load. //! * Expect: COPY and WAL rows render the same interval identically. //! +//! 13. `copy_load_matches_wal_for_postgis` +//! * COPY and WAL rows of each PostGIS subtype, plus a NULL row of each. +//! * Expect: 2-D subtypes land in CH geo columns, the rest as `ST_AsText` +//! WKT, and COPY rows read like WAL rows. +//! //! Source-side `config_*` install runs the real `sql/runtime_config_install.sql` //! inside the bootstrap schema dump, so the drills double as install-script @@ -1308,18 +1313,29 @@ async fn copy_load_converges( ch.query("CREATE DATABASE IF NOT EXISTS walshadow_test") .expect("create db"); - let mut pipeline = fx::build_pipeline(fx::BuildPipelineArgs { - tmp: &tmp, - source: &source, - shadow: &shadow, - shadow_filter_dir: &shadow_filter_dir, - shadow_stream_state, - ch_database: "walshadow_test", - ch_tcp_port: slot.ch_tcp, - mappings: vec![], - app_name, - ddl: Some(overlay_ddl_args()), - }) + // Worker binds only once recovery reaches consistency, so budget the dial + let socket = shadow.bridge_socket().expect("bridge configured"); + let bridge = walshadow::bridge::connect_with_budget(socket, 1, Duration::from_secs(30)) + .await + .expect("bridge connect"); + let mut pipeline = fx::build_pipeline_tuned( + fx::BuildPipelineArgs { + tmp: &tmp, + source: &source, + shadow: &shadow, + shadow_filter_dir: &shadow_filter_dir, + shadow_stream_state, + ch_database: "walshadow_test", + ch_tcp_port: slot.ch_tcp, + mappings: vec![], + app_name, + ddl: Some(overlay_ddl_args()), + }, + |_| {}, + Some(std::sync::Arc::new(walshadow::oracle::Oracle::new( + std::sync::Arc::new(bridge), + ))), + ) .await; let driver = fx::spawn_workload(&source, workload); @@ -1428,3 +1444,78 @@ async fn copy_load_pins_output_settings() { "COPY row renders like the WAL row" ); } + +/// PostGIS subtypes with a 2-D CH geo type land there, the rest as WKT; COPY +/// rows must read like WAL rows holding the same values +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn copy_load_matches_wal_for_postgis() { + if !fx::tools::requirements_available() || !fx::tools::extension("postgis") { + return; + } + let values = "'SRID=4326;POINT(0.1 0.30000000000000004)', 'LINESTRING(0 0,1 1,2 0)', \ + 'POLYGON((0 0,4 0,4 4,0 4,0 0),(1 1,2 1,2 2,1 1))', \ + 'MULTILINESTRING((0 0,1 1),(2 2,3 3,4 4))', \ + 'MULTIPOLYGON(((0 0,1 0,1 1,0 0)),((5 5,9 5,9 9,5 5),(6 6,7 6,7 7,6 6)))', \ + 'GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(0 0,1 1))', 'POINT Z (1 2 3)', \ + 'SRID=4326;POINT(30.5 50.25)', 'POINT(0.1 0.30000000000000004)'"; + let nulls = "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL"; + let schema_sql = format!( + "{INSTALL_SQL}\n\ + CREATE EXTENSION postgis;\n\ + CREATE SCHEMA app;\n\ + CREATE TABLE app.geo (id bigint PRIMARY KEY, pt geometry(Point,4326), \ + ln geometry(LineString), pg geometry(Polygon), mls geometry(MultiLineString), \ + mpg geometry(MultiPolygon), anyg geometry, ptz geometry(PointZ), \ + gpt geography(Point,4326), wkt geometry);\n\ + INSERT INTO app.geo VALUES (1, {values}), (3, {nulls});\n" + ); + let (ch, got) = copy_load_converges( + &schema_sql, + vec![ + "INSERT INTO walshadow.config_table (namespace, relname, replicate, initial_load) \ + VALUES ('app', 'geo', true, 'copy')" + .into(), + format!("INSERT INTO app.geo VALUES (2, {values}), (4, {nulls})"), + "SELECT pg_switch_wal()".into(), + ], + "walshadow-copy-postgis", + "SELECT count() FROM walshadow_test.app_geo FINAL WHERE _is_deleted = 0", + "4", + ) + .await; + assert_eq!(got, "4", "COPY and WAL rows reach CH"); + let types = ch + .query( + "SELECT groupArray(type) FROM (SELECT type FROM system.columns \ + WHERE database = 'walshadow_test' AND table = 'app_geo' \ + AND name IN ('pt', 'ln', 'pg', 'mls', 'mpg', 'anyg', 'ptz', 'gpt', 'wkt') \ + ORDER BY position)", + ) + .expect("ch column types"); + assert_eq!( + types, + "['Nullable(Point)','LineString','Polygon','MultiLineString','MultiPolygon',\ + 'Nullable(String)','Nullable(String)','Nullable(Point)','Nullable(String)']" + ); + let row = |id: u32| { + ch.query(&format!( + "SELECT pt, ln, pg, mls, mpg, anyg, ptz, gpt, wkt FROM walshadow_test.app_geo \ + FINAL WHERE _is_deleted = 0 AND id = {id}" + )) + .expect("ch row") + }; + let (copy, wal) = (row(1), row(2)); + assert_eq!(copy, wal, "COPY row reads like the WAL row"); + assert_eq!( + copy, + "(0.1,0.30000000000000004)\t[(0,0),(1,1),(2,0)]\t\ + [[(0,0),(4,0),(4,4),(0,4),(0,0)],[(1,1),(2,1),(2,2),(1,1)]]\t\ + [[(0,0),(1,1)],[(2,2),(3,3),(4,4)]]\t\ + [[[(0,0),(1,0),(1,1),(0,0)]],[[(5,5),(9,5),(9,9),(5,5)],[(6,6),(7,6),(7,7),(6,6)]]]\t\ + GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(0 0,1 1))\tPOINT Z (1 2 3)\t(30.5,50.25)\t\ + POINT(0.1 0.3)" + ); + let (copy, wal) = (row(3), row(4)); + assert_eq!(copy, wal, "NULL COPY row reads like the NULL WAL row"); + assert_eq!(copy, "\\N\t[]\t[]\t[]\t[]\t\\N\t\\N\t\\N\t\\N"); +}