Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion pgext/overlay.c
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,10 @@ ws_emit_type(StringInfo out, HeapTuple tup, TupleDesc desc) {

ws_put_oid(out, f->oid);
ws_put_name(out, &f->typname);
ws_put_oid(out, f->typbasetype);
ws_put_int(out, f->typtypmod);
ws_put_oid(out, f->typelem);
ws_put_oid(out, f->typarray);
}

typedef void (*WsEmitRow)(StringInfo out, HeapTuple tup, TupleDesc desc);
Expand Down Expand Up @@ -396,7 +400,7 @@ ws_catalog_plan(WsCatalog cat, WsCatalogPlan* plan) {
plan->indexid = InvalidOid;
plan->keyattno = InvalidAttrNumber;
plan->min_attnum = 0;
plan->ncols = 2;
plan->ncols = 6;
plan->emit = ws_emit_type;
return true;
}
Expand Down
2 changes: 1 addition & 1 deletion pgext/walshadow.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* of an unchanged layout changes */
#define WS_PROTO_VERSION 7
/* Bumped when any catalog projection changes shape */
#define WS_PROJECTION_VERSION 2
#define WS_PROJECTION_VERSION 3

/* request opcodes */
#define WS_OP_HELLO 0x01
Expand Down
16 changes: 13 additions & 3 deletions src/backfill/backfill_bootstrap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,12 @@ pub async fn seed_catalog_from_source(client: &Client) -> Result<CatalogMap> {
)
.await
.context("bootstrap: enumerate user relations on source")?;
let types = crate::pg::type_rows(
&client
.query(crate::pg::TYPE_SQL, &[])
.await
.context("bootstrap: read pg_type on source")?,
);

let mut by_oid: ahash::HashMap<Oid, (Arc<RelDescriptor>, Option<Oid>)> =
ahash::HashMap::default();
Expand All @@ -349,7 +355,7 @@ pub async fn seed_catalog_from_source(client: &Client) -> Result<CatalogMap> {
rel_node,
};
let replident = fetch_replident(client, replident_char, oid).await?;
let attributes = fetch_attributes(client, oid).await?;
let attributes = fetch_attributes(client, oid, &types).await?;
let desc = RelDescriptor {
rfn,
oid,
Expand Down Expand Up @@ -432,12 +438,16 @@ async fn fetch_replident(client: &Client, c: char, rel_oid: Oid) -> Result<ReplI
.map_err(|e| anyhow::anyhow!("bootstrap: {e} for relation {rel_oid}"))
}

async fn fetch_attributes(client: &Client, rel_oid: Oid) -> Result<Vec<RelAttr>> {
async fn fetch_attributes(
client: &Client,
rel_oid: Oid,
types: &ahash::HashMap<Oid, crate::ops::bridge::TypeRow>,
) -> Result<Vec<RelAttr>> {
let rows = client.query(crate::pg::ATTR_SQL, &[&rel_oid]).await?;
rows.iter()
.map(|row| {
crate::pg::RawAttr::from_row(row)
.build()
.build(types)
.map_err(|e| anyhow::anyhow!("bootstrap: {e}"))
})
.collect()
Expand Down
97 changes: 81 additions & 16 deletions src/catalog/shadow_catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -507,19 +507,14 @@ impl ShadowCatalog {
},
)
.await?;
// DROP COLUMN zeroes atttypid, and no pg_type row for it is what leaves
// the slot's type_name empty
let types = self
.resolve_names(
.resolve_types(
bridge,
"SELECT oid::oid, typname::text FROM pg_type \
WHERE oid = ANY($1::oid[])",
&attrs
.iter()
.map(|a| a.atttypid)
.filter(|oid| *oid != 0)
.collect(),
|r: TypeRow| (r.oid, r.typname),
top_xid,
NamePin {
lsn: pinned,
Expand Down Expand Up @@ -602,6 +597,34 @@ impl ShadowCatalog {
Ok(names)
}

/// Type rows for `wanted` and their domain bases, read in
/// [`Self::resolve_names`] order
async fn resolve_types(
&mut self,
bridge: &Bridge,
wanted: &BTreeSet<Oid>,
top_xid: u32,
pin: NamePin,
) -> Result<HashMap<Oid, TypeRow>> {
if wanted.is_empty() {
return Ok(HashMap::new());
}
if !pin.by_caller {
let list: Vec<Oid> = wanted.iter().copied().collect();
let rows = self.query_retry(&TYPE_CLOSURE_SQL, &[&list]).await?;
let types = crate::pg::type_rows(&rows);
if wanted.iter().all(|oid| types.contains_key(oid)) {
return Ok(types);
}
}
let scan = bridge.scan_at(Catalog::Type, top_xid, &[], pin.lsn).await?;
Ok(scan
.parse::<TypeRow>()?
.into_iter()
.map(|row| (row.oid, row))
.collect())
}

pub async fn current_database_oid(&mut self) -> Result<Oid> {
let row = self
.query_one_retry(
Expand Down Expand Up @@ -724,7 +747,7 @@ fn mirror_sql(scope_pred: &str) -> String {
), att AS MATERIALIZED (\
SELECT a.* FROM pg_attribute a JOIN scoped s ON s.oid = a.attrelid \
WHERE a.attnum >= 1\
) \
), {types} \
SELECT {position}, ARRAY[coalesce(pg_last_wal_replay_lsn()::text, '{no_replay}')] \
UNION ALL SELECT {class}, ARRAY[\
format('%s', c.oid), format('%s', c.relnamespace), format('%s', c.relname), \
Expand All @@ -750,8 +773,9 @@ fn mirror_sql(scope_pred: &str) -> String {
FROM pg_inherits i JOIN scoped s ON s.oid = i.inhrelid AND s.relispartition \
UNION ALL SELECT {namespace}, ARRAY[format('%s', n.oid), format('%s', n.nspname)] \
FROM pg_namespace n WHERE n.oid IN (SELECT relnamespace FROM scoped) \
UNION ALL SELECT {typ}, ARRAY[format('%s', t.oid), format('%s', t.typname)] \
FROM pg_type t WHERE t.oid IN (SELECT atttypid FROM att)",
UNION ALL SELECT {typ}, ARRAY[format('%s', t.oid), format('%s', t.typname), \
format('%s', t.typbasetype), format('%s', t.typtypmod), \
format('%s', t.typelem), format('%s', t.typarray)] FROM types t",
position = MIRROR_POSITION,
no_replay = NO_REPLAY,
class = Catalog::Class as u8,
Expand All @@ -760,9 +784,32 @@ fn mirror_sql(scope_pred: &str) -> String {
inherits = Catalog::Inherits as u8,
namespace = Catalog::Namespace as u8,
typ = Catalog::Type as u8,
types = type_closure("SELECT atttypid FROM att"),
)
}

/// `types` CTE over `seed` oids plus their domain bases, array elements and
/// array types, enough for [`crate::pg::RawAttr::build`]
fn type_closure(seed: &str) -> String {
format!(
"types AS (\
SELECT oid, typname, typbasetype, typtypmod, typelem, typarray \
FROM pg_type WHERE oid IN ({seed}) \
UNION \
SELECT t.oid, t.typname, t.typbasetype, t.typtypmod, t.typelem, t.typarray \
FROM types d JOIN pg_type t ON t.oid IN (d.typbasetype, d.typelem, d.typarray)\
)"
)
}

static TYPE_CLOSURE_SQL: LazyLock<String> = LazyLock::new(|| {
format!(
"WITH RECURSIVE {} \
SELECT oid, typname::text, typbasetype, typtypmod, typelem, typarray FROM types",
type_closure("SELECT unnest($1::oid[])")
)
});

static MIRROR_BATCH_SQL: LazyLock<String> = LazyLock::new(|| mirror_sql("c.oid = ANY($1::oid[])"));

/// [`MIRROR_BATCH_SQL`] over every eligible user relation. Predicate is
Expand Down Expand Up @@ -901,7 +948,7 @@ struct DescriptorRows {
indexes: Vec<IndexRow>,
inherits: Vec<InheritsRow>,
namespaces: HashMap<Oid, String>,
types: HashMap<Oid, String>,
types: HashMap<Oid, TypeRow>,
/// Requested relations. Other `class` rows supply ancestor information.
/// Use `None` to keep every row
requested: Option<BTreeSet<Oid>>,
Expand Down Expand Up @@ -940,7 +987,7 @@ impl DescriptorRows {
}
Catalog::Type => {
let row = TypeRow::parse(&cols)?;
out.types.insert(row.oid, row.typname);
out.types.insert(row.oid, row);
}
}
}
Expand Down Expand Up @@ -1037,7 +1084,7 @@ fn descriptor_from_rows(
attrs: &[AttributeRow],
indexes: &[IndexRow],
namespaces: &HashMap<Oid, String>,
types: &HashMap<Oid, String>,
types: &HashMap<Oid, TypeRow>,
db_node: Oid,
default_tablespace: Oid,
) -> Result<RelDescriptor> {
Expand Down Expand Up @@ -1080,14 +1127,13 @@ fn descriptor_from_rows(
typmod: attr.atttypmod,
not_null: attr.attnotnull,
dropped: attr.attisdropped,
type_name: types.get(&attr.atttypid).cloned(),
type_byval: attr.attbyval,
type_len: attr.attlen,
type_align: attr.attalign.to_string(),
type_storage: attr.attstorage.to_string(),
missing: None,
};
let mut rel_attr = raw.build().map_err(CatalogError::Parse)?;
let mut rel_attr = raw.build(types).map_err(CatalogError::Parse)?;
if let Some(hex) = attr.attmissingval.as_deref()
&& let Some(bytes) = decode_hex(hex)
{
Expand Down Expand Up @@ -1203,6 +1249,25 @@ mod tests {
pairs.iter().map(|(o, n)| (*o, (*n).to_owned())).collect()
}

fn types(pairs: &[(Oid, &str)]) -> HashMap<Oid, TypeRow> {
pairs
.iter()
.map(|(oid, name)| {
(
*oid,
TypeRow {
oid: *oid,
typname: (*name).into(),
typbasetype: 0,
typtypmod: -1,
typelem: 0,
typarray: 0,
},
)
})
.collect()
}

fn overlay(
class: &ClassRow,
attrs: &[AttributeRow],
Expand All @@ -1214,7 +1279,7 @@ mod tests {
attrs,
indexes,
namespaces,
&names(&[(23, "int4"), (25, "text")]),
&types(&[(23, "int4"), (25, "text")]),
5,
DEFAULT_TABLESPACE,
)
Expand Down Expand Up @@ -1346,7 +1411,7 @@ mod tests {
},
],
namespaces: names(&[(2200, "public")]),
types: names(&[(23, "int4")]),
types: types(&[(23, "int4")]),
requested: Some([16500, 16510].into_iter().collect()),
..DescriptorRows::default()
};
Expand Down
13 changes: 11 additions & 2 deletions src/ops/bridge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use crate::toast::FetchedValue;
/// Frame and op layouts. Must equal `WS_PROTO_VERSION` in `pgext/walshadow.h`
pub const PROTO_VERSION: u32 = 7;
/// Catalog column plans. Must equal `WS_PROJECTION_VERSION`
pub const PROJECTION_VERSION: u32 = 2;
pub const PROJECTION_VERSION: u32 = 3;

/// Match `WS_MAX_REQUEST_BYTES`
pub const MAX_REQUEST_BYTES: usize = 256 * 1024 * 1024;
Expand Down Expand Up @@ -112,7 +112,8 @@ impl Catalog {
Catalog::Class => 10,
Catalog::Attribute => 12,
Catalog::Index => 5,
Catalog::Namespace | Catalog::Type | Catalog::Inherits => 2,
Catalog::Namespace | Catalog::Inherits => 2,
Catalog::Type => 6,
}
}
}
Expand Down Expand Up @@ -1049,6 +1050,10 @@ impl ScanRow for NamespaceRow {
pub struct TypeRow {
pub oid: u32,
pub typname: String,
pub typbasetype: u32,
pub typtypmod: i32,
pub typelem: u32,
pub typarray: u32,
}

impl ScanRow for TypeRow {
Expand All @@ -1058,6 +1063,10 @@ impl ScanRow for TypeRow {
Ok(Self {
oid: field(row, 0)?.parse().map_err(|_| bad(row, 0))?,
typname: field(row, 1)?.to_owned(),
typbasetype: field(row, 2)?.parse().map_err(|_| bad(row, 2))?,
typtypmod: field(row, 3)?.parse().map_err(|_| bad(row, 3))?,
typelem: field(row, 4)?.parse().map_err(|_| bad(row, 4))?,
typarray: field(row, 5)?.parse().map_err(|_| bad(row, 5))?,
})
}
}
Expand Down
Loading
Loading