From 13b9c00e51ad46fe1e4f70b8ce2c5c08ad1e6106 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sun, 15 Nov 2026 21:45:48 -0800 Subject: [PATCH] init: the database's password comes from the sealed env relay, not MMDS The node generates it and the API stores only a sealed blob. PID 1 asks for it over the substrate vsock, writes it into the metadata file before the supervisor spawns, and fails closed when a substrate gives none. post_start's ALTER ROLE already runs on every start, so a fork's data directory takes this instance's value. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01LBGuEttMZVcyx4YEaaAvWE --- Cargo.lock | 1 + DECISIONS.md | 2 + DESIGN.md | 2 +- beyond-pg-init/ARCHITECTURE.md | 11 ++ beyond-pg-init/Cargo.toml | 1 + beyond-pg-init/src/bootsetup.rs | 27 ++++ beyond-pg-init/src/env_relay.rs | 240 ++++++++++++++++++++++++++++++++ beyond-pg-init/src/main.rs | 3 + beyond-pg-init/src/substrate.rs | 16 +++ 9 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 beyond-pg-init/src/env_relay.rs diff --git a/Cargo.lock b/Cargo.lock index 9aedf01..5e5a5d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,7 @@ dependencies = [ "nix 0.31.2", "rmp-serde", "serde", + "serde_bytes", "serde_json", "tokio", "tokio-vsock", diff --git a/DECISIONS.md b/DECISIONS.md index d777a01..edadc7f 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -770,6 +770,8 @@ resized VM picks up new tuning automatically. ### G-002 — MMDS for superuser password and tier configuration +**Update.** `POSTGRES_PASSWORD` no longer comes from MMDS: the node generates it and PID 1 reads it from the sealed env relay (`beyond-pg-init/ARCHITECTURE.md`, "The database's password"); the rest of this entry stands. + **Decision.** `beyond-pg` reads `POSTGRES_PASSWORD`, `POSTGRES_DATABASE`, `BEYOND_PG_TIER`, `BEYOND_VOLUME_EPHEMERAL`, `BEYOND_PG_ARCHIVE_TARGET` directly from MMDS at diff --git a/DESIGN.md b/DESIGN.md index f746330..6879800 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -371,7 +371,7 @@ subcommand because there's no external invoker that needs a CLI surface. 1. **Read MMDS** (direct HTTP to `169.254.169.254`; retry with backoff; env var fallback for local dev). Read: - `BEYOND_PG_TIER` (default `single`) - `BEYOND_VOLUME_EPHEMERAL` (default `false`) - - `POSTGRES_PASSWORD` (required; fail closed if missing) + - `POSTGRES_PASSWORD` (required; fail closed if missing). Not from MMDS on Beyond: `beyond-pg-init` takes it from the sealed env relay and writes it into the metadata file first - `POSTGRES_DATABASE` (default `postgres`) - `BEYOND_PG_ARCHIVE_TARGET` (optional) - `BEYOND_PG_WAL_SINK` (optional; HTTP endpoint of WAL sink; enables Tier 1.5 durability) diff --git a/beyond-pg-init/ARCHITECTURE.md b/beyond-pg-init/ARCHITECTURE.md index 048da69..65b9b62 100644 --- a/beyond-pg-init/ARCHITECTURE.md +++ b/beyond-pg-init/ARCHITECTURE.md @@ -35,6 +35,12 @@ bootsetup::run() │ └─ on failure → POSTGRES_PASSWORD env var → synthetic JSON │ absent? ──► exit(1) FATAL │ + ├─ (after the substrate thread starts) adopt_relay_password() + │ ├─ the substrate loop sends the env ask ([0xE1] in an AppMessage) every second until answered + │ ├─ answer ([0xE2][{generation, vars}]) holds POSTGRES_PASSWORD → merged into /run/mmds/metadata.json + │ ├─ no substrate at all (Docker test image) → metadata file used as it is + │ └─ a substrate and no password in 300s → drop any MMDS password, exit(1) FATAL + │ └─ setup_zram() ├─ skip if zram0 already in /proc/swaps ├─ modprobe zram @@ -256,6 +262,7 @@ All configuration arrives via MMDS at boot. `beyond-pg-init` itself has no confi | `/proc` mount fails | `exit(1)` immediately — no further initialization | None; VM is unusable without `/proc` | | Other filesystem mount fails | `eprintln!` WARNING; boot continues | Best-effort; postgres may work without e.g. `/dev/shm` on older kernels | | MMDS unreachable after 30 retries (300ms) | Falls back to `POSTGRES_PASSWORD` env var | If env var also absent, `exit(1)` FATAL | +| Env relay gives no password in 300s (substrate present) | `exit(1)` FATAL; any POSTGRES_PASSWORD MMDS carried is dropped, never used | The VM does not serve a database with a password nobody can name | | MMDS write fails | `exit(1)` FATAL | None | | `zram` unavailable | WARNING; no swap configured | Postgres runs without swap; OOM killer may terminate it under memory pressure | | Supervisor crashes | `pidfd` becomes readable; supervisor reaped; respawn after 1s with retry loop | Automatic; in-flight RPCs at crash time are lost | @@ -267,3 +274,7 @@ All configuration arrives via MMDS at boot. `beyond-pg-init` itself has no confi | Upgrade binary path outside allow-list | Error response `{ ok: false, error: "..." }` | No exec attempted; caller can retry with a valid path | | Shutdown during handoff | SIGTERM queued; main loop sees it after `perform_handoff` returns | If handoff was in progress it completes (or aborts) first, then drain+poweroff runs | | `poweroff` syscall fails | `exit(1)` — kernel panics when PID 1 exits | Kernel panic causes Firecracker to halt the VM | + +## The database's password (`env_relay.rs`) + +The superuser password is generated by the node's postgres driver, sealed to the tenant's node key and stored by the API as a sealed variable: no central process holds it, and MMDS never carries it. PID 1 asks for it over the substrate vsock it already owns (the ask and the answer are `AppMessage` frames; `rustlib/wire` `env.rs` in the Beyond repo defines the kinds, mirrored in `env_relay.rs` and pinned by its tests), writes it into the metadata file before the supervisor spawns, and the supervisor's `post_start` runs `ALTER ROLE postgres PASSWORD` on **every** start, so a data directory that carries another instance's role hash (a fork, a restore) takes this instance's value. Fail closed: a substrate and no password means init exits. diff --git a/beyond-pg-init/Cargo.toml b/beyond-pg-init/Cargo.toml index 9ac3669..8c1810e 100644 --- a/beyond-pg-init/Cargo.toml +++ b/beyond-pg-init/Cargo.toml @@ -25,3 +25,4 @@ nix = { version = "0.31", features = ["mount"] } tokio = { version = "1", features = ["rt", "macros", "net", "time", "io-util", "sync"] } tokio-vsock = "0.7" rmp-serde = "1" +serde_bytes = "0.11" diff --git a/beyond-pg-init/src/bootsetup.rs b/beyond-pg-init/src/bootsetup.rs index 27ed3d0..6381631 100644 --- a/beyond-pg-init/src/bootsetup.rs +++ b/beyond-pg-init/src/bootsetup.rs @@ -298,6 +298,33 @@ fn fetch_mmds() { } } +/// The database's password comes from the sealed env relay (`env_relay`), written into the metadata file the supervisor reads, before the +/// supervisor starts. With a substrate and no password within the wait, init exits: a database is never started without one, and never with one +/// MMDS carried. With no substrate at all (the Docker test image) the metadata is used as it is. +pub fn adopt_relay_password() { + let path = beyond_pg_core::mmds::MMDS_PATH; + let read = || { + std::fs::read_to_string(path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .unwrap_or_else(|| serde_json::json!({})) + }; + match crate::env_relay::wait(crate::env_relay::WAIT) { + crate::env_relay::Outcome::Password(pw) => { + write_mmds_file(&crate::env_relay::with_password(read(), &pw)); + eprintln!("[init] database password received from the env relay"); + } + crate::env_relay::Outcome::NoSubstrate => { + eprintln!("[init] no substrate: using the metadata file as it is"); + } + crate::env_relay::Outcome::TimedOut => { + write_mmds_file(&crate::env_relay::without_password(read())); + eprintln!("[init] FATAL: the env relay gave no database password"); + std::process::exit(1); + } + } +} + fn write_mmds_file(json: &serde_json::Value) { let path = beyond_pg_core::mmds::MMDS_PATH; let tmp = format!("{path}.tmp"); diff --git a/beyond-pg-init/src/env_relay.rs b/beyond-pg-init/src/env_relay.rs new file mode 100644 index 0000000..637ed9a --- /dev/null +++ b/beyond-pg-init/src/env_relay.rs @@ -0,0 +1,240 @@ +//! The database's password comes from the sealed env relay, never from MMDS. +//! +//! The password is generated by the node's postgres driver (the platform-generated secrets of the Beyond repo), sealed to the tenant's node key +//! and stored by the API as a sealed variable: nothing central ever holds it. PID 1 asks for it over the substrate vsock it already owns, the way +//! every guest asks its driver for its env, and writes it into the metadata file the supervisor reads, before the supervisor starts. The +//! supervisor's `post_start` then runs `ALTER ROLE postgres PASSWORD` on EVERY start, so a data directory that carries another instance's role +//! hash (a fork, a restore) takes this instance's value. +//! +//! Wire (`rustlib/wire` `env.rs` and `rustlib/vsock-protocol` in the Beyond repo, mirrored here so this repo builds standalone and pinned by the +//! tests below): the ask is an `AppMessage` (0x20) whose bytes are `[0xE1]`; the answer is an `AppMessage` whose bytes are `[0xE2][msgpack +//! {generation, vars}]`. The ask is repeated every second until answered: the driver answers only once the API has stored the value. +//! +//! Fail closed: with a substrate and no value after [`WAIT`], init exits (the VM never serves a database with a password nobody can name). With +//! no substrate at all (the Docker test image) the metadata file is used as it is. + +use std::collections::BTreeMap; +use std::sync::{Condvar, Mutex}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +/// Guest -> driver: "send me my env". `wire::nats::PLATFORM_ENV_REQUEST_KIND`. +pub const ASK_KIND: u8 = 0xE1; +/// Driver -> guest: the env. `wire::nats::PLATFORM_ENV_KIND`. +pub const ANSWER_KIND: u8 = 0xE2; +/// `vsock_protocol::MessageType::AppMessage`. +pub const MSG_APP: u8 = 0x20; +/// How often the ask is repeated until it is answered. +pub const ASK_EVERY: Duration = Duration::from_secs(1); +/// How long init waits for the password before it gives up (and the VM with it). +pub const WAIT: Duration = Duration::from_secs(300); +/// The one variable the supervisor needs from the relay. +pub const PASSWORD_KEY: &str = "POSTGRES_PASSWORD"; + +#[derive(Serialize, Deserialize)] +struct AppMessage { + #[serde(with = "serde_bytes")] + bytes: Vec, +} + +#[derive(Deserialize)] +struct EnvPayload { + #[allow(dead_code)] + generation: u64, + vars: BTreeMap, +} + +/// What the wait ended with. +#[derive(Debug, PartialEq, Eq)] +pub enum Outcome { + /// The relay answered with a password. + Password(String), + /// There is no substrate (no AF_VSOCK): the metadata file is used as it is. + NoSubstrate, + /// A substrate, and no password within the wait. + TimedOut, +} + +#[derive(Default)] +struct State { + password: Option, + no_substrate: bool, +} + +static STATE: Mutex = Mutex::new(State { + password: None, + no_substrate: false, +}); +static CV: Condvar = Condvar::new(); + +fn state() -> std::sync::MutexGuard<'static, State> { + STATE.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// True until a password has arrived: the substrate loop keeps asking. +pub fn needs_password() -> bool { + state().password.is_none() +} + +/// The substrate's first connect failed: there is none, so nothing will ever answer. +pub fn no_substrate() { + state().no_substrate = true; + CV.notify_all(); +} + +/// The ask, as a complete substrate frame `[len][0x20][msgpack {bytes: [0xE1]}]`. +pub fn ask_frame() -> Vec { + let body = rmp_serde::to_vec_named(&AppMessage { + bytes: vec![ASK_KIND], + }) + .expect("an AppMessage encodes"); + let mut frame = ((body.len() as u32) + 1).to_be_bytes().to_vec(); + frame.push(MSG_APP); + frame.extend_from_slice(&body); + frame +} + +/// An inbound `AppMessage` payload (the msgpack after the type byte). If it is an env answer holding the password, keep it and wake the +/// waiter. Anything else (another primitive message, a malformed or empty answer) is ignored: the ask repeats. +pub fn offer(app_payload: &[u8]) { + let Ok(msg) = rmp_serde::from_slice::(app_payload) else { + return; + }; + let Some((&kind, rest)) = msg.bytes.split_first() else { + return; + }; + if kind != ANSWER_KIND { + return; + } + let Ok(env) = rmp_serde::from_slice::(rest) else { + return; + }; + if let Some(pw) = env.vars.get(PASSWORD_KEY).filter(|p| !p.is_empty()) { + state().password = Some(pw.clone()); + CV.notify_all(); + } +} + +/// Block until the relay has answered, there is no substrate, or `timeout` passes. +pub fn wait(timeout: Duration) -> Outcome { + let deadline = std::time::Instant::now() + timeout; + let mut st = state(); + loop { + if let Some(pw) = &st.password { + return Outcome::Password(pw.clone()); + } + if st.no_substrate { + return Outcome::NoSubstrate; + } + let left = deadline.saturating_duration_since(std::time::Instant::now()); + if left.is_zero() { + return Outcome::TimedOut; + } + st = CV + .wait_timeout(st, left) + .unwrap_or_else(|e| e.into_inner()) + .0; + } +} + +/// `metadata` with the relay's password in it, and no other source of one: the supervisor reads `latest.meta-data.POSTGRES_PASSWORD`. +pub fn with_password(mut metadata: serde_json::Value, password: &str) -> serde_json::Value { + if !metadata["latest"]["meta-data"].is_object() { + metadata["latest"] = serde_json::json!({ "meta-data": {} }); + } + metadata["latest"]["meta-data"][PASSWORD_KEY] = serde_json::Value::String(password.to_string()); + metadata +} + +/// Without a password at all: drop whatever MMDS carried, so a stale or planted value is never the one a database starts with. +pub fn without_password(mut metadata: serde_json::Value) -> serde_json::Value { + if let Some(m) = metadata["latest"]["meta-data"].as_object_mut() { + m.remove(PASSWORD_KEY); + } + metadata +} + +#[cfg(test)] +mod tests { + use super::*; + + fn answer(vars: &[(&str, &str)]) -> Vec { + let mut bytes = vec![ANSWER_KIND]; + let payload = serde_json::json!({ "generation": 3, "vars": vars.iter().cloned().collect::>() }); + // Named-field msgpack, as `vsock_protocol::encode` writes it. + bytes.extend(rmp_serde::to_vec_named(&payload).unwrap()); + rmp_serde::to_vec_named(&AppMessage { bytes }).unwrap() + } + + /// Pins the ask against the host's `AppMessagePayload` decoder: BE length, 0x20, a msgpack map {bytes: bin [0xE1]}. + #[test] + fn the_ask_frame_is_stable() { + let f = ask_frame(); + assert_eq!( + u32::from_be_bytes(f[0..4].try_into().unwrap()) as usize, + f.len() - 4 + ); + assert_eq!(f[4], MSG_APP); + let m: AppMessage = rmp_serde::from_slice(&f[5..]).unwrap(); + assert_eq!(m.bytes, vec![ASK_KIND]); + // bin, not an array of ints: the host's `serde_bytes` field only reads a bin ({bytes: bin8 [0xE1]}). + assert!(f[5..].windows(3).any(|w| w == [0xc4, 0x01, ASK_KIND])); + } + + #[test] + fn only_an_answer_with_a_password_counts() { + // Not tied to the global: parse by hand. + let ok = answer(&[(PASSWORD_KEY, "pw"), ("OTHER", "x")]); + let m: AppMessage = rmp_serde::from_slice(&ok).unwrap(); + assert_eq!(m.bytes[0], ANSWER_KIND); + let env: EnvPayload = rmp_serde::from_slice(&m.bytes[1..]).unwrap(); + assert_eq!(env.vars.get(PASSWORD_KEY).map(String::as_str), Some("pw")); + } + + #[test] + fn offer_wakes_the_waiter_and_empty_or_foreign_messages_do_not() { + // One test owns the global state (tests share the process). + *state() = State::default(); + offer(&answer(&[("OTHER", "x")])); + offer(&answer(&[(PASSWORD_KEY, "")])); + let mut foreign = rmp_serde::to_vec_named(&AppMessage { + bytes: vec![0x0c, 1, 2], + }) + .unwrap(); + offer(&foreign); + foreign.clear(); + assert_eq!(wait(Duration::from_millis(20)), Outcome::TimedOut); + let t = std::thread::spawn(|| { + std::thread::sleep(Duration::from_millis(30)); + offer(&answer(&[(PASSWORD_KEY, "s3cret")])); + }); + assert_eq!( + wait(Duration::from_secs(5)), + Outcome::Password("s3cret".into()) + ); + t.join().unwrap(); + assert!(!needs_password()); + *state() = State::default(); + no_substrate(); + assert_eq!(wait(Duration::from_secs(1)), Outcome::NoSubstrate); + *state() = State::default(); + } + + #[test] + fn the_password_replaces_any_mmds_value_and_its_absence_removes_one() { + let mmds = serde_json::json!({"latest": {"meta-data": {"POSTGRES_PASSWORD": "stale", "BEYOND_PG_TIER": "single"}}}); + let with = with_password(mmds.clone(), "fresh"); + assert_eq!(with["latest"]["meta-data"]["POSTGRES_PASSWORD"], "fresh"); + assert_eq!(with["latest"]["meta-data"]["BEYOND_PG_TIER"], "single"); + assert!( + without_password(mmds)["latest"]["meta-data"] + .get("POSTGRES_PASSWORD") + .is_none() + ); + assert_eq!( + with_password(serde_json::json!({}), "p")["latest"]["meta-data"]["POSTGRES_PASSWORD"], + "p" + ); + } +} diff --git a/beyond-pg-init/src/main.rs b/beyond-pg-init/src/main.rs index dfa0497..7f0a7f9 100644 --- a/beyond-pg-init/src/main.rs +++ b/beyond-pg-init/src/main.rs @@ -9,6 +9,7 @@ #[cfg(target_os = "linux")] mod bootsetup; +mod env_relay; #[cfg(target_os = "linux")] mod substrate; #[cfg(target_os = "linux")] @@ -31,6 +32,8 @@ fn main() -> ! { // before considering the create successful. Spawned after boot setup (so // the network/MMDS are up) and before the supervise loop takes over. substrate::spawn_handshake(); + // The password is the one thing the supervisor cannot start without, and it comes from the env relay over that connection. + bootsetup::adopt_relay_password(); supervise::run(); } diff --git a/beyond-pg-init/src/substrate.rs b/beyond-pg-init/src/substrate.rs index ca13e87..2b0c95a 100644 --- a/beyond-pg-init/src/substrate.rs +++ b/beyond-pg-init/src/substrate.rs @@ -203,6 +203,7 @@ fn run() { Ok(rt) => rt, Err(e) => { eprintln!("[init] WARNING: substrate-vsock tokio runtime build failed: {e}"); + crate::env_relay::no_substrate(); return; } }; @@ -244,6 +245,7 @@ async fn handshake() { eprintln!( "[init] WARNING: substrate vsock connect failed; guest-ready unreported: {e}" ); + crate::env_relay::no_substrate(); return; } eprintln!( @@ -267,6 +269,7 @@ async fn handshake() { // outage. Bail rather than spin. Err(e) => { eprintln!("[init] WARNING: encode Ready frame failed: {e}"); + crate::env_relay::no_substrate(); return; } }; @@ -274,6 +277,7 @@ async fn handshake() { if let Err(e) = conn.write_all(&frame).await { if !connected_once { eprintln!("[init] WARNING: substrate Ready write failed: {e}"); + crate::env_relay::no_substrate(); return; } eprintln!("[init] substrate Ready write failed on redial ({e}); retrying"); @@ -332,9 +336,19 @@ where let mut psi_interval = tokio::time::interval(RESOURCE_STATS_PERIOD); psi_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // The database's password: asked for until the driver answers (it answers once the API has stored the value). + let mut ask_interval = tokio::time::interval(crate::env_relay::ASK_EVERY); + ask_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut len_buf = [0u8; 4]; loop { tokio::select! { + _ = ask_interval.tick(), if crate::env_relay::needs_password() => { + if conn.write_all(&crate::env_relay::ask_frame()).await.is_err() { + return ConnEnd::Disconnected; + } + let _ = conn.flush().await; + } // Periodic PSI report. _ = psi_interval.tick() => { if let Some(frame) = encode_resource_stats_frame() { @@ -367,6 +381,8 @@ where } let _ = conn.flush().await; } + // A primitive message from the driver: the env answer, if that is what it is. + crate::env_relay::MSG_APP => crate::env_relay::offer(&frame[1..]), MSG_SHUTDOWN => { eprintln!("[init] substrate requested shutdown; vsock loop exiting"); return ConnEnd::Shutdown;