From 640b303eb301db99f0c8a03c53ca9f3b40b0c4a8 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 09:23:14 -0700 Subject: [PATCH 01/61] fix(api): correct date/time param epoch and NUMERIC insert scale - Date, Timestamp and OffsetTimestamp parameters were bound with absolute Julian values, but Hyper's binary Bind expects PostgreSQL-epoch values (2000-01-01) and applies the Julian offset itself, so a bound 2024-01-15 read back as 8736-02-22. - Inserter/AsyncInserter::add_numeric ignored the value's own scale, so Numeric::new(12345, 1) into NUMERIC(10, 2) was stored as 123.45. The value is now rescaled to the column's scale; a lossy or overflowing rescale is an Error::Conversion. Both are covered by tests that failed against the real hyperd before the fix. Also fixes the varchar round-trip test and updates add_numeric docs. --- hyperdb-api/CHANGELOG.md | 15 +++ hyperdb-api/src/async_inserter.rs | 45 +++---- hyperdb-api/src/inserter.rs | 157 ++++++++++++++++--------- hyperdb-api/src/params.rs | 16 +-- hyperdb-api/tests/param_types_tests.rs | 41 ++++++- hyperdb-api/tests/roundtrip_tests.rs | 98 +++++++++++++-- 6 files changed, 271 insertions(+), 101 deletions(-) diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index f1d96a54..d3b790a2 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed + +- **`Date`, `Timestamp` and `OffsetTimestamp` query parameters were bound with + the wrong epoch.** `ToSqlParam::encode_param` sent absolute Julian-based + values, but Hyper's binary `Bind` expects PostgreSQL-epoch values (days / + microseconds since 2000-01-01) and applies the Julian offset itself, so a + bound `2024-01-15` came back as `8736-02-22`. The parameters now send + PG-epoch values. The `Inserter` (HyperBinary COPY) path was unaffected. +- **`Inserter::add_numeric` and `AsyncInserter::add_numeric` ignored the + value's own scale.** The unscaled integer was written as-is, so + `Numeric::new(12345, 1)` (1234.5) inserted into a `NUMERIC(10, 2)` column was + stored as `123.45` — a silent 10× magnitude error. The value is now rescaled + to the column's declared scale; a value that would lose non-zero digits or + overflow is rejected with `Error::Conversion` instead of being stored wrong. + ## [1.0.0-rc.4] - 2026-09-08 ### Fixed diff --git a/hyperdb-api/src/async_inserter.rs b/hyperdb-api/src/async_inserter.rs index 2001490d..497013b6 100644 --- a/hyperdb-api/src/async_inserter.rs +++ b/hyperdb-api/src/async_inserter.rs @@ -174,35 +174,22 @@ impl<'conn> AsyncInserter<'conn> { pub fn add_interval(&mut self, value: Interval) -> Result<()> { self.chunk.add_interval(value) } - /// Adds a `Numeric` value (NUMERIC). The encoding (small vs big) is - /// chosen from the table definition's column precision at this position. + /// Adds a `Numeric` value (NUMERIC). The value is rescaled to the + /// column's declared scale, and the encoding (small vs big) is chosen from + /// the column's precision at this position. /// /// # Errors /// - /// Returns an error if the column's precision cannot be determined from - /// the table definition (NUMERIC columns must be declared with explicit - /// precision/scale). + /// - Returns an error if the column's precision and scale cannot be + /// determined from the table definition (NUMERIC columns must be + /// declared with explicit precision/scale). + /// - Returns [`Error::Conversion`] if the value cannot be represented at + /// the column's scale without losing digits or overflowing. pub fn add_numeric(&mut self, value: Numeric) -> Result<()> { let column_index = self.chunk.column_index(); - let precision = self - .table_def - .columns - .get(column_index) - .and_then(super::table_definition::ColumnDefinition::sql_type) - .and_then(|t| t.precision()) - .ok_or_else(|| { - let col_name = self - .table_def - .columns - .get(column_index) - .map_or("", |c| c.name.as_str()); - Error::conversion(format!( - "Cannot determine numeric precision for column '{col_name}' at index {column_index}. \ - Ensure the column is defined with explicit SqlType including precision." - )) - })?; + let (precision, unscaled) = + crate::inserter::numeric_for_column(&self.table_def, column_index, value)?; if precision <= Numeric::SMALL_NUMERIC_MAX_PRECISION { - let unscaled = value.unscaled_value(); let narrowed = i64::try_from(unscaled).map_err(|_| { Error::conversion(format!( "Numeric value {unscaled} is out of range for i64 storage (precision {precision})" @@ -210,7 +197,7 @@ impl<'conn> AsyncInserter<'conn> { })?; self.chunk.add_i64(narrowed) } else { - self.chunk.add_data128(&value.to_packed()) + self.chunk.add_data128(&unscaled.to_le_bytes()) } } @@ -221,8 +208,9 @@ impl<'conn> AsyncInserter<'conn> { /// /// - Returns [`Error::InvalidTableDefinition`] if the column count for the row doesn't match /// the table definition. - /// - Returns [`Error::Server`] / [`Error::Io`] on transport failures - /// during the auto-flush. + /// - Returns [`Error::Server`] if the server rejects the COPY or its data. + /// - Returns [`Error::Connection`] on a transport I/O failure, or + /// [`Error::Closed`] if the server closed the connection. pub async fn end_row(&mut self) -> Result<()> { self.chunk.end_row()?; self.row_count += 1; @@ -272,8 +260,9 @@ impl<'conn> AsyncInserter<'conn> { /// # Errors /// /// - Returns [`Error::InvalidTableDefinition`] if there's an incomplete row (partial column). - /// - Returns [`Error::Server`] / [`Error::Io`] if the COPY session or - /// transport fails. + /// - Returns [`Error::Server`] if the server rejects the COPY or its data. + /// - Returns [`Error::Connection`] on a transport I/O failure, or + /// [`Error::Closed`] if the server closed the connection. pub async fn execute(&mut self) -> Result { if self.chunk.column_index() != 0 { return Err(Error::invalid_table_definition( diff --git a/hyperdb-api/src/inserter.rs b/hyperdb-api/src/inserter.rs index 6d5b5fb0..f6e75de1 100644 --- a/hyperdb-api/src/inserter.rs +++ b/hyperdb-api/src/inserter.rs @@ -209,8 +209,7 @@ impl<'conn> Inserter<'conn> { /// /// * `connection` - The database connection. /// * `inserter_def` - Defines the columns to be provided to the inserter (staging table). - /// * `target_table` - The qualified name of the target table to insert into. - /// Use `TableDefinition::qualified_name()` for properly escaped names like `"schema"."table"`. + /// * `target_table` - The target table; anything convertible to [`TableName`](crate::TableName), e.g. `"schema.table"`. /// * `mappings` - Column mappings defining how values are transformed. /// /// # Example @@ -250,8 +249,7 @@ impl<'conn> Inserter<'conn> { /// ColumnMapping::with_expression("created_at", "NOW()"), /// ]; /// - /// // For simple table names in the public schema, use quoted name - /// // For qualified names, use target_table_def.qualified_name() + /// // Any value convertible to `TableName` works: "orders", "public.orders", or a `TableName`. /// let mut inserter = Inserter::with_column_mappings(&conn, &inserter_def, "orders", &mappings)?; /// /// inserter.add_row(&[&1i32, &"Widget", &5i32, &10.0f64])?; @@ -295,7 +293,9 @@ impl<'conn> Inserter<'conn> { self.table_def.column_count() } - /// Returns the number of complete rows buffered. + /// Returns the number of complete rows added so far, including rows already flushed to the server. + /// + /// Reset to 0 by [`execute`](Self::execute) and [`cancel`](Self::cancel). #[must_use] pub fn row_count(&self) -> u64 { self.row_count @@ -378,7 +378,8 @@ impl<'conn> Inserter<'conn> { /// /// # Errors /// - /// See [`add_bool`](Self::add_bool). + /// See [`add_bool`](Self::add_bool). Returns [`Error::Conversion`] if the value is longer + /// than `u32::MAX` bytes. #[inline] pub fn add_str(&mut self, value: &str) -> Result<()> { self.chunk.add_str(value) @@ -388,7 +389,8 @@ impl<'conn> Inserter<'conn> { /// /// # Errors /// - /// See [`add_bool`](Self::add_bool). + /// See [`add_bool`](Self::add_bool). Returns [`Error::Conversion`] if the value is longer + /// than `u32::MAX` bytes. #[inline] pub fn add_bytes(&mut self, value: &[u8]) -> Result<()> { self.chunk.add_bytes(value) @@ -454,8 +456,8 @@ impl<'conn> Inserter<'conn> { /// (COPY is TCP-only) and no COPY session exists yet. /// - Returns [`Error::Server`] if the server rejects the `COPY IN` start /// or the subsequent data send. - /// - Returns [`Error::Io`] on transport-level I/O failures while writing - /// the chunk. + /// - Returns [`Error::Connection`] on a transport I/O failure, or + /// [`Error::Closed`] if the server closed the connection. pub fn flush(&mut self) -> Result<()> { if self.chunk.is_empty() { return Ok(()); @@ -616,7 +618,8 @@ impl<'conn> Inserter<'conn> { /// /// # Errors /// - /// See [`add_bool`](Self::add_bool). + /// See [`add_bool`](Self::add_bool). Returns [`Error::Conversion`] if the value is longer + /// than `u32::MAX` bytes. #[inline] pub fn add_geography(&mut self, value: &Geography) -> Result<()> { self.chunk.add_geography(value) @@ -624,41 +627,24 @@ impl<'conn> Inserter<'conn> { /// Adds a Numeric value. /// - /// For NUMERIC(precision, scale) where precision ≤ [`Numeric::SMALL_NUMERIC_MAX_PRECISION`] - /// (18), the value is stored as i64. For higher precision, 128-bit storage is used. + /// The value is rescaled to the column's declared scale (`NUMERIC(p, s)`), + /// so `Numeric::new(12345, 1)` (1234.5) inserted into a `NUMERIC(10, 2)` + /// column is stored as `1234.50`. For precision ≤ + /// [`Numeric::SMALL_NUMERIC_MAX_PRECISION`] (18) the value is stored as + /// i64. For higher precision, 128-bit storage is used. /// /// # Errors /// - /// Returns an error if the column's precision cannot be determined from the - /// table definition. Ensure that NUMERIC columns are defined with explicit - /// `SqlType` information including precision. + /// - Returns an error if the column's precision and scale cannot be + /// determined from the table definition. Ensure that NUMERIC columns are + /// defined with explicit `SqlType` information including precision. + /// - Returns [`Error::Conversion`] if the value cannot be represented at + /// the column's scale without losing digits or overflowing. pub fn add_numeric(&mut self, value: Numeric) -> Result<()> { let column_index = self.chunk.column_index(); - - // Check the column's precision to determine storage format - let precision = self - .table_def - .columns - .get(column_index) - .and_then(super::table_definition::ColumnDefinition::sql_type) - .and_then(|t| t.precision()) - .ok_or_else(|| { - let col_name = self - .table_def - .columns - .get(column_index) - .map_or("", |c| c.name.as_str()); - Error::conversion(format!( - "Cannot determine numeric precision for column '{col_name}' at index {column_index}. \ - Ensure the column is defined with explicit SqlType including precision.\n\n\ - Example fix:\n \ - table_def.add_column_with_type(\"{col_name}\", SqlType::Numeric {{ precision: 10, scale: 2 }}, true);" - )) - })?; - + let (precision, unscaled) = numeric_for_column(&self.table_def, column_index, value)?; if precision <= Numeric::SMALL_NUMERIC_MAX_PRECISION { // Small numeric: stored as i64 - let unscaled = value.unscaled_value(); let narrowed = i64::try_from(unscaled).map_err(|_| { Error::conversion(format!( "Numeric value {unscaled} is out of range for i64 storage (precision {precision})" @@ -667,7 +653,7 @@ impl<'conn> Inserter<'conn> { self.chunk.add_i64(narrowed) } else { // Big numeric: stored as 128-bit - self.chunk.add_data128(&value.to_packed()) + self.chunk.add_data128(&unscaled.to_le_bytes()) } } @@ -683,10 +669,10 @@ impl<'conn> Inserter<'conn> { /// /// # Errors /// - /// Returns an error if: - /// - There's an incomplete row (`column_index` != 0) - /// - The COPY connection fails to start - /// - Sending data fails + /// - Returns [`Error::InvalidTableDefinition`] if a row is incomplete. + /// - Returns [`Error::FeatureNotSupported`] on a gRPC connection. + /// - Returns [`Error::Server`] if the server rejects the COPY or its data. + /// - Returns [`Error::Connection`] or [`Error::Closed`] on transport failure. pub fn execute(&mut self) -> Result { if self.chunk.column_index() != 0 { return Err(Error::invalid_table_definition( @@ -774,7 +760,7 @@ impl<'conn> Inserter<'conn> { /// Column mappings allow you to: /// - Insert values directly from the inserter stream /// - Compute values using SQL expressions -/// - Use server-side functions like `NOW()` or `DEFAULT` +/// - Use server-side functions such as `NOW()` /// /// # Example /// @@ -879,8 +865,9 @@ impl ColumnMapping { /// - `bool` /// - `&str`, `String` /// - `Option` where `T: IntoValue` (for nullable columns) -/// - Date/time types: `Date`, `Time`, `Timestamp`, `Interval` -/// - `Numeric`, `Geography`, `Vec` (bytes) +/// - Date/time types: `Date`, `Time`, `Timestamp`, `OffsetTimestamp`, `Interval` +/// - `Numeric`, `Geography`, `[u8]` / `Vec` (bytes) +/// - Shared references to the above, except `Option` and `Vec` /// /// # Example /// @@ -912,8 +899,8 @@ pub trait IntoValue { /// # Errors /// /// Implementations call the matching `Inserter::add_*` method and - /// forward its error — see [`Inserter::add_bool`] for the shared - /// failure modes (too many columns, NULL into non-nullable, etc). + /// forward its error: see [`Inserter::add_bool`] (too many columns) and + /// [`Inserter::add_null`] (NULL into a `NOT NULL` column). fn add_to_inserter(&self, inserter: &mut Inserter<'_>) -> Result<()>; } @@ -1672,7 +1659,8 @@ impl InsertChunk { /// /// # Errors /// - /// See [`add_bool`](Self::add_bool). + /// See [`add_bool`](Self::add_bool). Returns [`Error::Conversion`] if the value is longer + /// than `u32::MAX` bytes. pub fn add_str(&mut self, value: &str) -> Result<()> { self.add_bytes(value.as_bytes()) } @@ -1681,7 +1669,8 @@ impl InsertChunk { /// /// # Errors /// - /// See [`add_bool`](Self::add_bool). + /// See [`add_bool`](Self::add_bool). Returns [`Error::Conversion`] if the value is longer + /// than `u32::MAX` bytes. pub fn add_bytes(&mut self, value: &[u8]) -> Result<()> { if self.column_index >= self.column_count { return Err(Error::invalid_table_definition("Too many columns in row")); @@ -1826,7 +1815,8 @@ impl InsertChunk { /// /// # Errors /// - /// See [`add_bool`](Self::add_bool). + /// See [`add_bool`](Self::add_bool). Returns [`Error::Conversion`] if the value is longer + /// than `u32::MAX` bytes. pub fn add_geography(&mut self, value: &Geography) -> Result<()> { // Geography uses the same varbinary path as add_bytes self.add_bytes(value.as_bytes()) @@ -2064,8 +2054,9 @@ impl<'conn> ChunkSender<'conn> { /// /// - Returns [`Error::Internal`] with message `"ChunkSender mutex poisoned"` /// if a sender thread panicked while holding the writer lock. - /// - Returns [`Error::Server`] or [`Error::Io`] if sending the COPY - /// trailer or finishing the COPY operation fails. + /// - Returns [`Error::Server`] if the server rejects the COPY trailer or data. + /// - Returns [`Error::Connection`] on a transport I/O failure, or + /// [`Error::Closed`] if the server closed the connection. pub fn finish(self) -> Result { let mut writer_guard = self .writer @@ -2100,6 +2091,66 @@ impl<'conn> ChunkSender<'conn> { } } +/// Resolves the column's `NUMERIC(precision, scale)` and rescales `value` to +/// that scale, returning `(precision, unscaled value at the column's scale)`. +/// +/// Hyper stores a NUMERIC column as an unscaled integer at the column's scale, +/// so a value carrying a different scale must be converted or its magnitude is +/// silently corrupted. Scaling up is exact (overflow is an error); scaling down +/// is only allowed when no non-zero digit is dropped. +pub(crate) fn numeric_for_column( + table_def: &TableDefinition, + column_index: usize, + value: Numeric, +) -> Result<(u32, i128)> { + let column = table_def.columns.get(column_index); + let col_name = column.map_or("", |c| c.name.as_str()); + let (precision, col_scale) = column + .and_then(super::table_definition::ColumnDefinition::sql_type) + .and_then(|t| Some((t.precision()?, t.scale()?))) + .ok_or_else(|| { + Error::conversion(format!( + "Cannot determine numeric precision for column '{col_name}' at index {column_index}. \ + Ensure the column is defined with explicit SqlType including precision.\n\n\ + Example fix:\n \ + table_def.add_column_with_type(\"{col_name}\", SqlType::Numeric {{ precision: 10, scale: 2 }}, true);" + )) + })?; + let unscaled = rescale_numeric(value, col_scale).map_err(|reason| { + Error::conversion(format!( + "Numeric value {value} cannot be stored in column '{col_name}' with scale {col_scale}: {reason}" + )) + })?; + Ok((precision, unscaled)) +} + +/// Converts `value` to an unscaled integer at `target_scale`. +fn rescale_numeric(value: Numeric, target_scale: u32) -> std::result::Result { + let value_scale = u32::from(value.scale()); + let unscaled = value.unscaled_value(); + match value_scale.cmp(&target_scale) { + std::cmp::Ordering::Equal => Ok(unscaled), + std::cmp::Ordering::Less => { + let factor = 10_i128 + .checked_pow(target_scale - value_scale) + .ok_or("the scale difference overflows 128-bit storage")?; + unscaled + .checked_mul(factor) + .ok_or("the value overflows 128-bit storage at the column's scale") + } + std::cmp::Ordering::Greater => { + let factor = 10_i128 + .checked_pow(value_scale - target_scale) + .ok_or("the value has more decimal digits than the column keeps")?; + if unscaled % factor == 0 { + Ok(unscaled / factor) + } else { + Err("the value has more decimal digits than the column keeps") + } + } + } +} + #[cfg(test)] mod tests { use crate::table_definition::TableDefinition; diff --git a/hyperdb-api/src/params.rs b/hyperdb-api/src/params.rs index e4c41afc..ddb4eb93 100644 --- a/hyperdb-api/src/params.rs +++ b/hyperdb-api/src/params.rs @@ -393,9 +393,10 @@ impl ToSqlParam for Option { impl ToSqlParam for Date { fn encode_param(&self) -> Option> { - // Date is stored as i32 Julian day offset from 2000-01-01. - // Big-endian per the PG Bind protocol (format code 1). - Some(self.to_julian_day().to_be_bytes().to_vec()) + // Sent as i32 days since 2000-01-01 (the PostgreSQL date epoch), big-endian + // per the PG Bind protocol (format code 1). The server adds the Julian + // epoch itself, so the absolute Julian day number must not be sent here. + Some(self.days().to_be_bytes().to_vec()) } fn sql_oid(&self) -> Oid { @@ -424,8 +425,9 @@ impl ToSqlParam for Time { impl ToSqlParam for Timestamp { fn encode_param(&self) -> Option> { - // Timestamp is stored as i64 microseconds since 2000-01-01. - Some(self.to_microseconds().to_be_bytes().to_vec()) + // Sent as i64 microseconds since 2000-01-01 (the PostgreSQL timestamp + // epoch); the server applies the Julian offset itself. + Some(self.microseconds().to_be_bytes().to_vec()) } fn sql_oid(&self) -> Oid { @@ -439,8 +441,8 @@ impl ToSqlParam for Timestamp { impl ToSqlParam for OffsetTimestamp { fn encode_param(&self) -> Option> { - // OffsetTimestamp is stored as i64 microseconds UTC since 2000-01-01. - Some(self.to_microseconds_utc().to_be_bytes().to_vec()) + // Sent as i64 UTC microseconds since 2000-01-01 (the PostgreSQL epoch). + Some(self.timestamp().microseconds().to_be_bytes().to_vec()) } fn sql_oid(&self) -> Oid { diff --git a/hyperdb-api/tests/param_types_tests.rs b/hyperdb-api/tests/param_types_tests.rs index 7ff5ed4e..57f42844 100644 --- a/hyperdb-api/tests/param_types_tests.rs +++ b/hyperdb-api/tests/param_types_tests.rs @@ -50,8 +50,7 @@ fn test_interval_param() { let interval = Interval::new(2, 5, 0); // 2 months, 5 days, 0 microseconds // CAST the bound interval param to text so we can assert the VALUE, not // just non-null — this proves the [us BE][days BE][months BE] encoding - // was interpreted correctly (Interval doesn't yet implement RowValue, so - // we can't read it back as a typed Interval). + // was interpreted correctly. let result = test .connection .query_params( @@ -640,3 +639,41 @@ fn test_untyped_prepare_rejects_parameters() { "expected 42601 unexpected-parameter error, got: {err}" ); } + +/// Date, Timestamp and OffsetTimestamp parameters must round-trip: the +/// bound value must be read back by the server as the same instant. +/// Regression: they were encoded with the absolute Julian epoch instead of +/// the PostgreSQL 2000-01-01 epoch, so the server stored them ~6,700 years off. +#[test] +fn test_date_time_params_round_trip() { + use hyperdb_api::{Date, OffsetTimestamp, Time, Timestamp}; + + let test = TestConnection::new().expect("Failed to create test connection"); + + let date = Date::new(2024, 1, 15); + let timestamp = Timestamp::new(Date::new(2024, 1, 15), Time::new(13, 45, 30, 123_456)); + let offset_ts = OffsetTimestamp::new(timestamp, 0); + + let result = test + .connection + .query_params( + "SELECT CAST($1 AS text), CAST($2 AS text), \ + $3 = TIMESTAMPTZ '2024-01-15 13:45:30.123456+00'", + &[ + &date as &dyn ToSqlParam, + ×tamp as &dyn ToSqlParam, + &offset_ts as &dyn ToSqlParam, + ], + ) + .expect("query_params failed"); + let rows = result.collect_rows().expect("collect_rows failed"); + assert_eq!(rows.len(), 1); + + let d: Option = rows[0].get(0); + let ts: Option = rows[0].get(1); + // TIMESTAMPTZ text renders in the session time zone, so compare the instant. + let same_instant: Option = rows[0].get(2); + assert_eq!(d.as_deref(), Some("2024-01-15")); + assert_eq!(ts.as_deref(), Some("2024-01-15 13:45:30.123456")); + assert_eq!(same_instant, Some(true), "timestamptz instant mismatch"); +} diff --git a/hyperdb-api/tests/roundtrip_tests.rs b/hyperdb-api/tests/roundtrip_tests.rs index 5b4b50f0..673f25fe 100644 --- a/hyperdb-api/tests/roundtrip_tests.rs +++ b/hyperdb-api/tests/roundtrip_tests.rs @@ -41,7 +41,6 @@ where /// Test SMALLINT round-trip. /// /// This test verifies that SMALLINT values round-trip correctly through insert/query. -/// The text/binary format detection uses protocol metadata from `RowDescription`. #[test] fn test_roundtrip_smallint() { let test = TestConnection::new().expect("Failed to create test connection"); @@ -342,10 +341,11 @@ fn test_roundtrip_varchar() { test.execute_command("CREATE TABLE rt_varchar (val VARCHAR(100))") .expect("Failed to create table"); + let max_length_value = "x".repeat(100); let test_values: Vec> = vec![ Some(""), Some("short"), - Some("exactly one hundred chars - padded to make it longer for testing purposes!!!!!"), + Some(max_length_value.as_str()), None, ]; @@ -409,7 +409,8 @@ fn test_roundtrip_date() { } inserter.execute().expect("Failed to execute"); - // Query and verify using raw string comparison (since we might not have Date parsing) + // Query and verify using raw string comparison, which checks the server's + // rendering independently of the client-side decoder let result = test .connection .execute_query("SELECT val::TEXT FROM rt_date") @@ -448,12 +449,13 @@ fn test_roundtrip_numeric_small() { .expect("Failed to get table definition"); let test_values: Vec> = vec![ - Some(Numeric::new(0, 2)), // 0.00 - Some(Numeric::new(100, 2)), // 1.00 - Some(Numeric::new(-100, 2)), // -1.00 - Some(Numeric::new(12345, 2)), // 123.45 - Some(Numeric::new(-12345, 2)), // -123.45 - Some(Numeric::new(99999999, 2)), // 999999.99 (near max for precision 10) + Some(Numeric::new(0, 2)), // 0.00 + Some(Numeric::new(100, 2)), // 1.00 + Some(Numeric::new(-100, 2)), // -1.00 + Some(Numeric::new(12345, 2)), // 123.45 + Some(Numeric::new(-12345, 2)), // -123.45 + Some(Numeric::new(99999999, 2)), // 999999.99 + Some(Numeric::new(9_999_999_999, 2)), // 99999999.99 (max for precision 10) None, ]; @@ -477,6 +479,7 @@ fn test_roundtrip_numeric_small() { Some("123.45".to_string()), Some("-123.45".to_string()), Some("999999.99".to_string()), + Some("99999999.99".to_string()), None, ]; @@ -488,6 +491,80 @@ fn test_roundtrip_numeric_small() { ); } +/// A `Numeric` whose scale differs from the column's must be rescaled to the +/// column's scale, not written as a raw unscaled value. +/// Regression: `add_numeric` ignored the value's scale, so `1234.5` inserted +/// into `NUMERIC(10, 2)` was stored as `12.35`/`123.45` (magnitude corruption). +#[test] +fn test_roundtrip_numeric_scale_mismatch() { + let test = TestConnection::new().expect("Failed to create test connection"); + + // One small (i64) and one big (128-bit) storage column. + test.execute_command( + "CREATE TABLE rt_numeric_rescale (small NUMERIC(10, 2), big NUMERIC(30, 4))", + ) + .expect("Failed to create table"); + let table_def = Catalog::new(&test.connection) + .get_table_definition("rt_numeric_rescale") + .expect("Failed to get table definition"); + + let rows: Vec<(Numeric, Numeric)> = vec![ + (Numeric::new(12345, 1), Numeric::new(12345, 1)), // 1234.5 -> scale up + (Numeric::new(5, 0), Numeric::new(5, 0)), // 5 -> scale up + (Numeric::new(12_300, 4), Numeric::new(1_234_500, 6)), // trailing zeros -> scale down, exact + (Numeric::new(-25, 1), Numeric::new(-25, 1)), // -2.5 + ]; + + let mut inserter = + Inserter::new(&test.connection, &table_def).expect("Failed to create inserter"); + for (small, big) in &rows { + inserter + .add_row(&[&Some(*small), &Some(*big)]) + .expect("Failed to add row"); + } + inserter.execute().expect("Failed to execute"); + + let result = test + .connection + .execute_query("SELECT small::TEXT, big::TEXT FROM rt_numeric_rescale") + .expect("Failed to query"); + let returned: Vec<(Option, Option)> = + collect_rows(result, |row| (row.get(0), row.get(1))); + let want = [ + ("1234.50", "1234.5000"), + ("5.00", "5.0000"), + ("1.23", "1.2345"), + ("-2.50", "-2.5000"), + ]; + for ((got_s, got_b), (want_s, want_b)) in returned.iter().zip(want) { + assert_eq!(got_s.as_deref(), Some(want_s)); + assert_eq!(got_b.as_deref(), Some(want_b)); + } + assert_eq!(returned.len(), want.len()); +} + +/// A `Numeric` that cannot be represented at the column's scale without +/// losing digits must be rejected, not silently truncated. +#[test] +fn test_roundtrip_numeric_scale_mismatch_lossy_rejected() { + let test = TestConnection::new().expect("Failed to create test connection"); + test.execute_command("CREATE TABLE rt_numeric_lossy (v NUMERIC(10, 2))") + .expect("Failed to create table"); + let table_def = Catalog::new(&test.connection) + .get_table_definition("rt_numeric_lossy") + .expect("Failed to get table definition"); + + let mut inserter = + Inserter::new(&test.connection, &table_def).expect("Failed to create inserter"); + let err = inserter + .add_row(&[&Some(Numeric::new(12_345, 3))]) // 12.345 has 3 decimals, column holds 2 + .expect_err("lossy rescale must be rejected"); + assert!( + err.to_string().contains("scale"), + "error should explain the scale problem, got: {err}" + ); +} + // ============================================================================= // Binary Data Round-Trip Tests // ============================================================================= @@ -495,8 +572,7 @@ fn test_roundtrip_numeric_small() { /// Test BYTEA round-trip. /// /// This test verifies that BYTEA values round-trip correctly through insert/query. -/// `PostgreSQL` text format uses hex escape format (`\xHEXDIGITS`) which is automatically -/// decoded by the client. +/// BYTEA is returned as raw bytes in `HyperBinary`; `get::>` yields them unchanged. #[test] fn test_roundtrip_bytea() { let test = TestConnection::new().expect("Failed to create test connection"); From 1ca0f749d4ce230c5a9a680e54fbf1d8fe9384c1 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 09:23:14 -0700 Subject: [PATCH 02/61] fix(mcp): stop panicking on non-ASCII text in previews and type inference value_preview truncated at a byte offset, infer_string_type sliced by byte index, and query_stats logged &sql[..80]; a multi-byte character at the cut point aborted the request. Truncate by character / on a char boundary and only treat ASCII strings as ISO 8601 candidates. --- hyperdb-api/src/query_stats.rs | 127 ++++++++++++++++++--------------- hyperdb-mcp/CHANGELOG.md | 5 ++ hyperdb-mcp/src/inspect.rs | 34 +++++++-- hyperdb-mcp/src/schema.rs | 68 ++++++++++++------ 4 files changed, 148 insertions(+), 86 deletions(-) diff --git a/hyperdb-api/src/query_stats.rs b/hyperdb-api/src/query_stats.rs index 30a54138..3924f35a 100644 --- a/hyperdb-api/src/query_stats.rs +++ b/hyperdb-api/src/query_stats.rs @@ -3,52 +3,9 @@ //! Query statistics collection for Hyper database queries. //! -//! This module provides a mechanism to capture detailed query performance metrics -//! from Hyper, including parsing time, compilation time, execution time, memory usage, -//! storage I/O, and plan cache status. -//! -//! # Architecture -//! -//! The stats collection is abstracted behind the [`QueryStatsProvider`] trait, allowing -//! the implementation to be swapped without changing the user-facing API. Currently, -//! [`LogFileStatsProvider`] parses Hyper's JSON log file (`hyperd.log`) to extract -//! per-query statistics. If Hyper adds native wire-protocol stats in the future, a new -//! provider can replace the log-based one transparently. -//! -//! # Usage -//! -//! ```no_run -//! use hyperdb_api::{Connection, CreateMode, HyperProcess, Result}; -//! use hyperdb_api::LogFileStatsProvider; -//! -//! fn main() -> Result<()> { -//! let hyper = HyperProcess::new(None, None)?; -//! let mut conn = Connection::new(&hyper, "test.hyper", CreateMode::CreateIfNotExists)?; -//! -//! // Enable stats collection (auto-detect log path from HyperProcess) -//! conn.enable_query_stats(LogFileStatsProvider::from_process(&hyper)); -//! -//! // Execute a query -//! conn.execute_command("CREATE TABLE t (id INT)")?; -//! -//! // Retrieve stats for the last query -//! if let Some(stats) = conn.last_query_stats() { -//! println!("Total elapsed: {}s", stats.elapsed_s); -//! if let Some(ref pre) = stats.pre_execution { -//! println!(" Parse: {:?}s, Compile: {:?}s", -//! pre.parsing_time_s, pre.compilation_time_s); -//! } -//! } -//! -//! Ok(()) -//! } -//! ``` -//! -//! # Availability -//! -//! - **Local `HyperProcess`**: Full stats available via log file parsing. -//! - **Remote standalone Hyper**: Not available with the log-based provider (no local log file). -//! When Hyper adds native stats support, remote connections will work via a new provider. +//! Statistics are collected through the [`QueryStatsProvider`] trait. +//! [`LogFileStatsProvider`], the built-in implementation, parses Hyper's JSON log file +//! (`hyperd.log`) to extract per-query statistics. See [`QueryStats`] for usage. use std::any::Any; use std::fmt; @@ -65,12 +22,49 @@ use tracing::{debug, trace}; /// Detailed statistics for a single query execution. /// -/// All time fields are in seconds. Memory fields are in megabytes. -/// Fields are `Option` because not all queries produce all stats (e.g., a simple -/// `SET` command won't have execution storage stats). +/// Covers parsing time, compilation time, execution time, memory usage, +/// storage I/O, and plan cache status. All time fields are in seconds. Memory +/// fields are in megabytes. Fields are `Option` because not all queries produce +/// all stats (e.g., a simple `SET` command won't have execution storage stats). +/// +/// # Example +/// +/// ```no_run +/// use hyperdb_api::{Connection, CreateMode, HyperProcess, Result}; +/// use hyperdb_api::LogFileStatsProvider; +/// +/// fn main() -> Result<()> { +/// let hyper = HyperProcess::new(None, None)?; +/// let mut conn = Connection::new(&hyper, "test.hyper", CreateMode::CreateIfNotExists)?; +/// +/// // Enable stats collection (auto-detect log path from HyperProcess) +/// conn.enable_query_stats(LogFileStatsProvider::from_process(&hyper)); +/// +/// // Execute a query +/// conn.execute_command("CREATE TABLE t (id INT)")?; +/// +/// // Retrieve stats for the last query +/// if let Some(stats) = conn.last_query_stats() { +/// println!("Total elapsed: {}s", stats.elapsed_s); +/// if let Some(ref pre) = stats.pre_execution { +/// println!(" Parse: {:?}s, Compile: {:?}s", +/// pre.parsing_time_s, pre.compilation_time_s); +/// } +/// } +/// +/// Ok(()) +/// } +/// ``` +/// +/// # Availability +/// +/// - **Local `HyperProcess`**: Full stats available via log file parsing. +/// - **Remote standalone Hyper**: Not available with [`LogFileStatsProvider`], which needs a +/// readable local log file. #[derive(Debug, Clone, Default)] pub struct QueryStats { - /// Total elapsed wall-clock time for the query (seconds). + /// Total elapsed wall-clock time for the query (seconds); `0.0` if the log + /// entry has no `elapsed` field. pub elapsed_s: f64, /// Time spent committing the transaction (seconds). pub commit_time_s: Option, @@ -142,10 +136,10 @@ pub struct ExecutionStats { /// Trait for collecting query statistics from a Hyper server. /// -/// Implementations capture stats using different mechanisms (log file parsing, -/// future native protocol support, etc.). The trait uses an opaque token pattern: -/// `before_query` is called before execution and returns a token (e.g., a file -/// offset), which is passed to `after_query` after execution to extract the stats. +/// Implementations capture stats from a source such as Hyper's log file. The +/// trait uses an opaque token pattern: `before_query` is called before +/// execution and returns a token (e.g., a file offset), which is passed to +/// `after_query` after execution to extract the stats. /// /// # Thread Safety /// @@ -223,7 +217,8 @@ impl LogFileStatsProvider { /// Creates a new provider by auto-detecting the log path from a [`HyperProcess`]. /// - /// The log file is expected at `/hyperd.log`. + /// The log file is expected at `/hyperd.log`. If the process has no + /// log directory, `hyperd.log` in the current directory is used. /// /// [`HyperProcess`]: crate::HyperProcess #[must_use] @@ -376,7 +371,7 @@ impl QueryStatsProvider for LogFileStatsProvider { debug!( target: "hyperdb_api", offset = token.offset, - sql_prefix = &sql[..sql.len().min(80)], + sql_prefix = log_prefix(sql), "query-stats-not-found" ); } @@ -385,6 +380,16 @@ impl QueryStatsProvider for LogFileStatsProvider { } } +/// Returns at most the first 80 bytes of `sql` for log output, cut on a +/// character boundary so multi-byte text cannot cause a slicing panic. +fn log_prefix(sql: &str) -> &str { + let mut end = sql.len().min(80); + while !sql.is_char_boundary(end) { + end -= 1; + } + &sql[..end] +} + // ============================================================================= // JSON Parsing Helpers // ============================================================================= @@ -621,6 +626,16 @@ fn parse_query_end(v: &Value) -> QueryStats { mod tests { use super::*; + #[test] + fn log_prefix_never_splits_a_multibyte_char() { + // 79 ASCII bytes followed by a 3-byte char straddling the 80-byte cut. + let sql = format!("{}€ tail", "a".repeat(79)); + let prefix = log_prefix(&sql); + assert!(prefix.len() <= 80); + assert_eq!(prefix, "a".repeat(79)); + assert_eq!(log_prefix("SELECT 1"), "SELECT 1"); + } + #[test] fn test_normalize_for_matching() { assert_eq!( diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 244856e0..a95a8511 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -24,6 +24,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- **`inspect_file` and JSON schema inference no longer panic on non-ASCII + text.** Sample-value previews were truncated at a byte offset and ISO 8601 + date/timestamp detection sliced strings by byte index, so a multi-byte + character at the cut point aborted the request. Previews now truncate by + character, and date/timestamp detection only considers ASCII strings. - **`query_data` / `query_file` no longer leak their scratch table on a failed query** — the temp table is now dropped whether the query succeeds or fails, so a bad query no longer leaves a `_tmp_*` table behind in `describe`. diff --git a/hyperdb-mcp/src/inspect.rs b/hyperdb-mcp/src/inspect.rs index b56ddafb..6710665d 100644 --- a/hyperdb-mcp/src/inspect.rs +++ b/hyperdb-mcp/src/inspect.rs @@ -3,7 +3,7 @@ //! Dry-run file inspection for the `inspect_file` MCP tool. //! -//! Given a path to a CSV, Parquet, or Arrow IPC file, [`inspect_source`] +//! Given a path to a CSV, JSON, JSONL, Parquet, or Arrow IPC file, [`inspect_source`] //! returns the schema we would use to create a table (using the exact same //! inference + full-file numeric widening as [`crate::ingest`]) along with //! per-column diagnostics that help an LLM pick a safer schema override @@ -253,12 +253,15 @@ fn value_preview(v: Option<&Value>) -> String { Some(Value::Bool(b)) => b.to_string(), Some(other) => other.to_string(), }; - if raw.len() > MAX_LEN { - let mut s = raw[..MAX_LEN].to_string(); - s.push('…'); - s - } else { - raw + // Truncate by characters: slicing at a byte offset panics when it lands + // inside a multi-byte code point. + match raw.char_indices().nth(MAX_LEN) { + Some((cut, _)) => { + let mut s = raw[..cut].to_string(); + s.push('…'); + s + } + None => raw, } } @@ -397,3 +400,20 @@ fn inspect_arrow_ipc(path: &str, file_size: u64) -> Result Option { // --- Tier 2: JSON Schema Inference --- -/// Intermediate type tag used during schema inference. The widening order is: -/// Null < Bool < Int < `BigInt` < Double < Date < Timestamp < Text. -/// When a column has mixed non-numeric types, it collapses to Text. +/// Intermediate type tag used during schema inference. Mixed Int/`BigInt`/Double +/// widen to the widest seen; any other mix resolves to Text. #[derive(Debug, Clone, PartialEq)] enum InferredType { Null, @@ -271,14 +264,14 @@ fn infer_json_value_type(val: &Value) -> InferredType { /// /// # Safety of the string slices /// -/// Each indexing like `s[0..4]` requires `s.len()` to be at least the -/// upper bound (otherwise it panics). The `s.len() == 10` / `s.len() >= -/// 19` guards are the leftmost clauses in each `if` so Rust's -/// short-circuit `&&` evaluation proves the length invariant before the -/// slice operations run. +/// Both patterns are pure ASCII, so the leftmost clause of each `if` is +/// `s.is_ascii()`. That makes every byte index a char boundary, so the +/// `s[a..b]` slices below cannot panic on multi-byte input; the length +/// checks that follow it guarantee the upper bounds are in range. fn infer_string_type(s: &str) -> InferredType { // Try ISO 8601 date: YYYY-MM-DD - if s.len() == 10 + if s.is_ascii() + && s.len() == 10 && s.chars().nth(4) == Some('-') && s.chars().nth(7) == Some('-') && s[0..4].parse::().is_ok() @@ -288,7 +281,8 @@ fn infer_string_type(s: &str) -> InferredType { return InferredType::Date; } // Try ISO 8601 timestamp: YYYY-MM-DDThh:mm:ss - if s.len() >= 19 + if s.is_ascii() + && s.len() >= 19 && s.chars().nth(10) == Some('T') && s[0..10].contains('-') && s[11..].contains(':') @@ -658,10 +652,8 @@ pub fn parse_schema_override( /// practice some MCP clients forward this field as a **JSON-encoded string** /// rather than a raw JSON object — e.g. Windsurf/Cascade serializes /// `{"postal_code": "TEXT"}` as `"\"{\\\"postal_code\\\": \\\"TEXT\\\"}\""`. -/// If we only accepted `Value::Object` (the old `v.as_object().cloned()` -/// pattern) the override was silently dropped and ingest would fail with a -/// confusing `22P02 invalid input syntax` error from hyperd when a column that -/// the user explicitly wanted TEXT stayed INT. +/// Both a JSON object and a JSON-encoded string containing an object are +/// accepted, so the override is honored whichever shape the client forwards. /// /// Accepted shapes: /// @@ -797,3 +789,33 @@ pub fn apply_schema_override( } Ok(inferred) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn infer_string_type_recognizes_iso_dates_and_timestamps() { + assert_eq!(infer_string_type("2024-01-15"), InferredType::Date); + assert_eq!( + infer_string_type("2024-01-15T13:45:30"), + InferredType::Timestamp + ); + assert_eq!(infer_string_type("hello"), InferredType::Text); + } + + #[test] + fn infer_string_type_does_not_panic_on_non_ascii() { + // 10 bytes, '-' at char 4 and 7, but `s[0..4]` lands inside 'é'. + assert_eq!(infer_string_type("abcé-xy-z"), InferredType::Text); + // `s[0..10]` lands inside the 'é' that precedes the 'T' (char 10). + assert_eq!(infer_string_type("abcdefghiéT12345678"), InferredType::Text); + // Multibyte chars before the date/timestamp separators are not ISO 8601. + assert_eq!(infer_string_type("é024-01-15T13:45:30"), InferredType::Text); + assert_eq!(infer_string_type("2024-01-15T13:45:3é"), InferredType::Text); + assert_eq!( + infer_string_type("日本語日本語日本語日本語"), + InferredType::Text + ); + } +} From 86cec917de99e71803ac35e55a0cc36b307d30cd Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 09:23:14 -0700 Subject: [PATCH 03/61] ci: require the real check names in the release gates release.yml and npm-build-publish.yml required a check named `clippy` that no longer exists. They now require all 13 current check names to be present and completed. Also corrects stale comments in ci.yml, release-please.yml, the Makefile, .cargo/config.toml, deny.toml and .cargo/audit.toml, fixes the platform list in scripts/assemble-npm.sh, and removes the dead nested hyperdb-api-node CI workflow. --- .cargo/audit.toml | 2 +- .cargo/config.toml | 2 +- .github/workflows/ci.yml | 6 +- .github/workflows/npm-build-publish.yml | 28 ++- .github/workflows/release-please.yml | 6 +- .github/workflows/release.yml | 29 ++-- Makefile | 15 +- deny.toml | 18 +- hyperdb-api-node/.github/workflows/CI.yml | 197 ---------------------- scripts/assemble-npm.sh | 2 +- 10 files changed, 66 insertions(+), 239 deletions(-) delete mode 100644 hyperdb-api-node/.github/workflows/CI.yml diff --git a/.cargo/audit.toml b/.cargo/audit.toml index b8264cdd..772d5883 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -6,7 +6,7 @@ [advisories] ignore = [ # `rsa` Marvin Attack — no constant-time fix yet. Used only for outbound - # JWT signing in `hyperapi-salesforce`, where timing sidechannels are not + # JWT signing in `hyperdb-api-salesforce`, where timing sidechannels are not # observable to a remote attacker. Threat model does not apply. "RUSTSEC-2023-0071", # `ttf-parser` unmaintained (RUSTSEC-2026-0192, informational). Transitive via diff --git a/.cargo/config.toml b/.cargo/config.toml index 525adc30..58812b30 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -7,7 +7,7 @@ download-hyperd = "run --release --package hyperdb-bootstrap --bin hyperdb-boots # Use mold on Linux to avoid lld OOM-crashing on large debug link jobs # (CI ubuntu-latest runners have ~7 GB RAM; linking the full workspace in # debug mode with lld exhausts it and triggers a Bus error / signal 7). -# mold is pre-installed on ubuntu-latest since actions/runner-images 20240310. +# CI installs mold via apt in every Linux job that builds code. # No effect on macOS or Windows (target cfg guards the override). [target.x86_64-unknown-linux-gnu] linker = "clang" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa3aa46d..a73db7c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,9 +252,9 @@ jobs: cargo test --workspace \ --exclude hyperdb-api-node \ --exclude hyperdb-bootstrap - # hyperdb-api-node needs napi-rs + a Node.js toolchain; it gets - # its own workflow when wired up. hyperdb-bootstrap has its own - # coverage (next step) and doesn't need hyperd running. + # hyperdb-api-node needs napi-rs + a Node.js toolchain; the + # node-bindings job builds and smoke-tests it. hyperdb-bootstrap has + # its own coverage (next step) and doesn't need hyperd running. - name: hyperdb-compile-check tests # Declares its own [workspace], so the run above cannot see it. Nothing diff --git a/.github/workflows/npm-build-publish.yml b/.github/workflows/npm-build-publish.yml index fff0240a..49959204 100644 --- a/.github/workflows/npm-build-publish.yml +++ b/.github/workflows/npm-build-publish.yml @@ -1,8 +1,9 @@ name: npm-build-publish # Builds hyperdb-mcp and hyperdb-api-node npm packages with bundled -# hyperd binaries. Triggered after a successful release workflow creates -# a GitHub release, or manually via workflow_dispatch. +# hyperd binaries. Triggered by the hand-created GitHub Release +# (`release: published`, in parallel with release.yml), or manually via +# workflow_dispatch. on: workflow_dispatch: @@ -58,17 +59,23 @@ jobs: # them through bash → jq → Oniguruma was fragile and broke # silently on the v0.2.2 release. Exact-string match avoids # the whole class of regex-escaping bugs. + # These names must match the `name:` of the ci.yml jobs; the + # msrv entry embeds the MSRV, so it changes whenever ci.yml does. + # Every name must be present: a renamed or missing job makes the + # gate time out instead of silently lowering the count. REQUIRED_JSON='[ "rustfmt","clippy (ubuntu-24.04)","clippy (windows-2025)", - "cargo-audit","cargo-deny", + "msrv (1.88)","doc","cargo-audit","cargo-deny", "version consistency","publish dry-run", "test (ubuntu-24.04)","test (macos-26)","test (windows-2025)", "hyperdb-api-node (build + smoke)" ]' + EXPECTED=$(echo "$REQUIRED_JSON" | jq 'length') for i in $(seq 1 "$MAX_ATTEMPTS"); do RUNS=$(gh api "repos/$REPO/commits/$SHA/check-runs?per_page=100" \ --jq "[.check_runs[] | select(.name as \$n | $REQUIRED_JSON | index(\$n))]") TOTAL=$(echo "$RUNS" | jq 'length') + PRESENT=$(echo "$RUNS" | jq '[.[].name] | unique | length') COMPLETED=$(echo "$RUNS" | jq '[.[] | select(.status=="completed")] | length') FAILED=$(echo "$RUNS" | jq '[.[] | select(.status=="completed" and (.conclusion!="success" and .conclusion!="skipped" and .conclusion!="neutral"))] | length') echo "Attempt $i/$MAX_ATTEMPTS: total=$TOTAL completed=$COMPLETED failed=$FAILED" @@ -77,7 +84,7 @@ jobs: echo "$RUNS" | jq -r '.[] | select(.status=="completed" and (.conclusion!="success" and .conclusion!="skipped" and .conclusion!="neutral")) | " - \(.name): \(.conclusion)"' exit 1 fi - if [[ "$TOTAL" -gt 0 && "$COMPLETED" == "$TOTAL" ]]; then + if [[ "$PRESENT" == "$EXPECTED" && "$COMPLETED" == "$TOTAL" ]]; then echo "CI passed ($TOTAL required check-runs completed successfully)." exit 0 fi @@ -331,6 +338,19 @@ jobs: - name: Copy README into hyperdb-mcp npm publish directory run: cp hyperdb-mcp/README.md hyperdb-mcp/npm/README.md + - name: Copy license files into every published npm package directory + # npm auto-includes only files named LICENSE / LICENCE, so the + # hyphenated names are listed in each package.json `files` array and + # copied in here, the same way as the README above. The platform + # list must match PLATFORMS in the version-injection step below. + run: | + set -euo pipefail + for d in hyperdb-mcp/npm hyperdb-api-node \ + hyperdb-mcp/npm/{darwin-arm64,linux-x64-gnu,win32-x64-msvc} \ + hyperdb-api-node/npm/{darwin-arm64,linux-x64-gnu,win32-x64-msvc}; do + cp LICENSE-MIT.txt LICENSE-APACHE.txt NOTICE "$d/" + done + - name: Resolve version and dist-tag from event id: version # Source-of-truth: the git tag that triggered the workflow. diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 1b3ed358..7096b2b1 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -11,9 +11,9 @@ name: release-please # anchor on the freshly created tag and correct the next release PR (see #308). # # Uses a fine-grained PAT (RELEASE_PLEASE_TOKEN) instead of the default -# GITHUB_TOKEN so that PRs and tags created by this workflow trigger -# downstream CI and publish workflows. With GITHUB_TOKEN, GitHub -# suppresses workflow triggers on events it creates (anti-recursion). +# GITHUB_TOKEN so that PRs created by this workflow trigger downstream +# CI. With GITHUB_TOKEN, GitHub suppresses workflow triggers on events +# it creates (anti-recursion). # # After release-please opens or updates the release PR, a follow-up # step reconciles Cargo.lock against the bumped Cargo.toml and pushes diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 37f08564..9df9f857 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,11 +10,9 @@ name: release # - a manual workflow_dispatch with an explicit existing tag (for # re-runs or emergency releases). # -# Note: the `push: tags:` trigger was removed because release-please -# now uses a PAT (RELEASE_PLEASE_TOKEN) that fires both `release: -# published` AND `push: tags:` events. Keeping both caused duplicate -# runs. For emergency releases that bypass release-please, use -# workflow_dispatch. +# There is no `push: tags:` trigger: pushing a tag alone publishes +# nothing. Use workflow_dispatch for re-runs or for emergency releases +# that bypass release-please. # # Stable tags (v0.1.0) mark the GitHub release as latest; pre-release # tags (v0.1.0-rc.1) mark it as prerelease so it doesn't show as @@ -60,16 +58,18 @@ jobs: ref: ${{ github.event.inputs.tag || github.event.release.tag_name || github.ref }} - name: Wait for CI to pass - # The full test matrix already ran on the merge commit before - # release-please tagged it. Re-running it here was ~20 minutes - # of redundant CI per release. Instead, poll the check-runs + # The full test matrix already ran on the release-PR merge + # commit, and re-running it here would cost ~20 minutes of + # redundant CI per release. Instead, poll the check-runs # API for the tag's SHA and verify all required checks are # green. Mirrors the verify-ci pattern in npm-build-publish.yml. # # Risk: a maintainer who runs workflow_dispatch with a tag on # a SHA that never had CI would skip all test coverage. Guard - # against that by requiring TOTAL > 0 — if no required check - # ever ran on this SHA, abort. + # against that by requiring every name in REQUIRED_JSON to be + # present and completed — if a required check never ran on + # this SHA (or was renamed), the gate times out instead of + # passing. env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} @@ -85,16 +85,21 @@ jobs: # them through bash → jq → Oniguruma was fragile and broke # silently on the v0.2.2 release. Exact-string match avoids # the whole class of regex-escaping bugs. + # These names must match the `name:` of the ci.yml jobs; the + # msrv entry embeds the MSRV, so it changes whenever ci.yml does. REQUIRED_JSON='[ - "rustfmt","clippy","cargo-audit","cargo-deny", + "rustfmt","clippy (ubuntu-24.04)","clippy (windows-2025)", + "msrv (1.88)","doc","cargo-audit","cargo-deny", "version consistency","publish dry-run", "test (ubuntu-24.04)","test (macos-26)","test (windows-2025)", "hyperdb-api-node (build + smoke)" ]' + EXPECTED=$(echo "$REQUIRED_JSON" | jq 'length') for i in $(seq 1 "$MAX_ATTEMPTS"); do RUNS=$(gh api "repos/$REPO/commits/$SHA/check-runs?per_page=100" \ --jq "[.check_runs[] | select(.name as \$n | $REQUIRED_JSON | index(\$n))]") TOTAL=$(echo "$RUNS" | jq 'length') + PRESENT=$(echo "$RUNS" | jq '[.[].name] | unique | length') COMPLETED=$(echo "$RUNS" | jq '[.[] | select(.status=="completed")] | length') FAILED=$(echo "$RUNS" | jq '[.[] | select(.status=="completed" and (.conclusion!="success" and .conclusion!="skipped" and .conclusion!="neutral"))] | length') echo "Attempt $i/$MAX_ATTEMPTS: total=$TOTAL completed=$COMPLETED failed=$FAILED" @@ -103,7 +108,7 @@ jobs: echo "$RUNS" | jq -r '.[] | select(.status=="completed" and (.conclusion!="success" and .conclusion!="skipped" and .conclusion!="neutral")) | " - \(.name): \(.conclusion)"' exit 1 fi - if [[ "$TOTAL" -gt 0 && "$COMPLETED" == "$TOTAL" ]]; then + if [[ "$PRESENT" == "$EXPECTED" && "$COMPLETED" == "$TOTAL" ]]; then echo "CI passed ($TOTAL required check-runs completed successfully)." exit 0 fi diff --git a/Makefile b/Makefile index a6f3e954..7e74c04f 100644 --- a/Makefile +++ b/Makefile @@ -99,15 +99,14 @@ build-api-release: cargo build --release -p hyperdb-api-core -p hyperdb-api # Run tests (debug) with proper environment -# Mirrors the CI `test` job so a local pass means a CI pass. It previously -# covered only 3 of the 8 crates, which made its pass count look like whole- -# workspace coverage when it was a subset -- use `test-api` for the narrow run. +# Mirrors the CI `test` job so a local pass means a CI pass; use `test-api` +# for the narrow API-only run. # # Excludes match CI: hyperdb-api-node needs napi-rs plus a Node toolchain and -# has its own workflow, and hyperdb-bootstrap runs separately below because it -# does not need hyperd. hyperdb-compile-check declares its own [workspace], so -# `--workspace` cannot see it -- and until this target included it, nothing in -# CI or the Makefile ran that published crate's tests at all. +# is built and smoke-tested by the node-bindings job, and hyperdb-bootstrap +# runs separately below because it does not need hyperd. hyperdb-compile-check +# declares its own [workspace], so `--workspace` cannot see it and the last +# command below runs its tests explicitly. test: @echo "Environment:" @echo " HYPERD_PATH=$(HYPERD_PATH)" @@ -147,7 +146,7 @@ download-hyperd: cargo run --release -p hyperdb-bootstrap --bin hyperdb-bootstrap -- download $(ARGS) # Network-only check: probe each supported platform's wheel URL for the pinned -# release and cross-check its digest against PyPI. Intended for CI (nightly + +# release and cross-check its digest against PyPI. Intended for CI (weekly + # on PRs touching hyperd-version.toml). verify-hyperd-pin: cargo run --release -p hyperdb-bootstrap --bin hyperdb-bootstrap -- verify $(ARGS) diff --git a/deny.toml b/deny.toml index b2ff4b63..c2635aa2 100644 --- a/deny.toml +++ b/deny.toml @@ -5,7 +5,8 @@ # mirrors the advisory ignore list. [graph] -# Check the dependency graph for every workspace crate's default features. +# A local `cargo deny check` follows each workspace crate's default features; +# CI runs `--all-features --workspace`, which overrides these two settings. all-features = false no-default-features = false @@ -54,7 +55,7 @@ yanked = "warn" # short — each entry needs a comment explaining the mitigation. ignore = [ # `rsa` Marvin Attack: no patch available. The `rsa` crate is used by - # `jsonwebtoken` inside `hyperapi-salesforce` for signing client JWTs for + # `jsonwebtoken` inside `hyperdb-api-salesforce` for signing client JWTs for # Salesforce OAuth — this is *outbound* signing where timing sidechannels # are not observable to a remote attacker, so the Marvin Attack threat # model does not apply. Re-evaluate when `rsa` publishes the constant-time @@ -78,13 +79,12 @@ ignore = [ # Multiple versions of the same crate bloat binaries and increase audit # surface. Start permissive (warn) — we'll tighten as the workspace stabilizes. multiple-versions = "warn" -# Downgraded to `warn` until the publishing story is settled for internal -# helper crates (`hyper-types`, `hyper-protocol`, `hyper-client`). The -# published crates (`hyperapi`, `hyperapi-salesforce`, `sea-query-hyper`) -# currently depend on them via `path` without a `version` field — this is -# fine for local development but `cargo publish` will reject it until the -# helpers are either inlined into `hyperapi` or published themselves. Track -# separately; the Microsoft Rust Guidelines adoption does not block on it. +# `wildcards` stays at `warn` because some workspace dependencies are +# path-only on purpose: sibling dev-dependencies carry no version, since a +# versioned one would create a circular crates.io publish order (for example +# hyperdb-api and hyperdb-api-derive dev-depend on each other), and the +# unpublished hyperdb-api-node (`publish = false`) depends on hyperdb-api by +# path. `allow-wildcard-paths` keeps those entries from being flagged. wildcards = "warn" allow-wildcard-paths = true # Explicit allow / deny lists. Empty by default; add entries as needed with diff --git a/hyperdb-api-node/.github/workflows/CI.yml b/hyperdb-api-node/.github/workflows/CI.yml deleted file mode 100644 index 379bcb58..00000000 --- a/hyperdb-api-node/.github/workflows/CI.yml +++ /dev/null @@ -1,197 +0,0 @@ -name: CI - -env: - DEBUG: napi:* - APP_NAME: hyperdb-api-node - MACOSX_DEPLOYMENT_TARGET: '10.13' - -permissions: - contents: write - id-token: write - -on: - push: - branches: [main, hyperapi-node-dev] - tags-ignore: ['**'] - paths: ['hyperdb-api-node/**', '.github/workflows/CI.yml'] - pull_request: - paths: ['hyperdb-api-node/**'] - -jobs: - build: - strategy: - fail-fast: false - matrix: - settings: - - host: macos-latest - target: aarch64-apple-darwin - build: cargo build -p hyperdb-api-node --release --target aarch64-apple-darwin - - - host: macos-13 - target: x86_64-apple-darwin - build: cargo build -p hyperdb-api-node --release --target x86_64-apple-darwin - - - host: ubuntu-latest - target: x86_64-unknown-linux-gnu - build: cargo build -p hyperdb-api-node --release --target x86_64-unknown-linux-gnu - - - host: ubuntu-latest - target: x86_64-unknown-linux-musl - build: >- - apt-get update && apt-get install -y musl-tools && - cargo build -p hyperdb-api-node --release --target x86_64-unknown-linux-musl - - - host: ubuntu-latest - target: aarch64-unknown-linux-gnu - build: >- - sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu && - cargo build -p hyperdb-api-node --release --target aarch64-unknown-linux-gnu - env: - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - - - host: windows-latest - target: x86_64-pc-windows-msvc - build: cargo build -p hyperdb-api-node --release --target x86_64-pc-windows-msvc - - name: Build - ${{ matrix.settings.target }} - runs-on: ${{ matrix.settings.host }} - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.settings.target }} - - - name: Cache cargo registry - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ matrix.settings.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install npm dependencies - working-directory: hyperdb-api-node - run: npm install - - - name: Build native module - working-directory: . - run: ${{ matrix.settings.build }} - env: ${{ matrix.settings.env || '{}' }} - - - name: Copy artifact to platform package - shell: bash - run: | - node hyperdb-api-node/scripts/ci-copy-artifact.js ${{ matrix.settings.target }} - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: bindings-${{ matrix.settings.target }} - path: hyperdb-api-node/npm/*/hyperdb-api-node.*.node - if-no-files-found: error - - test: - name: Test - ${{ matrix.settings.host }} - needs: build - strategy: - fail-fast: false - matrix: - settings: - - host: macos-latest - target: aarch64-apple-darwin - - host: ubuntu-latest - target: x86_64-unknown-linux-gnu - - host: windows-latest - target: x86_64-pc-windows-msvc - runs-on: ${{ matrix.settings.host }} - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install npm dependencies - working-directory: hyperdb-api-node - run: npm install - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: bindings-${{ matrix.settings.target }} - path: hyperdb-api-node - - - name: Move .node file to root - shell: bash - working-directory: hyperdb-api-node - run: | - find npm -name '*.node' -exec cp {} . \; - ls -la *.node - - - name: Verify module loads - working-directory: hyperdb-api-node - run: node -e "const m = require('./index.js'); console.log('Exports:', Object.keys(m))" - - publish: - name: Publish - runs-on: ubuntu-latest - needs: test - if: startsWith(github.ref, 'refs/tags/v') - steps: - - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - registry-url: https://registry.npmjs.org - - - name: Install npm dependencies - working-directory: hyperdb-api-node - run: npm install - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: hyperdb-api-node/artifacts - - - name: Move artifacts to npm packages - working-directory: hyperdb-api-node - run: npx napi artifacts - - - name: List packages - working-directory: hyperdb-api-node - run: | - ls -laR npm/ - echo "---" - cat package.json - - - name: Publish platform packages - working-directory: hyperdb-api-node - run: | - for dir in npm/*/; do - if [ -f "$dir/package.json" ]; then - echo "Publishing $dir..." - cd "$dir" - npm publish --access public || true - cd - - fi - done - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Publish root package - working-directory: hyperdb-api-node - run: npm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/scripts/assemble-npm.sh b/scripts/assemble-npm.sh index 32dbebb6..18745f0b 100755 --- a/scripts/assemble-npm.sh +++ b/scripts/assemble-npm.sh @@ -161,7 +161,7 @@ if [[ -f "${NODE_LIB}" ]]; then ( cd "${ROOT}/hyperdb-api-node" npm pkg set "version=${VERSION}" - for p in darwin-arm64 linux-arm64-gnu linux-x64-gnu linux-x64-musl win32-x64-msvc; do + for p in darwin-arm64 linux-x64-gnu win32-x64-msvc; do npm pkg set "optionalDependencies.hyperdb-api-node-$p=${VERSION}" done npm pack --quiet From ad6fcd9ee7ec9f476d8760c5cd9ba684f17ae80c Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 09:23:14 -0700 Subject: [PATCH 04/61] build: ship license text with every crate and npm package No crate tarball or npm package carried the MIT/Apache-2.0 license text. Add LICENSE-MIT.txt and LICENSE-APACHE.txt to the eight crates (plus NOTICE in hyperdb-api-core for the code adapted from rust-postgres) and list them in the npm package.json `files` arrays. --- hyperdb-api-core/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-api-core/LICENSE-MIT.txt | 21 ++ hyperdb-api-core/NOTICE | 88 ++++++++ hyperdb-api-derive/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-api-derive/LICENSE-MIT.txt | 21 ++ .../npm/darwin-arm64/package.json | 5 +- hyperdb-api-node/npm/darwin-x64/package.json | 5 +- .../npm/linux-x64-gnu/package.json | 5 +- .../npm/win32-x64-msvc/package.json | 5 +- hyperdb-api-node/package.json | 5 +- hyperdb-api-salesforce/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-api-salesforce/LICENSE-MIT.txt | 21 ++ hyperdb-api/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-api/LICENSE-MIT.txt | 21 ++ hyperdb-bootstrap/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-bootstrap/LICENSE-MIT.txt | 21 ++ hyperdb-compile-check/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-compile-check/LICENSE-MIT.txt | 21 ++ hyperdb-mcp/LICENSE-APACHE.txt | 207 ++++++++++++++++++ hyperdb-mcp/LICENSE-MIT.txt | 21 ++ hyperdb-mcp/npm/darwin-arm64/package.json | 5 +- hyperdb-mcp/npm/darwin-x64/package.json | 5 +- hyperdb-mcp/npm/linux-x64-gnu/package.json | 5 +- hyperdb-mcp/npm/package.json | 5 +- hyperdb-mcp/npm/win32-x64-msvc/package.json | 5 +- sea-query-hyperdb/LICENSE-APACHE.txt | 207 ++++++++++++++++++ sea-query-hyperdb/LICENSE-MIT.txt | 21 ++ 27 files changed, 1952 insertions(+), 10 deletions(-) create mode 100644 hyperdb-api-core/LICENSE-APACHE.txt create mode 100644 hyperdb-api-core/LICENSE-MIT.txt create mode 100644 hyperdb-api-core/NOTICE create mode 100644 hyperdb-api-derive/LICENSE-APACHE.txt create mode 100644 hyperdb-api-derive/LICENSE-MIT.txt create mode 100644 hyperdb-api-salesforce/LICENSE-APACHE.txt create mode 100644 hyperdb-api-salesforce/LICENSE-MIT.txt create mode 100644 hyperdb-api/LICENSE-APACHE.txt create mode 100644 hyperdb-api/LICENSE-MIT.txt create mode 100644 hyperdb-bootstrap/LICENSE-APACHE.txt create mode 100644 hyperdb-bootstrap/LICENSE-MIT.txt create mode 100644 hyperdb-compile-check/LICENSE-APACHE.txt create mode 100644 hyperdb-compile-check/LICENSE-MIT.txt create mode 100644 hyperdb-mcp/LICENSE-APACHE.txt create mode 100644 hyperdb-mcp/LICENSE-MIT.txt create mode 100644 sea-query-hyperdb/LICENSE-APACHE.txt create mode 100644 sea-query-hyperdb/LICENSE-MIT.txt diff --git a/hyperdb-api-core/LICENSE-APACHE.txt b/hyperdb-api-core/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-api-core/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-api-core/LICENSE-MIT.txt b/hyperdb-api-core/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-api-core/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-api-core/NOTICE b/hyperdb-api-core/NOTICE new file mode 100644 index 00000000..389a89b2 --- /dev/null +++ b/hyperdb-api-core/NOTICE @@ -0,0 +1,88 @@ +================================================================================ +Third-Party Attributions for hyperdb-api-rust +================================================================================ + +This NOTICE file lists third-party software whose code, copyright notices, or +license texts are required to be reproduced when redistributing this project. +This project's own copyright and license terms live in LICENSE-MIT.txt and +LICENSE-APACHE.txt at the repo root. + +Each section below names the upstream project, the files in this repository +where adapted code lives, and reproduces the upstream copyright and license +notice as required by the upstream license terms. + +-------------------------------------------------------------------------------- +sfackler/rust-postgres (postgres-protocol, tokio-postgres, postgres-types) +-------------------------------------------------------------------------------- + +Source: https://github.com/sfackler/rust-postgres +License: MIT OR Apache-2.0 (this project includes the MIT terms below) + +Portions of the following files in this repository were adapted from +sfackler/rust-postgres. Per-file source-level credits with the specific +patterns adapted are in each file's module-level doc comment. + + - hyperdb-api-core/src/client/auth.rs + Adapted from postgres-protocol/src/authentication/sasl.rs: + SCRAM-SHA-256 client implementation. Hyper-specific additions on top. + + - hyperdb-api-core/src/protocol/message/backend.rs + Adapted from postgres-protocol/src/message/backend.rs: message-tag + constants, Header struct, and framing logic. Hyper-specific + additions on top. + + - hyperdb-api-core/src/protocol/message/frontend.rs + Adapted from postgres-protocol/src/message/frontend.rs: + startup-message framing and message-length helpers. Hyper-specific + additions on top. + + - hyperdb-api-core/src/types/oid.rs + Adapted from postgres-types: the Oid newtype and constants-module + pattern. Hyper-specific OIDs added on top. + + - hyperdb-api-core/src/client/row.rs + Adapted from tokio-postgres: the Row / StreamRow / BatchRow type + decomposition and pre-computed-offset random-access pattern. + Hyper-specific additions on top. + +The MIT License (MIT) + +Copyright (c) 2016 Steven Fackler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ +Other Inspirational Sources +================================================================================ + +The following crates informed API patterns in this repository. No code, +comments, or fingerprintable variable names from these projects appear in +this repository; the influence is at the level of public-API shape and idiom +choices. They are listed here for developer-facing transparency. + + - SQLx https://github.com/launchbadge/sqlx (MIT or Apache-2.0) + Inspired the fetch_one() / fetch_optional() / fetch_all() / + fetch_scalar() convenience-method shape on Connection and Transaction. + + - Diesel https://diesel.rs/ (MIT or Apache-2.0) + Informed builder-pattern ergonomics in TableDefinition and the + ConnectionBuilder family. + + - SeaORM https://www.sea-ql.org/SeaORM/ (MIT or Apache-2.0) + Informed modern Rust async patterns for database operations. diff --git a/hyperdb-api-derive/LICENSE-APACHE.txt b/hyperdb-api-derive/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-api-derive/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-api-derive/LICENSE-MIT.txt b/hyperdb-api-derive/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-api-derive/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-api-node/npm/darwin-arm64/package.json b/hyperdb-api-node/npm/darwin-arm64/package.json index ec826530..7f390e40 100644 --- a/hyperdb-api-node/npm/darwin-arm64/package.json +++ b/hyperdb-api-node/npm/darwin-arm64/package.json @@ -10,7 +10,10 @@ "files": [ "hyperdb-api-node.darwin-arm64.node", "hyperd", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "description": "hyperdb-api-node native binding for macOS ARM64 (Apple Silicon)", "license": "MIT OR Apache-2.0", diff --git a/hyperdb-api-node/npm/darwin-x64/package.json b/hyperdb-api-node/npm/darwin-x64/package.json index a62603b1..014b5e87 100644 --- a/hyperdb-api-node/npm/darwin-x64/package.json +++ b/hyperdb-api-node/npm/darwin-x64/package.json @@ -10,7 +10,10 @@ "files": [ "hyperdb-api-node.darwin-x64.node", "hyperd", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "description": "hyperdb-api-node native binding for macOS x64 (Intel)", "license": "MIT OR Apache-2.0", diff --git a/hyperdb-api-node/npm/linux-x64-gnu/package.json b/hyperdb-api-node/npm/linux-x64-gnu/package.json index e484d58d..0c8b0eaf 100644 --- a/hyperdb-api-node/npm/linux-x64-gnu/package.json +++ b/hyperdb-api-node/npm/linux-x64-gnu/package.json @@ -10,7 +10,10 @@ "files": [ "hyperdb-api-node.linux-x64-gnu.node", "hyperd", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "description": "hyperdb-api-node native binding for Linux x64 (glibc)", "license": "MIT OR Apache-2.0", diff --git a/hyperdb-api-node/npm/win32-x64-msvc/package.json b/hyperdb-api-node/npm/win32-x64-msvc/package.json index 9e90681a..355ec6c1 100644 --- a/hyperdb-api-node/npm/win32-x64-msvc/package.json +++ b/hyperdb-api-node/npm/win32-x64-msvc/package.json @@ -10,7 +10,10 @@ "files": [ "hyperdb-api-node.win32-x64-msvc.node", "hyperd.exe", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "description": "hyperdb-api-node native binding for Windows x64 (MSVC)", "license": "MIT OR Apache-2.0", diff --git a/hyperdb-api-node/package.json b/hyperdb-api-node/package.json index 20fdb00a..ac75e717 100644 --- a/hyperdb-api-node/package.json +++ b/hyperdb-api-node/package.json @@ -50,7 +50,10 @@ "index.js", "index.d.ts", "arrow.mjs", - "pool.mjs" + "pool.mjs", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "peerDependencies": { "apache-arrow": ">=14.0.0" diff --git a/hyperdb-api-salesforce/LICENSE-APACHE.txt b/hyperdb-api-salesforce/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-api-salesforce/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-api-salesforce/LICENSE-MIT.txt b/hyperdb-api-salesforce/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-api-salesforce/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-api/LICENSE-APACHE.txt b/hyperdb-api/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-api/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-api/LICENSE-MIT.txt b/hyperdb-api/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-api/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-bootstrap/LICENSE-APACHE.txt b/hyperdb-bootstrap/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-bootstrap/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-bootstrap/LICENSE-MIT.txt b/hyperdb-bootstrap/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-bootstrap/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-compile-check/LICENSE-APACHE.txt b/hyperdb-compile-check/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-compile-check/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-compile-check/LICENSE-MIT.txt b/hyperdb-compile-check/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-compile-check/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-mcp/LICENSE-APACHE.txt b/hyperdb-mcp/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/hyperdb-mcp/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/hyperdb-mcp/LICENSE-MIT.txt b/hyperdb-mcp/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/hyperdb-mcp/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/hyperdb-mcp/npm/darwin-arm64/package.json b/hyperdb-mcp/npm/darwin-arm64/package.json index b29f0f7a..55e43516 100644 --- a/hyperdb-mcp/npm/darwin-arm64/package.json +++ b/hyperdb-mcp/npm/darwin-arm64/package.json @@ -10,7 +10,10 @@ "files": [ "hyperdb-mcp", "hyperd", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "license": "MIT OR Apache-2.0", "engines": { diff --git a/hyperdb-mcp/npm/darwin-x64/package.json b/hyperdb-mcp/npm/darwin-x64/package.json index 821a26bd..92779353 100644 --- a/hyperdb-mcp/npm/darwin-x64/package.json +++ b/hyperdb-mcp/npm/darwin-x64/package.json @@ -6,7 +6,10 @@ "files": [ "hyperdb-mcp", "hyperd", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "license": "MIT OR Apache-2.0", "engines": { diff --git a/hyperdb-mcp/npm/linux-x64-gnu/package.json b/hyperdb-mcp/npm/linux-x64-gnu/package.json index 19505712..2dbf5f2c 100644 --- a/hyperdb-mcp/npm/linux-x64-gnu/package.json +++ b/hyperdb-mcp/npm/linux-x64-gnu/package.json @@ -13,7 +13,10 @@ "files": [ "hyperdb-mcp", "hyperd", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "license": "MIT OR Apache-2.0", "engines": { diff --git a/hyperdb-mcp/npm/package.json b/hyperdb-mcp/npm/package.json index 88290a16..8051fbce 100644 --- a/hyperdb-mcp/npm/package.json +++ b/hyperdb-mcp/npm/package.json @@ -6,7 +6,10 @@ }, "files": [ "bin.js", - "README.md" + "README.md", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "keywords": [ "hyper", diff --git a/hyperdb-mcp/npm/win32-x64-msvc/package.json b/hyperdb-mcp/npm/win32-x64-msvc/package.json index 44e045c7..a83eef12 100644 --- a/hyperdb-mcp/npm/win32-x64-msvc/package.json +++ b/hyperdb-mcp/npm/win32-x64-msvc/package.json @@ -10,7 +10,10 @@ "files": [ "hyperdb-mcp.exe", "hyperd.exe", - "LICENSE-HYPERD" + "LICENSE-HYPERD", + "LICENSE-MIT.txt", + "LICENSE-APACHE.txt", + "NOTICE" ], "license": "MIT OR Apache-2.0", "engines": { diff --git a/sea-query-hyperdb/LICENSE-APACHE.txt b/sea-query-hyperdb/LICENSE-APACHE.txt new file mode 100644 index 00000000..1a67589f --- /dev/null +++ b/sea-query-hyperdb/LICENSE-APACHE.txt @@ -0,0 +1,207 @@ +Apache License Version 2.0 + +Copyright (c) 2026 Salesforce, Inc. +All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/sea-query-hyperdb/LICENSE-MIT.txt b/sea-query-hyperdb/LICENSE-MIT.txt new file mode 100644 index 00000000..7bd98f95 --- /dev/null +++ b/sea-query-hyperdb/LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 Salesforce, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From c22379f88a3dc48bc8f9d94cef097e29e64e26f5 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 09:23:15 -0700 Subject: [PATCH 05/61] docs: pre-1.0 audit pass over docs, rustdoc and comments Corrects stale rustdoc, READMEs, DEVELOPMENT.md files, error-variant names, transport lists and example text across the workspace; documents that query_timeout is recorded but not applied; normalizes #[must_use] placement on the connection builders. --- AGENTS.md | 8 +- CODE_OF_CONDUCT.md | 2 +- CONTRIBUTING.md | 6 +- DEVELOPMENT.md | 4 +- README.md | 17 +- docs/BENCHMARK_GUIDE.md | 22 +- docs/GITHUB_OPERATIONS.md | 53 ++--- docs/ROW_MAPPING.md | 7 +- docs/RUST_DOCUMENTATION_STYLE.md | 18 +- docs/RUST_GUIDELINES.md | 2 +- docs/TRANSACTIONS.md | 4 +- docs/superpowers/specs/MIGRATING-0.3.md | 4 +- hyperdb-api-core/build.rs | 9 +- hyperdb-api-core/docs/DEVELOPMENT-client.md | 40 ++-- hyperdb-api-core/docs/DEVELOPMENT-protocol.md | 26 +-- hyperdb-api-core/docs/DEVELOPMENT-types.md | 38 ++-- .../src/client/grpc/authenticated_client.rs | 38 ++-- hyperdb-api-core/src/client/grpc/error.rs | 2 +- hyperdb-api-core/src/client/prepare.rs | 25 +-- hyperdb-api-core/src/client/statement.rs | 4 +- hyperdb-api-core/src/types/special.rs | 40 ++-- hyperdb-api-derive/src/lib.rs | 8 +- hyperdb-api-derive/src/table_derive.rs | 13 +- hyperdb-api-node/AGENTS.md | 32 ++- hyperdb-api-node/DEVELOPMENT.md | 94 ++++----- hyperdb-api-node/README.md | 106 ++++------ .../examples/hyper-explorer/README.md | 41 ++-- hyperdb-api-node/src/arrow_inserter.rs | 11 +- hyperdb-api-node/src/connection.rs | 11 +- hyperdb-api-node/src/inserter.rs | 39 +--- hyperdb-api-node/src/prepared.rs | 21 +- hyperdb-api-node/src/result.rs | 10 +- hyperdb-api-salesforce/CHANGELOG.md | 9 +- hyperdb-api-salesforce/DEVELOPMENT.md | 22 +- hyperdb-api-salesforce/README.md | 18 +- hyperdb-api-salesforce/src/lib.rs | 5 +- hyperdb-api-salesforce/src/token.rs | 9 +- hyperdb-api/README.md | 23 +- hyperdb-api/benches/benchmark.rs | 15 +- .../additional_examples/async_usage.rs | 8 +- .../additional_examples/threaded_inserter.rs | 5 +- .../additional_examples/transactions.rs | 37 ++-- .../examples/create_hyper_file_from_csv.rs | 4 +- hyperdb-api/src/arrow_inserter.rs | 34 +-- hyperdb-api/src/arrow_reader.rs | 2 +- hyperdb-api/src/arrow_result.rs | 18 +- hyperdb-api/src/async_arrow_inserter.rs | 42 ++-- hyperdb-api/src/async_connection_builder.rs | 26 ++- hyperdb-api/src/async_transaction.rs | 7 +- hyperdb-api/src/catalog.rs | 4 +- hyperdb-api/src/connection.rs | 160 +++++++------- hyperdb-api/src/connection_builder.rs | 50 +++-- hyperdb-api/src/copy.rs | 18 +- hyperdb-api/src/error.rs | 2 +- hyperdb-api/src/grpc_connection.rs | 49 ++--- hyperdb-api/src/lib.rs | 46 ++-- hyperdb-api/src/process.rs | 76 +++---- hyperdb-api/src/result.rs | 151 ++++++-------- hyperdb-api/src/row_accessor.rs | 2 +- hyperdb-api/src/table_copy.rs | 10 +- hyperdb-api/src/transaction.rs | 5 +- hyperdb-api/src/transport.rs | 8 +- hyperdb-api/tests/arrow_inserter_tests.rs | 197 +++++++++++------- hyperdb-api/tests/grpc_streaming_tests.rs | 29 +-- hyperdb-api/tests/remaining_features_tests.rs | 12 +- hyperdb-bootstrap/CHANGELOG.md | 7 +- hyperdb-bootstrap/README.md | 73 +++---- hyperdb-bootstrap/src/install.rs | 2 +- hyperdb-compile-check/CHANGELOG.md | 4 +- hyperdb-compile-check/src/db.rs | 25 ++- hyperdb-compile-check/src/dry_run.rs | 4 +- hyperdb-compile-check/src/error_extract.rs | 10 +- hyperdb-mcp/README.md | 84 +++++++- hyperdb-mcp/build.rs | 26 +-- hyperdb-mcp/src/chart.rs | 4 +- hyperdb-mcp/src/engine.rs | 65 +++--- hyperdb-mcp/src/error.rs | 4 +- hyperdb-mcp/src/export.rs | 63 +++--- hyperdb-mcp/src/ingest.rs | 2 +- hyperdb-mcp/src/lib.rs | 8 +- hyperdb-mcp/src/main.rs | 7 +- hyperdb-mcp/src/paths.rs | 4 +- hyperdb-mcp/src/readme.rs | 31 +-- hyperdb-mcp/src/saved_queries.rs | 36 ++-- hyperdb-mcp/src/server.rs | 78 +++---- hyperdb-mcp/src/stats.rs | 2 +- hyperdb-mcp/src/subscriptions.rs | 30 ++- hyperdb-mcp/src/watcher.rs | 60 +++--- hyperdb-mcp/tests/daemon_tests.rs | 24 ++- hyperdb-mcp/tests/doctor_tests.rs | 21 +- hyperdb-mcp/tests/error_tests.rs | 14 +- hyperdb-mcp/tests/export_tests.rs | 78 ++----- hyperdb-mcp/tests/ingest_arrow_tests.rs | 10 +- hyperdb-mcp/tests/ingest_tests.rs | 9 +- hyperdb-mcp/tests/per_tool_database_tests.rs | 15 +- hyperdb-mcp/tests/readme_tests.rs | 2 +- hyperdb-mcp/tests/recovery_tests.rs | 18 +- hyperdb-mcp/tests/table_catalog_tests.rs | 35 ++-- hyperdb-mcp/tests/watcher_tests.rs | 9 +- sea-query-hyperdb/CHANGELOG.md | 4 +- sea-query-hyperdb/DEVELOPMENT.md | 2 +- sea-query-hyperdb/README.md | 39 ++-- sea-query-hyperdb/examples/basic_usage.rs | 10 +- 103 files changed, 1377 insertions(+), 1400 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 51e2226e..372848fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -261,8 +261,8 @@ cargo test -p hyperdb-api # Run a specific test cargo test -p hyperdb-api test_name -# Run tests in a specific file (use module path) -cargo test -p hyperdb-api --test integration_test +# Run tests in a specific file (use the file stem) +cargo test -p hyperdb-api --test connection_tests ``` ### Running Examples @@ -311,7 +311,7 @@ hyperdb-api-core/src/types/ # Unit tests (inline with code) **Test utilities:** - `hyperdb-api/tests/common/mod.rs` - Shared test helpers -- `hyperdb-api-core/src/client/test_util.rs` - Client test utilities +- `hyperdb-api-core/tests/common/mod.rs` - Client test utilities - Both use `HyperProcess::new()` to start temporary `hyperd` servers **Pattern:** Tests create temporary `.hyper` files and clean them up automatically. The `make clean-test-files` command removes any leftover test artifacts. @@ -346,7 +346,7 @@ Query results are **always streaming** to maintain constant memory usage: ```rust let mut result = conn.execute_query("SELECT * FROM large_table")?; -// Process in chunks (default: 16384 rows per chunk) +// Process in chunks (TCP: up to 65,536 rows per chunk) while let Some(chunk) = result.next_chunk()? { for row in &chunk { // Process row diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d6e51026..fa64cc81 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -99,7 +99,7 @@ It includes adaptions and additions from [Go Community Code of Conduct][golang-c This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us]. [contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/) -[golang-coc]: https://golang.org/conduct +[golang-coc]: https://go.dev/conduct [cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md [microsoft-coc]: https://opensource.microsoft.com/codeofconduct/ [cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34bd2817..6914bf44 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Use GitHub Issues page to submit issues, enhancement requests and discuss ideas. ### Bug Reports and Fixes - If you find a bug, please search for it in the [Issues](https://github.com/tableau/hyper-api-rust/issues), and if it isn't already tracked, - [create a new issue](https://github.com/tableau/hyper-api-rust/issues/new). Fill out the "Bug Report" section of the issue template. Even if an Issue is closed, feel free to comment and add details, it will still + [create a new issue](https://github.com/tableau/hyper-api-rust/issues/new). Include the crate and version, your OS, the `hyperd --version` output, a minimal reproduction, and the expected vs. actual behavior. Even if an Issue is closed, feel free to comment and add details, it will still be reviewed. - Issues that have already been identified as a bug (note: able to reproduce) will be labelled `bug`. - If you'd like to submit a fix for a bug, [send a Pull Request](#creating-a-pull-request) and mention the Issue number. @@ -140,7 +140,7 @@ GPG signing is also supported — see [GitHub's signing-commits guide](https://d 7. **Submit** a Pull Request against the `main` branch and refer to the issue(s) you are fixing. Try not to pollute your pull request with unintended changes. Keep it simple and small. 8. **Sign** the Salesforce CLA (you will be prompted to do so when submitting the Pull Request) -> **NOTE**: Be sure to [sync your fork](https://help.github.com/articles/syncing-a-fork/) before making a pull request. +> **NOTE**: Be sure to [sync your fork](https://docs.github.com/en/pull-requests/how-tos/work-with-forks/syncing-a-fork) before making a pull request. ## Contributor License Agreement ("CLA") @@ -274,4 +274,4 @@ Please follow our [Code of Conduct](CODE_OF_CONDUCT.md). ## License -By contributing your code, you agree to license your contribution under the terms of our project [MIT](LICENSE-MIT) and [Apache-2.0](LICENSE-APACHE) dual license, and to sign the [Salesforce CLA](https://cla.salesforce.com/sign-cla). +By contributing your code, you agree to license your contribution under the terms of our project [MIT](LICENSE-MIT.txt) and [Apache-2.0](LICENSE-APACHE.txt) dual license, and to sign the [Salesforce CLA](https://cla.salesforce.com/sign-cla). diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 898c7c53..568d4630 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -560,7 +560,7 @@ cargo test -p hyperdb-api cargo test -p hyperdb-api test_name # Run a specific integration test file -cargo test -p hyperdb-api --test integration_test +cargo test -p hyperdb-api --test connection_tests ``` ### Test Structure @@ -576,7 +576,7 @@ hyperdb-api-core/src/types/ # Unit tests (inline with code) **Test utilities:** - `hyperdb-api/tests/common/mod.rs` — shared test helpers -- `hyperdb-api-core/src/client/test_util.rs` — client test utilities +- `hyperdb-api-core/tests/common/mod.rs` — client test utilities - Both use `HyperProcess::new()` to start temporary `hyperd` servers Tests create temporary `.hyper` files and clean them up automatically. diff --git a/README.md b/README.md index b22f0b5b..97b96a48 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Add to your `Cargo.toml`: ```toml [dependencies] -hyperdb-api = { path = "hyperdb-api" } +hyperdb-api = "1.0" ``` #### Installing the CLIs @@ -291,14 +291,21 @@ window functions, CTEs, complex JOINs, and type-safe query composition: ```toml [dependencies] -sea-query = "0.32" -sea-query-hyperdb = { path = "sea-query-hyperdb" } +sea-query = "1.0" +sea-query-hyperdb = "1.0" ``` ```rust -use sea_query::{Query, Expr, Iden}; +use sea_query::{Expr, ExprTrait, Iden, Query}; use sea_query_hyperdb::HyperQueryBuilder; +#[derive(Iden)] +enum Users { + Table, + Name, + Age, +} + let sql = Query::select() .column(Users::Name) .from(Users::Table) @@ -315,7 +322,7 @@ and Refresh Token flows: ```toml [dependencies] -hyperdb-api-salesforce = { path = "hyperdb-api-salesforce" } +hyperdb-api-salesforce = "1.0" ``` ```rust diff --git a/docs/BENCHMARK_GUIDE.md b/docs/BENCHMARK_GUIDE.md index ded86e40..64644b31 100644 --- a/docs/BENCHMARK_GUIDE.md +++ b/docs/BENCHMARK_GUIDE.md @@ -15,7 +15,7 @@ same tables. The Rust-side specialized benches (`benchmark.rs`, `async_parallel_benchmark.rs`) are "dig-deeper" references for specific questions. -All benchmarks share the same schema so numbers compare directly: +`benchmark.rs`, `benchmark_suite.rs`, `async_parallel_benchmark.rs` and the Node.js bench share the same schema so numbers compare directly: ```sql measurements(id INT NOT NULL, sensor_id INT, value DOUBLE, timestamp BIGINT) @@ -25,15 +25,15 @@ measurements(id INT NOT NULL, sensor_id INT, value DOUBLE, timestamp BIGINT) The shared primitives (`ResourceStats`, `HostEnv`, formatting, deterministic row generators) live in [`hyperdb-api/benches/common.rs`](../hyperdb-api/benches/common.rs) and -are pulled into each bench via -`#[path = "common.rs"] mod common;`. +are pulled into `benchmark.rs`, `benchmark_suite.rs` and +`async_parallel_benchmark.rs` via `#[path = "common.rs"] mod common;`. --- ## Running the benchmark suite ```sh -export HYPERD_PATH=/path/to/hyperd +export HYPERD_PATH="$PWD/.hyperd/current" # from the workspace root # Default: 10M rows per workload, 4 parallel workers cargo run -p hyperdb-api --release --example benchmark_suite @@ -48,8 +48,8 @@ BENCH_TRANSPORT=ipc \ ``` The banner under `Configuration:` prints `Transport: Tcp` or `Transport: Ipc` -so the saved `benchmark_suite.md` always records which transport produced -the numbers. +to stdout only; the saved `benchmark_suite.md` and `.json` do not record it, +so note the transport next to any table you paste. The suite prints a live log and, at the end, writes two artifacts under `test_results/`: @@ -74,7 +74,7 @@ under `test_results/`: Parallel async queries run against the database populated by the parallel-Arrow insert (one table per worker), so the full-scan row -count is *N × rows-per-workload*. +count is the rows-per-workload total, split across the *N* tables. ### Other benches (deep-dive) @@ -109,7 +109,7 @@ path), and `executeQueryToArrow` (full Arrow IPC roundtrip). cd hyperdb-api-node npm install # first time only npm run build # builds hyperdb-api-node..node -HYPERD_PATH=/path/to/hyperd node __test__/benchmark.mjs [ROWS] +HYPERD_PATH="$PWD/../.hyperd/current" node __test__/benchmark.mjs [ROWS] ``` Default is 1M rows. 10M matches the Rust suite's default for @@ -200,7 +200,7 @@ block from the suite's stdout. - **Parallel reads are the standout** — `query.full_scan × 4` reaches roughly **73 M rows/s / 1763 MB/s**, very approximately 2× the single-connection sync scan. Per the note above these are order-of-magnitude figures, so do not read a precise speedup ratio out of them; the single-connection rows are the ones with a tight enough spread to compare. - **Parallelism no longer helps Arrow inserts.** Since the `0.0.26479` engine, single-connection `AsyncArrowInserter` (68.9 M rows/s) outruns `AsyncArrowInserter × 4`, so spending connections on an Arrow insert buys nothing on this host. - **Sync beats async on single-connection reads.** `query.full_scan` sync runs 31.1 M rows/s against async's 24.9 M rows/s, and `query.filtered` 33.2 vs 26.9 M rows/s. Async wins only once it can use multiple connections, so prefer the sync path for a single streaming consumer and reach for async when you have concurrency to exploit. -- **Async dominates single-connection *inserts*** — `AsyncArrowInserter` at 68.9 M rows/s versus sync `Inserter` at 25.5 M rows/s, a 2.7× gap. This is the one figure the `0.0.26479` engine bump moved: **+127%** (30.4 → 68.9 M rows/s), reproduced as **+75%** at 10M. Both are medians of 5 interleaved runs whose old and new ranges do not overlap, so the gain survives this workload's wide ±25–35% spread. Sync inserts were unaffected. +- **Async dominates single-connection *inserts*** — `AsyncArrowInserter` at 68.9 M rows/s versus sync `Inserter` at 25.0 M rows/s, a 2.8× gap. This is the one figure the `0.0.26479` engine bump moved: **+127%** (30.4 → 68.9 M rows/s), reproduced as **+75%** at 10M. Both are medians of 5 interleaved runs whose old and new ranges do not overlap, so the gain survives this workload's wide ±25–35% spread. Sync inserts were unaffected. - **Single-connection scans are much faster than the previous entry** (18.8 → 31.1 M rows/s sync full-scan). Note this is *not* a controlled comparison: the prior numbers were taken on a different `hyperd`, rustc 1.94, and macOS 26.4, so the gain cannot be attributed to any single change. #### Transport A/B — Unix Domain Socket vs TCP loopback @@ -579,14 +579,14 @@ out differently — worth measuring. 2. Run the suite: ```sh - HYPERD_PATH=/path/to/hyperd \ + HYPERD_PATH="$PWD/.hyperd/current" \ ./target/release/examples/benchmark_suite 100000000 4 ``` 3. Copy-paste: - The `Host:` block from stdout into the platform section as the hardware/software block. - The `| Workload | … |` markdown table at the end of stdout into the results block. -4. Commit both the doc update and the JSON artifact (`test_results/benchmark_suite.json`) so future runs can diff against yours. +4. Commit the doc update, and keep or attach the gitignored `test_results/benchmark_suite.json` so future runs can diff against yours. ## Tuning diff --git a/docs/GITHUB_OPERATIONS.md b/docs/GITHUB_OPERATIONS.md index fe9692bc..db34f122 100644 --- a/docs/GITHUB_OPERATIONS.md +++ b/docs/GITHUB_OPERATIONS.md @@ -22,9 +22,9 @@ Seven GitHub Actions workflows live under [`.github/workflows/`](../.github/work | Workflow | File | Triggers | Purpose | |---|---|---|---| -| `ci` | [ci.yml](../.github/workflows/ci.yml) | `push` to `main`, all PRs, manual | fmt, clippy, full test matrix, `cargo deny`, `cargo audit`, `cargo publish --dry-run` | +| `ci` | [ci.yml](../.github/workflows/ci.yml) | `push` to `main`, all PRs, manual | fmt, clippy, msrv, doc, full test matrix, Node bindings build, `cargo deny`, `cargo audit`, version consistency, `cargo publish --dry-run` | | `release-please` | [release-please.yml](../.github/workflows/release-please.yml) | `push` to `main`, GitHub Release `published`, manual | open/update the release PR with version bumps + CHANGELOG. Does **not** tag: `skip-github-release: true` means the maintainer creates the tag and Release by hand — see [Cutting a release](#cutting-a-release). The `release: published` re-run re-anchors the next `-rc.N` on the just-cut tag (#308) | -| `release` | [release.yml](../.github/workflows/release.yml) | GitHub Release `published`, manual (`workflow_dispatch` against an existing tag) | re-run tests, publish the 8 Rust crates to crates.io (`hyperdb-api-node` is published separately to npm). **Not** a tag push: that trigger was removed to stop duplicate runs, so pushing a tag alone publishes nothing | +| `release` | [release.yml](../.github/workflows/release.yml) | GitHub Release `published`, manual (`workflow_dispatch` against an existing tag) | check that CI already passed on the tagged SHA, publish the 8 Rust crates to crates.io (`hyperdb-api-node` is published separately to npm). **Not** a tag push: that trigger was removed to stop duplicate runs, so pushing a tag alone publishes nothing | | `npm-build-publish` | [npm-build-publish.yml](../.github/workflows/npm-build-publish.yml) | GitHub Release published, manual | build npm platform packages with bundled hyperd, publish to npm registry | | `verify-hyperd-pin` | [verify-hyperd-pin.yml](../.github/workflows/verify-hyperd-pin.yml) | changes to `hyperdb-bootstrap/hyperd-version.toml` or its source, weekly cron, manual | `HEAD` every pinned hyperd release URL to catch Tableau yanks / typos | | `verify-release-pr-version` | [verify-release-pr-version.yml](../.github/workflows/verify-release-pr-version.yml) | `pull_request` to `main` | fail the release-please PR if it proposes a version behind `main` (backward-bump guard, #308). Runs on every PR but only acts on the `release-please--branches--*` branch, so non-release PRs report a plain `success` | @@ -35,11 +35,15 @@ Seven GitHub Actions workflows live under [`.github/workflows/`](../.github/work Runs on **every PR** and on **every push to `main`**. Jobs: - `rustfmt` — `cargo fmt --all --check`. -- `clippy` — `cargo clippy --workspace --all-targets -- -D warnings` (single runner; lints are platform-independent). -- `test` — full workspace test matrix on `ubuntu-24.04`, `macos-26`, `windows-2025`. -- `publish-dry-run` — `cargo publish --dry-run` for each publishable crate so a broken publish manifest is caught before a tag is cut. +- `clippy (ubuntu-24.04)` and `clippy (windows-2025)` — `cargo clippy --workspace --all-targets --all-features -- -D warnings`. +- `msrv (1.88)` — `cargo check` of the workspace and `hyperdb-compile-check` at the MSRV floor. +- `doc` — `cargo doc --no-deps` with `RUSTDOCFLAGS="-D warnings"` for the eight workspace crates (mirrors `make doc`); `hyperdb-compile-check` sits outside the workspace and is not covered. +- `test (ubuntu-24.04)`, `test (macos-26)`, `test (windows-2025)` — full workspace test matrix. +- `hyperdb-api-node (build + smoke)` — builds the native addon and runs a smoke test. +- `publish dry-run` — `cargo publish --dry-run` for `hyperdb-bootstrap` and `sea-query-hyperdb` only; the other crates depend on sibling versions that exist only after the release wave lands, so their dry-run cannot resolve. - `cargo-deny` — license and advisory policy enforcement per [`deny.toml`](../deny.toml). - `cargo-audit` — RustSec advisories, `--deny warnings`. +- `version consistency` — `version.txt` matches the Cargo workspace version, and the npm `package.json` files carry no in-source `version` field (`npm-build-publish.yml` injects it at publish time). In-progress PR CI runs are **cancelled** when a new commit is pushed to the PR. Main-branch runs always complete. This is set via the @@ -53,7 +57,7 @@ is `true`) or via manual `workflow_dispatch` with an explicit tag input (for re-runs or emergency releases). Structure: ```text -verify ← full test suite + hyperd URL check, single-platform +verify ← polls the CI check-runs on the tagged SHA + hyperd URL check │ └─► publish ← crates.io publish in dependency order ``` @@ -72,9 +76,11 @@ verification step resolves the just-published dep): break the optional cycle with `hyperdb-api-core`) 2. `hyperdb-api-core` 3. `hyperdb-api` -4. `hyperdb-mcp` -5. `hyperdb-bootstrap` -6. `sea-query-hyperdb` +4. `hyperdb-compile-check` (outside the workspace; published with `--allow-dirty`) +5. `hyperdb-api-derive` +6. `hyperdb-mcp` +7. `hyperdb-bootstrap` +8. `sea-query-hyperdb` `hyperdb-api-node` is **not** on the crates.io list (its `Cargo.toml` has `publish = false`) — it ships as npm `hyperdb-api-node` through napi-rs's @@ -85,7 +91,7 @@ automatically for tags containing `-rc.`, `-alpha.`, or `-beta.` — they show up on the Releases page but are not flagged as "Latest release". **Concurrency:** only one release workflow runs at a time (the `concurrency: -release` group at the top of the file); a second tag push during a +release` group at the top of the file); a second release event during a release will queue, not clobber. ### npm-publish (`npm-build-publish.yml`) @@ -97,8 +103,8 @@ without needing Rust toolchains or manual hyperd setup. **Triggers:** -- **GitHub Release published** — fires automatically after `release.yml` - creates/updates a GitHub Release. +- **GitHub Release published** — fires on the same `release: published` + event as `release.yml`; the two workflows run in parallel. - **Manual `workflow_dispatch`** — tag/branch input is optional; leave empty to build from the default branch HEAD (useful for testing the pipeline without tagging). @@ -106,9 +112,9 @@ without needing Rust toolchains or manual hyperd setup. **Structure:** ```text -verify-ci ← checks that CI passed for this commit (gh api commit status) +verify-ci ← checks that CI passed for this commit (gh api check-runs) │ - └─► build-npm (matrix × 4 platforms) + └─► build-npm (matrix × 3 platforms) build hyperdb-mcp + hyperdb-api-node native binaries download hyperd via curl with SHA256 verification assemble platform packages (binary + hyperd + LICENSE-HYPERD) @@ -143,10 +149,10 @@ Intel-Mac users must build from source. | `hyperdb-api-node` | Main (napi-rs) | JS bindings + `getHyperdPath()` helper | | `hyperdb-api-node-*` | Platform | `.node` addon + `hyperd` + `LICENSE-HYPERD` | -**CI gate:** The `verify-ci` job checks that the combined commit status -is `success` before building. If CI hasn't passed (e.g., someone -triggers a manual dispatch on a broken commit), the workflow aborts -immediately. Note: this does **not** prevent tagging — git tags can be +**CI gate:** On a `release: published` event the `verify-ci` job polls the +tagged commit's CI check-runs and aborts the workflow if a required check +failed. A manual `workflow_dispatch` skips `verify-ci` and builds without +that gate. Note: this does **not** prevent tagging — git tags can be created regardless of CI status. Use GitHub Rulesets (repo Settings → Rules) to enforce tag-creation restrictions if needed. @@ -316,7 +322,7 @@ gh release create vX.Y.Z \ After the tag is created: 1. Watch [`release.yml`](https://github.com/tableau/hyper-api-rust/actions/workflows/release.yml) — - it re-runs the verify suite on the tagged SHA, then publishes the + it confirms CI already passed on the tagged SHA, then publishes the crates to crates.io in dependency order. 2. Watch [`npm-build-publish.yml`](https://github.com/tableau/hyper-api-rust/actions/workflows/npm-build-publish.yml) in parallel. @@ -727,8 +733,8 @@ The `release` workflow is mostly idempotent but there are two sharp edges: `Publish in dependency order` step. Wait for the cooldown printed in the error and rerun via Actions → `release` → "Run workflow", entering the same tag name in the `tag` input. Already-published crates fail loudly - with "already uploaded" and the run will continue past them via the - per-crate retry below. + with "already exists on" and the publish step skips that crate and + continues with the next one. ### Re-running release.yml against an existing tag @@ -746,7 +752,7 @@ The workflow's regex validator rejects malformed tag names, and | Secret | Used by | Scope | |---|---|---| -| `RELEASE_PLEASE_TOKEN` | [release-please.yml](../.github/workflows/release-please.yml) | Classic PAT; triggers CI on release-please PRs/tags (see below) | +| `RELEASE_PLEASE_TOKEN` | [release-please.yml](../.github/workflows/release-please.yml) | Classic PAT; triggers CI on release-please PRs (see below) | | `CARGO_REGISTRY_TOKEN` | [release.yml](../.github/workflows/release.yml) `publish` job | `cargo publish` to crates.io | | `NPM_TOKEN` | [npm-build-publish.yml](../.github/workflows/npm-build-publish.yml) `publish-npm` job | `npm publish` to npmjs.org | | `GITHUB_TOKEN` | Every workflow | Auto-provided by GitHub Actions; used to post releases, download artifacts, verify CI status | @@ -755,8 +761,7 @@ The workflow's regex validator rejects malformed tag names, and GitHub Actions suppresses workflow triggers on events created by `GITHUB_TOKEN` (anti-recursion protection). Without a PAT, PRs opened -by release-please don't trigger CI, and tags it pushes don't trigger -`release.yml` or `npm-build-publish.yml`. The workaround is a PAT +by release-please don't trigger CI. The workaround is a PAT stored as `RELEASE_PLEASE_TOKEN`. ### Option A: Classic PAT (current setup) diff --git a/docs/ROW_MAPPING.md b/docs/ROW_MAPPING.md index 88c0d7a0..faec400a 100644 --- a/docs/ROW_MAPPING.md +++ b/docs/ROW_MAPPING.md @@ -183,10 +183,11 @@ and `Option` fields use `get_opt` (NULL → `None`) instead of `get` (NULL Use `#[hyperdb(rename = "col_name")]` when a field name doesn't match its column name, or `#[hyperdb(index = N)]` for positional access. +The derive macro lives in the `hyperdb-api-derive` crate; add `hyperdb-api-derive = "1.0"` to `[dependencies]`. + ```rust -use hyperdb_api::{ - Connection, CreateMode, FromRow, HyperProcess, Result, -}; +use hyperdb_api::{Connection, CreateMode, HyperProcess, Result}; +use hyperdb_api_derive::FromRow; // The derive generates: impl FromRow for Product { fn from_row(...) { ... } } // Each field maps to the column with the same name. diff --git a/docs/RUST_DOCUMENTATION_STYLE.md b/docs/RUST_DOCUMENTATION_STYLE.md index c76d7931..58ef3774 100644 --- a/docs/RUST_DOCUMENTATION_STYLE.md +++ b/docs/RUST_DOCUMENTATION_STYLE.md @@ -157,26 +157,26 @@ Use rustdoc's intra-doc links: `[`Connection`]` or `[`crate::Connection`]`. Use these conventional heading sections in doc comments where applicable: ```rust -/// Opens a connection to the database. +/// Connects to a database on a running `hyperd` instance. /// /// # Examples /// /// ```no_run -/// use hyperdb_api::{Connection, HyperProcess}; +/// use hyperdb_api::{Connection, CreateMode, HyperProcess}; /// /// let hyper = HyperProcess::new(None, None)?; -/// let conn = Connection::open(&hyper, "example.hyper")?; +/// let conn = Connection::new(&hyper, "example.hyper", CreateMode::CreateIfNotExists)?; /// # Ok::<(), hyperdb_api::Error>(()) /// ``` /// /// # Errors /// -/// Returns [`Error::ConnectionFailed`] if the server is unreachable. -/// -/// # Panics -/// -/// Panics if `path` contains a null byte. -pub fn open(process: &HyperProcess, path: &str) -> Result { +/// Returns [`Error::Connection`] if the server is unreachable. +pub fn new( + instance: &HyperProcess, + database_path: impl AsRef, + create_mode: CreateMode, +) -> Result { ``` * **`# Examples`** — Provide for all non-trivial public APIs. Doc examples are compiled and run by `cargo test`, so they also serve as integration tests. Use `no_run` for examples that need a running `hyperd` server. Use `#` prefix to hide boilerplate lines. diff --git a/docs/RUST_GUIDELINES.md b/docs/RUST_GUIDELINES.md index e62a8604..0d3f96b3 100644 --- a/docs/RUST_GUIDELINES.md +++ b/docs/RUST_GUIDELINES.md @@ -21,7 +21,7 @@ reviewed 2026-09-04 against the full 92-guideline export. [msft-about]: https://microsoft.github.io/rust-guidelines/guidelines/index.html [msft-all]: https://microsoft.github.io/rust-guidelines/agents/all.txt -[msft-changelog]: https://microsoft.github.io/rust-guidelines/changelog.html +[msft-changelog]: https://microsoft.github.io/rust-guidelines/changelog/index.html This page cites the subset that is either machine-enforced here or has been a live review topic; the remainder are adopted by reference. diff --git a/docs/TRANSACTIONS.md b/docs/TRANSACTIONS.md index 46af1998..417667e2 100644 --- a/docs/TRANSACTIONS.md +++ b/docs/TRANSACTIONS.md @@ -15,7 +15,7 @@ The recommended way to drive transactions is the **RAII guard** (`Transaction<'c ### RAII Transaction Guard (Sync) ```rust -use hyperdb_api::Transaction; +use hyperdb_api::{Connection, CreateMode, Transaction}; let mut conn = Connection::connect("localhost:7483", "db.hyper", CreateMode::DoNotCreate)?; let txn: Transaction<'_> = conn.transaction()?; // exclusively borrows conn @@ -93,7 +93,7 @@ txn.commit()?; ### RAII Transaction Guard (Async) ```rust -use hyperdb_api::AsyncTransaction; +use hyperdb_api::{AsyncConnection, AsyncTransaction, CreateMode}; let mut conn = AsyncConnection::connect("localhost:7483", "db.hyper", CreateMode::DoNotCreate).await?; let txn: AsyncTransaction<'_> = conn.transaction().await?; // exclusively borrows conn diff --git a/docs/superpowers/specs/MIGRATING-0.3.md b/docs/superpowers/specs/MIGRATING-0.3.md index edc67a19..d3f4c623 100644 --- a/docs/superpowers/specs/MIGRATING-0.3.md +++ b/docs/superpowers/specs/MIGRATING-0.3.md @@ -206,7 +206,7 @@ AsyncConnection::commit(&self) // -> #[doc(hidden)] #[deprecated] AsyncConnection::rollback(&self) // -> #[doc(hidden)] #[deprecated] ``` -At 0.3 these methods still existed and still worked, so a build saw compiler warnings rather than errors. **They were removed in 1.0.0.** If the guard's `&mut conn` borrow is impossible in your code, the replacements are `begin_transaction_unguarded` / `commit_unguarded` / `rollback_unguarded` — see [docs/TRANSACTIONS.md](docs/TRANSACTIONS.md#unguarded-transaction-control). +At 0.3 these methods still existed and still worked, so a build saw compiler warnings rather than errors. **They were removed in 1.0.0.** If the guard's `&mut conn` borrow is impossible in your code, the replacements are `begin_transaction_unguarded` / `commit_unguarded` / `rollback_unguarded` — see [docs/TRANSACTIONS.md](../../TRANSACTIONS.md#unguarded-transaction-control). ### Migration recipe @@ -268,7 +268,7 @@ Two in-tree holdouts remain, both by design rather than oversight — they are t The `FromRow` trait was redesigned around a new [`RowAccessor`] type and a new [`#[derive(FromRow)]`][derive] proc-macro. The blanket tuple impls (1/2/3/4-tuple) were deleted; hand-written impls have a new signature. [`RowAccessor`]: https://docs.rs/hyperdb-api/latest/hyperdb_api/struct.RowAccessor.html -[derive]: https://docs.rs/hyperdb-api/latest/hyperdb_api/derive.FromRow.html +[derive]: https://docs.rs/hyperdb-api-derive/latest/hyperdb_api_derive/derive.FromRow.html ### What's changed diff --git a/hyperdb-api-core/build.rs b/hyperdb-api-core/build.rs index 85baad72..f7535a87 100644 --- a/hyperdb-api-core/build.rs +++ b/hyperdb-api-core/build.rs @@ -1,10 +1,9 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Build script for hyper-client. +//! Build script for hyperdb-api-core. //! //! This generates Rust code from the `HyperService` protobuf definitions. -//! gRPC support is always available. fn main() -> Result<(), Box> { // Path to proto files (relative to this crate's root) @@ -12,14 +11,12 @@ fn main() -> Result<(), Box> { // Compile the HyperService and ErrorInfo protos. // - // `.bytes(["."])` tells prost to generate `bytes::Bytes` for every `bytes` + // `.bytes(".")` tells prost to generate `bytes::Bytes` for every `bytes` // field in the protos (instead of the default `Vec`). This lets prost // decode directly from the HTTP/2 frame buffer with a refcount bump // instead of an allocation + copy, which is critical on the gRPC result // path where payloads are full Arrow IPC streams. - // In tonic 0.14 the proto-compile entry point moved from `tonic-build` - // to the new `tonic-prost-build` crate; `tonic-build` itself now only - // handles TokenStream-level service codegen. + // Proto compilation uses `tonic-prost-build`. let mut config = tonic_prost_build::configure() .build_server(false) // We only need the client .build_client(true) diff --git a/hyperdb-api-core/docs/DEVELOPMENT-client.md b/hyperdb-api-core/docs/DEVELOPMENT-client.md index 747eb529..8170d88e 100644 --- a/hyperdb-api-core/docs/DEVELOPMENT-client.md +++ b/hyperdb-api-core/docs/DEVELOPMENT-client.md @@ -1,15 +1,15 @@ -# hyper-client Development Guide +# hyperdb-api-core `client` module: Development Guide -Contributor-facing documentation for the `hyper-client` crate -- the connection layer -between `hyperapi` (high-level API) and `hyper-protocol` (wire protocol). +Contributor-facing documentation for the `client` module of the internal `hyperdb-api-core` crate -- the connection layer +between `hyperdb-api` (high-level API) and the `protocol` module (wire protocol). -For user-facing documentation, see [README.md](README.md). +For user-facing documentation, see the [hyperdb-api README](../../hyperdb-api/README.md). --- ## Architecture Overview -`hyper-client` provides two transport families with both sync and async variants: +The `client` module provides two transport families with both sync and async variants: | Transport | Protocol | Capabilities | Variants | |-----------|----------|-------------|----------| @@ -93,7 +93,7 @@ COPY IN uses the PostgreSQL COPY subprotocol: 5. Server responds with `CommandComplete` + `ReadyForQuery` Supported formats: `HYPERBINARY` (default), `ARROWSTREAM`, `CSV`. -The higher-level `hyperapi::Inserter` handles binary encoding; `CopyInWriter` is +The higher-level `hyperdb_api::Inserter` handles binary encoding; `CopyInWriter` is a transport-level data pump. ### Connection Health and Desynchronization @@ -152,7 +152,7 @@ directory) into Rust types via `tonic-build`. The generated code lives in To regenerate after proto changes: ```bash -cargo build -p hyper-client +cargo build -p hyperdb-api-core ``` --- @@ -198,7 +198,7 @@ via `GrpcConfig` (auto-detected from `https://` endpoints). Integration tests require a running Hyper server. The `tests/common/mod.rs` module provides `TestServer`, which: -1. Starts a `HyperProcess` (via `hyperapi` dev-dependency) +1. Starts a `HyperProcess` (via `hyperdb-api` dev-dependency) 2. Creates a temporary database 3. Provides `Config` and `Client` helpers for the test 4. Cleans up on drop @@ -207,7 +207,7 @@ provides `TestServer`, which: use crate::common::TestServer; #[test] -fn test_something() -> hyperapi::Result<()> { +fn test_something() -> hyperdb_api::Result<()> { let server = TestServer::new()?; let client = server.connect()?; // ... test with real server ... @@ -215,19 +215,19 @@ fn test_something() -> hyperapi::Result<()> { } ``` -Test output (databases, logs) goes to `hyper-client/test_results/`. +Test output (databases, logs) goes to `hyperdb-api-core/test_results/`. ### Running Tests ```bash -# All hyper-client tests (requires hyperd on PATH or HYPERD_PATH set) -cargo test -p hyper-client +# All hyperdb-api-core tests (requires HYPERD_PATH or a discoverable .hyperd/current) +cargo test -p hyperdb-api-core # Unit tests only (no server needed) -cargo test -p hyper-client --lib +cargo test -p hyperdb-api-core --lib # Specific test file -cargo test -p hyper-client --test client_tests +cargo test -p hyperdb-api-core --test client_tests ``` ### TLS Tests @@ -242,7 +242,7 @@ without requiring pre-generated certificates. - Use `TestServer::without_database()` for tests that manage databases explicitly - Use `#[test]` for sync tests, `#[tokio::test]` for async tests - Keep test databases in `test_results/` (auto-managed by `TestServer`) -- gRPC tests live in `hyperapi/tests/` since they exercise the full stack +- gRPC tests live in `hyperdb-api/tests/` since they exercise the full stack --- @@ -250,7 +250,7 @@ without requiring pre-generated certificates. | Feature | Dependencies Added | What It Enables | |---------|--------------------|-----------------| -| `salesforce-auth` | `hyperapi-salesforce`, `chrono`, `arrow` | `AuthenticatedGrpcClient`, `with_data_cloud_token()` on `GrpcConfig` | +| `salesforce-auth` | `hyperdb-api-salesforce`, `arrow` | `AuthenticatedGrpcClient`, `with_data_cloud_token()` on `GrpcConfig` | Everything else (TCP clients, gRPC clients, TLS, auth) is always available. @@ -304,7 +304,7 @@ See `cancel.rs` for the full rationale. ## Related Documentation -- [Root DEVELOPMENT.md](../DEVELOPMENT.md) -- workspace-wide build, test, CI -- [hyper-protocol README](../hyper-protocol/README.md) -- wire protocol details -- [hyper-types README](../hyper-types/README.md) -- type system and binary formats -- [hyperapi README](../hyperapi/README.md) -- high-level API built on this crate +- [Root DEVELOPMENT.md](../../DEVELOPMENT.md) -- workspace-wide build, test, CI +- [Protocol development guide](DEVELOPMENT-protocol.md) -- wire protocol details +- [Types development guide](DEVELOPMENT-types.md) -- type system and binary formats +- [hyperdb-api README](../../hyperdb-api/README.md) -- high-level API built on this crate diff --git a/hyperdb-api-core/docs/DEVELOPMENT-protocol.md b/hyperdb-api-core/docs/DEVELOPMENT-protocol.md index 197722ce..87e21dbc 100644 --- a/hyperdb-api-core/docs/DEVELOPMENT-protocol.md +++ b/hyperdb-api-core/docs/DEVELOPMENT-protocol.md @@ -1,7 +1,7 @@ -# hyper-protocol Development Guide +# hyperdb-api-core `protocol` module: Development Guide -Contributor-facing documentation for the `hyper-protocol` crate. For the -user-facing overview, see [README.md](README.md). +Contributor-facing documentation for the `protocol` module of the internal `hyperdb-api-core` crate. For the +user-facing overview, see the [hyperdb-api README](../../hyperdb-api/README.md). --- @@ -59,7 +59,7 @@ into the original buffer via `bytes::Bytes::split_to()`. The `escape` module uses the newtype-with-Display pattern (`SqlIdentifier`, `SqlLiteral`) rather than functions returning `String`. This avoids intermediate allocations when escaping is used inside `format!()` calls -- which is the -primary use case in `hyperapi`'s SQL generation code. See the module-level doc +primary use case in `hyperdb-api`'s SQL generation code. See the module-level doc comment in `escape.rs` for the full rationale. --- @@ -89,10 +89,10 @@ cargo install --locked kani-verifier cargo kani setup # Run all proofs -cargo kani -p hyper-protocol +cargo kani -p hyperdb-api-core # Run a specific proof -cargo kani -p hyper-protocol --harness read_i32_no_panic +cargo kani -p hyperdb-api-core --harness read_i32_no_panic ``` ### Limitations @@ -154,7 +154,7 @@ unit tests instead. Each module has inline `#[cfg(test)] mod tests` blocks: ```bash -cargo test -p hyper-protocol +cargo test -p hyperdb-api-core ``` ### Doc Tests @@ -162,13 +162,13 @@ cargo test -p hyper-protocol Public functions with `/// # Example` blocks are compiled and run by: ```bash -cargo test -p hyper-protocol --doc +cargo test -p hyperdb-api-core --doc ``` ### Checking Documentation ```bash -RUSTDOCFLAGS="-D warnings" cargo doc -p hyper-protocol --no-deps +RUSTDOCFLAGS="-D warnings" cargo doc -p hyperdb-api-core --no-deps ``` This catches broken intra-doc links and missing docs (the crate enables @@ -197,7 +197,7 @@ This catches broken intra-doc links and missing docs (the crate enables ## Related Documentation -- [README.md](README.md) -- User-facing overview, key differences from PostgreSQL -- [hyper-types README](../hyper-types/README.md) -- Type system and binary serialization -- [hyper-client README](../hyper-client/README.md) -- Connection management -- [AGENTS.md](../AGENTS.md) -- AI assistant guidance for the full workspace +- [hyperdb-api README](../../hyperdb-api/README.md) -- User-facing overview; see [Key Differences from PostgreSQL](../../DEVELOPMENT.md#key-differences-from-postgresql) for protocol deviations +- [Types development guide](DEVELOPMENT-types.md) -- Type system and binary serialization +- [Client development guide](DEVELOPMENT-client.md) -- Connection management +- [AGENTS.md](../../AGENTS.md) -- AI assistant guidance for the full workspace diff --git a/hyperdb-api-core/docs/DEVELOPMENT-types.md b/hyperdb-api-core/docs/DEVELOPMENT-types.md index b234abf6..022d7097 100644 --- a/hyperdb-api-core/docs/DEVELOPMENT-types.md +++ b/hyperdb-api-core/docs/DEVELOPMENT-types.md @@ -1,8 +1,8 @@ -# hyper-types Development Guide +# hyperdb-api-core `types` module: Development Guide -Contributor-facing documentation for the `hyper-types` crate: internal architecture, how to extend the type system, testing, and known design decisions. +Contributor-facing documentation for the `types` module of the internal `hyperdb-api-core` crate: internal architecture, how to extend the type system, testing, and known design decisions. -For user-facing documentation (type mapping, serialization traits, API surface), see [README.md](README.md). +For user-facing documentation (API surface and usage examples), see the [hyperdb-api README](../../hyperdb-api/README.md). --- @@ -33,12 +33,12 @@ ToHyperBinary::to_hyper_binary[_not_null]() ← serialization bytes::BytesMut (LittleEndian wire bytes) │ ▼ -hyper-protocol COPY writer / Inserter ← sent to Hyper +protocol COPY writer / Inserter ← sent to Hyper │ ...round-trip through hyperd... │ ▼ -hyper-protocol COPY reader / RowDescription ← received from Hyper +protocol COPY reader / RowDescription ← received from Hyper │ ▼ FromHyperBinary::from_hyper_binary() ← deserialization @@ -127,18 +127,18 @@ impl ToHyperBinary for MyType { ### 5. Wire up higher layers -These changes are outside `hyper-types` but complete the integration: +These changes are outside the `types` module but complete the integration: -- `hyper-protocol`: Handle the type in COPY row parsing (if variable-length) -- `hyperapi`: Add `row.get::()` support in `result.rs` -- `hyperapi`: Add `Inserter` support if the type can be inserted +- `protocol` module: Handle the type in COPY row parsing (if variable-length) +- `hyperdb-api`: Add `row.get::()` support in `result.rs` +- `hyperdb-api`: Add `Inserter` support if the type can be inserted ### 6. Add tests - Unit tests in the same file as the implementation - Roundtrip tests: serialize then deserialize and compare - Edge cases: minimum/maximum values, NULL handling -- Integration tests in `hyperapi/tests/` with a real `hyperd` server +- Integration tests in `hyperdb-api/tests/` with a real `hyperd` server ### 7. Optional: Add chrono integration (`chrono_integration.rs`) @@ -155,14 +155,14 @@ For fixed-size types, add roundtrip and no-panic proof harnesses. ### Running Tests ```bash -# All hyper-types tests -cargo test -p hyper-types +# All hyperdb-api-core tests (narrow with a `types::` filter) +cargo test -p hyperdb-api-core # A specific test -cargo test -p hyper-types test_numeric_from_binary_with_scale +cargo test -p hyperdb-api-core test_numeric_from_binary_with_scale # Doc tests only -cargo test -p hyper-types --doc +cargo test -p hyperdb-api-core --doc ``` ### Test Patterns @@ -207,10 +207,10 @@ cargo install --locked kani-verifier cargo kani setup # Run all proofs -cargo kani -p hyper-types +cargo kani -p hyperdb-api-core # Run a specific proof -cargo kani -p hyper-types --harness date_encode_decode_roundtrip +cargo kani -p hyperdb-api-core --harness date_encode_decode_roundtrip ``` **Note:** Kani proofs avoid calling trait methods that return `Box` because Kani's model of dynamic dispatch causes infinite unwinding on `fmt::Debug` vtables. Instead, proofs verify at the byte level using `to_le_bytes` / `from_le_bytes` directly. @@ -227,7 +227,7 @@ cargo kani -p hyper-types --harness date_encode_decode_roundtrip ## Related Documentation -- [README.md](README.md) -- User-facing crate overview and type mapping -- [../AGENTS.md](../AGENTS.md) -- Repository-wide AI assistant guidance +- [hyperdb-api README](../../hyperdb-api/README.md) -- User-facing overview and usage examples +- [../../AGENTS.md](../../AGENTS.md) -- Repository-wide AI assistant guidance - [../../docs/RUST_DOCUMENTATION_STYLE.md](../../docs/RUST_DOCUMENTATION_STYLE.md) -- Documentation conventions -- Source-level docs: `cargo doc -p hyper-types --open` +- Source-level docs: `cargo doc -p hyperdb-api-core --open` diff --git a/hyperdb-api-core/src/client/grpc/authenticated_client.rs b/hyperdb-api-core/src/client/grpc/authenticated_client.rs index 3914ba84..5e83fcae 100644 --- a/hyperdb-api-core/src/client/grpc/authenticated_client.rs +++ b/hyperdb-api-core/src/client/grpc/authenticated_client.rs @@ -225,8 +225,8 @@ impl AuthenticatedGrpcClient { /// Sets the transfer mode for queries. /// /// - `Sync`: Wait for complete results (best for small results) - /// - `Async`: Stream results as they become available - /// - `Adaptive`: Server chooses based on result size (default) + /// - `Async`: Returns only the result header; all chunks are fetched with `GetQueryResult` + /// - `Adaptive`: Returns the first chunk inline and fetches the rest with `GetQueryResult` (default) #[must_use] pub fn with_transfer_mode(mut self, mode: TransferMode) -> Self { self.transfer_mode = mode; @@ -398,9 +398,10 @@ impl AuthenticatedGrpcClient { /// /// # Errors /// - /// - Returns [`Error::Authentication`] if the DC JWT cannot be - /// refreshed through the underlying Salesforce token provider - /// (including after the auth-retry budget is exhausted). + /// - Returns [`Error::Authentication`] if the proactive DC JWT refresh + /// fails, or [`Error::Config`] / [`Error::Connection`] if the channel + /// cannot be rebuilt. If a refresh triggered by an auth failure fails, + /// the original request error is returned. /// - Propagates any error from /// [`GrpcClient::execute_query_with_params_and_options`](super::GrpcClient::execute_query_with_params_and_options) — /// SQL / transport / protocol failures. @@ -469,8 +470,10 @@ impl AuthenticatedGrpcClient { /// /// # Errors /// - /// - Returns [`Error::Authentication`] if every retry attempt - /// still surfaces an auth error after forcing a token refresh. + /// - Returns [`Error::Authentication`] if the proactive DC JWT refresh + /// fails, or [`Error::Config`] / [`Error::Connection`] if the channel + /// cannot be rebuilt. If a refresh triggered by an auth failure fails, + /// the original request error is returned. /// - Propagates any other [`Error`] from the underlying gRPC /// executor (SQL errors, transport failures). pub async fn execute_query_with_options( @@ -576,8 +579,11 @@ impl AuthenticatedGrpcClient { /// /// # Errors /// - /// - Returns [`Error::Other`] if every retry attempt still - /// fails with an auth error after forcing a token refresh. + /// - Returns [`Error::Authentication`] if the proactive DC JWT refresh + /// fails, or [`Error::Config`] / [`Error::Connection`] if the channel + /// cannot be rebuilt. + /// - Returns the server's auth error if it persists after one forced + /// DC JWT refresh, or if that refresh fails. /// - Propagates any error from /// [`GrpcClient::cancel_query`](super::GrpcClient::cancel_query) (transport failure, `tonic::Status`). pub async fn cancel_query(&mut self, query_id: &str) -> Result<()> { @@ -636,7 +642,7 @@ impl AuthenticatedGrpcClient { /// Returns a list of schema names from the database. /// /// This queries the `pg_catalog` to get all user-defined schemas, - /// excluding system schemas like '`pg_catalog`', '`pg_temp`', and '`pg_toast`'. + /// excluding the `pg_catalog`, `pg_temp`, `pg_toast` and `information_schema` schemas. /// /// # Example /// @@ -677,7 +683,7 @@ impl AuthenticatedGrpcClient { /// Returns a list of table information from the database. /// /// This queries the `pg_catalog` to get all tables, views, and materialized views, - /// excluding system schemas. + /// excluding tables in `pg_catalog` and `pg_toast`. /// /// # Returns /// @@ -1252,10 +1258,12 @@ impl AuthenticatedGrpcClient { /// Checks if an error is an authentication error that should trigger /// a DC JWT refresh and query retry. /// - /// Only matches gRPC `UNAUTHENTICATED` (code 16) and HTTP 401 errors, - /// which are the server-side signals that the DC JWT is no longer valid. - /// Broader substring matches (e.g. "token", "expired") are intentionally - /// avoided to prevent spurious retries on unrelated errors. + /// Returns true for any [`Error::Authentication`] (gRPC `UNAUTHENTICATED` or + /// `PERMISSION_DENIED`, SQLSTATE class 28, or a client-side DC JWT failure) + /// and for any error whose message contains `unauthenticated`, + /// `unauthorized` or `401`. Broader substring matches (e.g. "token", + /// "expired") are intentionally avoided to prevent spurious retries on + /// unrelated errors. fn is_auth_error(error: &Error) -> bool { if matches!(error, Error::Authentication(_)) { return true; diff --git a/hyperdb-api-core/src/client/grpc/error.rs b/hyperdb-api-core/src/client/grpc/error.rs index 5d50c3a4..07341fcd 100644 --- a/hyperdb-api-core/src/client/grpc/error.rs +++ b/hyperdb-api-core/src/client/grpc/error.rs @@ -4,7 +4,7 @@ //! gRPC-specific error types. //! //! This module handles conversion from gRPC status codes and Hyper's structured -//! error details to the common [`crate::Error`] type. +//! error details to the common [`crate::client::Error`] type. use std::fmt; diff --git a/hyperdb-api-core/src/client/prepare.rs b/hyperdb-api-core/src/client/prepare.rs index e51710c4..fa7638e9 100644 --- a/hyperdb-api-core/src/client/prepare.rs +++ b/hyperdb-api-core/src/client/prepare.rs @@ -5,7 +5,7 @@ //! //! # Parameter Encoding //! -//! Use the \[`params!`\] macro for ergonomic parameter encoding: +//! Use the [`params!`](crate::params) macro for ergonomic parameter encoding: //! //! ```no_run //! # use hyperdb_api_core::{params, client::{Client, Config}}; @@ -29,13 +29,12 @@ use super::row::Row; use super::statement::{Column, ParamFormat, bind_format_codes}; use super::sync_stream::SyncStream; // ============================================================================= -// SqlParam trait - Zero-cost parameter encoding +// SqlParam trait - binary parameter encoding // ============================================================================= /// Trait for types that can be encoded as SQL prepared statement parameters. /// -/// This trait enables the \[`params!`\] macro to automatically encode values. -/// All implementations use `#[inline]` for zero-cost abstraction. +/// This trait enables the [`params!`](crate::params) macro to automatically encode values. pub trait SqlParam { /// Encodes the value as binary bytes. fn encode(&self) -> Vec; @@ -161,7 +160,7 @@ fn generate_statement_name() -> String { /// with different parameters efficiently. The statement is prepared once on /// the server and can be executed many times with different parameter values. /// -/// For automatic cleanup, use \[`OwnedPreparedStatement`\] via \[`crate::Client::prepare`\]. +/// Usually obtained through [`OwnedPreparedStatement::statement`]; [`Client::prepare`](crate::client::Client::prepare) returns the owned, self-closing wrapper. /// /// # Example /// @@ -180,9 +179,9 @@ pub struct PreparedStatement { name: String, /// Original SQL query string. query: String, - /// Parameter type OIDs (empty if types were inferred by the server). + /// Parameter type OIDs from the server's `ParameterDescription`, one per placeholder. param_types: Vec, - /// Result column descriptions (populated after first execution). + /// Result column descriptions from the `RowDescription` returned at prepare time (empty for statements without results). columns: Vec, } @@ -236,7 +235,7 @@ impl OwnedPreparedStatement { self.statement.query() } - /// Returns the parameter types. + /// Returns the parameter type OIDs the server reported for this statement. #[must_use] pub fn param_types(&self) -> &[Oid] { self.statement.param_types() @@ -319,7 +318,9 @@ impl PreparedStatement { &self.query } - /// Returns the parameter types. + /// Returns the parameter type OIDs the server reported for this statement. + /// + /// Hyper echoes the OIDs passed to `prepare_typed`, so an unspecified (`0`) OID reads back as `0`. #[must_use] pub fn param_types(&self) -> &[Oid] { &self.param_types @@ -349,7 +350,7 @@ impl PreparedStatement { /// # Errors /// /// - Returns [`Error`] (connection) if the connection mutex is poisoned. -/// - Returns [`Error`] (server) if the server rejects the `Parse` request +/// - Returns [`Error::Query`] if the server rejects the `Parse` request /// (SQL syntax error, unknown parameter OIDs, etc.). /// - Returns [`Error`] (I/O) / [`Error`] (closed) on wire-protocol I/O /// failure. @@ -437,7 +438,7 @@ pub fn prepare( /// # Errors /// /// - Returns [`Error`] (connection) if the connection mutex is poisoned. -/// - Returns [`Error`] (server) if the server rejects `Bind` / `Execute` +/// - Returns [`Error::Query`] if the server rejects `Bind` / `Execute` /// (parameter type mismatch, constraint violation, etc.). /// - Returns [`Error`] (I/O) / [`Error`] (closed) on wire-protocol I/O /// failure. @@ -590,7 +591,7 @@ pub fn execute_prepared_no_result_with_formats( /// # Errors /// /// - Returns [`Error`] (connection) if the connection mutex is poisoned. -/// - Returns [`Error`] (server) if the server reports an `ErrorResponse` +/// - Returns [`Error::Query`] if the server reports an `ErrorResponse` /// during `Close`/`Sync`. /// - Returns [`Error`] (I/O) / [`Error`] (closed) on wire-protocol I/O /// failure. diff --git a/hyperdb-api-core/src/client/statement.rs b/hyperdb-api-core/src/client/statement.rs index 4956fe55..9c22e98b 100644 --- a/hyperdb-api-core/src/client/statement.rs +++ b/hyperdb-api-core/src/client/statement.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Prepared statement handling. +//! Wire format codes and result-column metadata shared by the query and prepared-statement paths. use super::error::{Error, Result}; use crate::types::Oid; @@ -111,7 +111,7 @@ pub(crate) fn bind_format_codes( /// significantly impact performance. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ColumnFormat { - /// Text format (human-readable ASCII). + /// PostgreSQL text format (format code `0`, UTF-8). /// /// Values are sent as UTF-8 strings. Slower but human-readable. /// Use for debugging or when compatibility with text-based tools is needed. diff --git a/hyperdb-api-core/src/types/special.rs b/hyperdb-api-core/src/types/special.rs index 2fbcde9f..7a35ce64 100644 --- a/hyperdb-api-core/src/types/special.rs +++ b/hyperdb-api-core/src/types/special.rs @@ -1424,8 +1424,8 @@ impl Numeric { /// A geography (spatial) value. /// /// This type wraps raw geography bytes and provides type safety when working -/// with spatial data. With the `geography` feature enabled, it also supports -/// WKT/WKB parsing and conversion to/from `geo_types::Geometry`. +/// with spatial data. It also supports WKT/WKB parsing and conversion to/from +/// `geo_types::Geometry`. /// /// # Binary Format Notes /// @@ -1434,14 +1434,13 @@ impl Numeric { /// 1. **Hyper's Legacy Format**: Data read from Hyper query results is stored in /// Hyper's proprietary legacy serialization format. This format is **not** WKB-compatible. /// -/// 2. **WKB Format**: When created via `from_wkt()` or `from_wkb()` (with the -/// `geography` feature), data is stored in WKB (Well-Known Binary) format. +/// 2. **WKB Format**: When created via `from_wkt()` or `from_wkb()`, data is +/// stored in WKB (Well-Known Binary) format. /// -/// Methods like `to_geometry()` and `to_wkt()` (available with the `geography` -/// feature) expect WKB format and will **fail** when called on data in Hyper's -/// legacy format. +/// Methods like `to_geometry()` and `to_wkt()` expect WKB format and will **fail** +/// when called on data in Hyper's legacy format. /// -/// # Basic Usage (Always Available) +/// # Basic Usage /// /// ```no_run /// use hyperdb_api_core::types::Geography; @@ -1453,12 +1452,12 @@ impl Numeric { /// let bytes: &[u8] = geo.as_bytes(); /// ``` /// -/// # WKT/WKB Parsing (Requires `geography` Feature) +/// # WKT/WKB Parsing /// -/// ```ignore +/// ``` /// use hyperdb_api_core::types::Geography; /// -/// // Create from WKT string (requires "geography" feature) +/// // Create from WKT string /// let geo = Geography::from_wkt("POINT(-122.4194 37.7749)")?; /// /// // Convert to geo-types for processing @@ -1466,6 +1465,7 @@ impl Numeric { /// /// // Export as WKT /// let wkt_string = geo.to_wkt()?; +/// # Ok::<(), Box>(()) /// ``` #[derive(Clone, PartialEq, Eq, Hash)] pub struct Geography { @@ -1504,9 +1504,8 @@ impl Geography { /// # Format Compatibility /// /// Data read from Hyper is in Hyper's proprietary legacy format, **not** WKB. - /// Methods like `to_geometry()` and `to_wkt()` (available with the `geography` - /// feature) expect WKB format and will **fail** when called on data created - /// via this method. Use `format()` to check the format before calling these methods. + /// Methods like `to_geometry()` and `to_wkt()` expect WKB format and will **fail** + /// when called on data created via this method. Use `format()` to check the format before calling these methods. pub fn from_bytes(data: impl Into>) -> Self { Self { data: data.into(), @@ -1530,8 +1529,7 @@ impl Geography { /// Use this to check whether WKB-dependent methods like `to_geometry()` /// and `to_wkt()` will work on this value. /// - /// Note: When the `geography` feature is enabled, there's also a `format()` - /// method that performs runtime detection and returns the actual data. + /// See also `format()`, which performs runtime detection on the actual data. pub fn binary_format(&self) -> GeographyBinaryFormat { self.format } @@ -1673,7 +1671,7 @@ impl FromHyperBinary for Geography { } // ============================================================================= -// Geography - WKT/WKB support (requires `geography` feature) +// Geography - WKT/WKB support // ============================================================================= mod geo_impl { @@ -1722,13 +1720,13 @@ mod geo_impl { /// /// # Example /// - /// ```ignore + /// ``` /// use hyperdb_api_core::types::Geography; /// - /// // Requires "geography" feature /// let point = Geography::from_wkt("POINT(-122.4194 37.7749)")?; /// let line = Geography::from_wkt("LINESTRING(0 0, 1 1, 2 2)")?; /// let polygon = Geography::from_wkt("POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))")?; + /// # Ok::<(), Box>(()) /// ``` /// /// # Errors @@ -1844,13 +1842,13 @@ mod geo_impl { /// /// # Example /// - /// ```ignore + /// ``` /// use hyperdb_api_core::types::Geography; /// use geo_types::Point; /// - /// // Requires "geography" feature /// let point = Point::new(-122.4194, 37.7749); /// let geo = Geography::from_geometry(&point.into())?; + /// # Ok::<(), Box>(()) /// ``` /// /// # Errors diff --git a/hyperdb-api-derive/src/lib.rs b/hyperdb-api-derive/src/lib.rs index a93acf41..f55d4d63 100644 --- a/hyperdb-api-derive/src/lib.rs +++ b/hyperdb-api-derive/src/lib.rs @@ -77,8 +77,8 @@ enum FieldSource { /// /// # Attributes (field level) /// -/// - `#[hyperdb(primary_key)]` — marks the column as NOT NULL (always true -/// for non-`Option` fields, but documents intent). +/// - `#[hyperdb(primary_key)]` — documents intent only; it emits no constraint. +/// Nullability follows the field type (`Option` is nullable). /// - `#[hyperdb(rename = "col")]` — use a different SQL column name. #[proc_macro_derive(Table, attributes(hyperdb))] pub fn table_derive(input: TokenStream) -> TokenStream { @@ -248,9 +248,7 @@ fn expand_query_scalar(input: &TokenStream2) -> syn::Result { let sql_lit: Option = syn::parse2(quote!(#sql_expr)).ok(); if let Some(sql_lit) = sql_lit { let sql_str = sql_lit.value(); - // Validate SQL structure (syntax + table existence) using a dummy - // struct name that won't be in the registry — we only care about - // one-column check, not struct-field matching. + // Validate syntax, table registration, and that exactly one column is projected. match hyperdb_compile_check::validate_scalar_sql(&sql_str) { Ok(()) => {} Err(e) => { diff --git a/hyperdb-api-derive/src/table_derive.rs b/hyperdb-api-derive/src/table_derive.rs index 5e4e8bfd..21b0943c 100644 --- a/hyperdb-api-derive/src/table_derive.rs +++ b/hyperdb-api-derive/src/table_derive.rs @@ -108,7 +108,10 @@ struct StructOpts { table_name: Option, /// Whether `#[hyperdb(register)]` was present. /// Only used when `compile-time` feature is enabled. - #[allow(dead_code, reason = "only used when compile-time feature is enabled")] + #[cfg_attr( + not(feature = "compile-time"), + expect(dead_code, reason = "read only with the compile-time feature") + )] register: bool, } @@ -162,9 +165,11 @@ struct FieldOpts { /// Positional access (from `#[hyperdb(index = N)]`). Named columns are /// excluded from the column-subset validation check. index: Option, - /// Whether the field is the primary key. - /// Parsed but not yet used — reserved for v2 (schema enforcement). - #[allow(dead_code, reason = "reserved for v2 schema enforcement")] + /// Whether the field carries `#[hyperdb(primary_key)]`. + #[expect( + dead_code, + reason = "attribute accepted for documentation; no DDL emitted" + )] primary_key: bool, } diff --git a/hyperdb-api-node/AGENTS.md b/hyperdb-api-node/AGENTS.md index bcd041b7..7e824bdd 100644 --- a/hyperdb-api-node/AGENTS.md +++ b/hyperdb-api-node/AGENTS.md @@ -34,14 +34,14 @@ For documentation conventions specific to JavaScript/TypeScript, see the [Docume | `src/lib.rs` | Rust module declarations | | `src/connection.rs` | napi-rs `Connection`, `ConnectionBuilder` | | `src/process.rs` | napi-rs `HyperProcess` | -| `src/inserter.rs` | napi-rs `Inserter` | +| `src/inserter.rs` | napi-rs `RowInserter` | | `src/catalog.rs` | napi-rs `Catalog` | | `src/result.rs` | napi-rs `RowData`, `ResultColumnInfo` | | `src/query_stream.rs` | napi-rs `QueryStream` | | `src/columnar.rs` | napi-rs `ColumnarStream`, `ColumnarChunk` | | `src/types.rs` | napi-rs `SqlType`, `TableDefinition`, `CreateMode` | | `src/query_stats.rs` | Query statistics | -| `index.js` | Native binding loader + JS extensions (tagged templates, parameterized queries, event hooks, `Symbol.asyncDispose`, `toJSON`) | +| `index.js` | Native binding loader + JS extensions (tagged templates, `Symbol.asyncDispose`, async iteration) | | `index.d.ts` | Hand-written TypeScript declarations (full IntelliSense) | | `pool.mjs` | `ConnectionPool` — pure JS connection pooling | | `arrow.mjs` | Arrow convenience helpers (`tableFromQuery`, `insertFromTable`, etc.) | @@ -50,7 +50,7 @@ For documentation conventions specific to JavaScript/TypeScript, see the [Docume ## Build and Test Commands -**Prerequisites:** Rust toolchain, Node.js >= 21, `HYPERD_PATH` set. +**Prerequisites:** Rust toolchain, `protoc` (needed by the `hyperdb-api-core` build script), Node.js >= 21, and `hyperd` (run `make download-hyperd` from the repo root; `HyperProcess` finds `.hyperd/current` by walking up, or set an absolute `HYPERD_PATH`). ```bash cd hyperdb-api-node @@ -81,20 +81,17 @@ npm run build && npm run benchmark ### Rust ↔ JS Bridge - Each Rust struct in `src/*.rs` uses `#[napi]` attribute macros to expose constructors, methods, and properties to JS. -- **Sync Rust → Async JS:** Blocking Rust methods are wrapped with `tokio::task::spawn_blocking` and return `Promise` to JavaScript. -- **Thread safety:** Connections are wrapped in `Arc>` for safe concurrent access from the JS event loop. +- **Async Rust → Async JS:** Each `#[napi] async fn` awaits the matching `hyperdb_api::AsyncConnection` method on the napi tokio runtime and returns a `Promise`; `tokio::task::spawn_blocking` is reserved for CPU-heavy work. +- **Shared connection:** `Connection` holds an `Arc`, so concurrent calls share it without a JS-side lock. ### JS Extensions in `index.js` The native binding loader (`index.js`) is **not just a loader** — it adds significant functionality on top of the napi-rs exports: -- **Parameterized queries** (`executeQueryParams`, `executeCommandParams`) — `$1`/`$2` placeholder substitution via `escapeParam()` -- **Tagged template literals** (`conn.sql\`...\``,`conn.command\`...\``) — safe SQL interpolation -- **Query event hooks** (`conn.on('query', ...)`) — wraps `executeQuery`/`executeCommand` with timing +- **Tagged template literals** (`conn.sql` and `conn.command`) — `${value}` placeholders are rewritten to `$n` parameters of a prepared statement +- **`conn.sqlTyped`, `conn.queryObjects()`, `conn.transaction()`** — keyed plain-object rows and a BEGIN/COMMIT/ROLLBACK helper - **`Symbol.asyncDispose` / `Symbol.dispose`** — resource management for `await using` -- **`RowData.toJSON()`** — serialization with optional column names - **`QueryStream[Symbol.asyncIterator]`** — `for await (const row of stream)` support -- **`createExtractTable()`** — Tableau Extract schema helper When modifying connection behavior, check both `src/connection.rs` (Rust) and `index.js` (JS wrappers). @@ -113,9 +110,9 @@ When modifying connection behavior, check both `src/connection.rs` (Rust) and `i ## Testing -- **Smoke tests:** `__test__/smoke.mjs` — covers all major features (connection, queries, inserts, streams, pool, tagged templates, BigInt, dates, JSON, event hooks) +- **Smoke tests:** `__test__/smoke.mjs` — covers all major features (connection, queries, inserts, streams, pool, tagged templates, BigInt, dates, JSON) - **Benchmarks:** `__test__/benchmark.mjs` — insert and query performance with configurable row counts -- Tests require `HYPERD_PATH` to be set +- Tests need `hyperd`: run `make download-hyperd` (found via `.hyperd/current`), or set an absolute `HYPERD_PATH` - Test artifacts go into `test_results/` (gitignored) ## npm Publishing @@ -126,13 +123,11 @@ Uses napi-rs platform packages for cross-platform prebuilt binaries: |----------|---------| | macOS ARM64 | `hyperdb-api-node-darwin-arm64` | | Linux x64 (glibc) | `hyperdb-api-node-linux-x64-gnu` | -| Linux x64 (musl) | `hyperdb-api-node-linux-x64-musl` | -| Linux ARM64 | `hyperdb-api-node-linux-arm64-gnu` | | Windows x64 | `hyperdb-api-node-win32-x64-msvc` | -macOS x64 (Intel) is currently disabled — `macos-13` GHA runners are unreliable. +Only these three platforms are published; macOS x64 (Intel), Linux x64 (musl) and Linux ARM64 have no prebuilt package. -CI builds all enabled platforms on push. Publishing is driven by release-please. +On every PR and push, `ci.yml` builds the Linux debug addon and runs the smoke test (the `hyperdb-api-node (build + smoke)` job). The per-platform release builds run in `npm-build-publish.yml`, which the hand-created GitHub Release triggers; release-please only prepares the version bump and changelog in the release PR. ## Common Development Scenarios @@ -154,7 +149,7 @@ CI builds all enabled platforms on push. Publishing is driven by release-please. ### Modifying Type Mappings - **Rust → JS:** Reading conversions are in `src/result.rs` (`RowData` getters) -- **JS → Rust:** Writing conversions are in `src/inserter.rs` and `index.js` (`escapeParam`) +- **JS → Rust:** Writing conversions are in `src/inserter.rs` and `index.js` (`toPreparedParam`) - Update the type mapping tables in `README.md` if you change behavior ## Documentation Conventions @@ -176,8 +171,9 @@ Follow the [Documentation Style Guide](DOCUMENTATION_STYLE.md) for JSDoc convent ## Codebase-Specific Reminders 1. **`index.d.ts` is hand-written** — always update it when changing the API surface -2. **`index.js` is more than a loader** — it contains significant logic (params, templates, events, disposal) +2. **`index.js` is more than a loader** — it contains significant logic (templates, disposal, async iteration) 3. **Build release for benchmarks** — `npm run build:debug` produces binaries 10x+ slower 4. **`apache-arrow` is an optional peer dependency** — `arrow.mjs` gracefully handles its absence 5. **Tests use `.mjs` (ESM)** — use `import`/`export`, not `require` in test files 6. **CJS entry point** — `index.js` uses CommonJS (`require`/`module.exports`) for maximum compatibility +7. **Update `hyperdb-api-node/CHANGELOG.md`** under `## [Unreleased]` for user-visible API changes (see root [AGENTS.md](../AGENTS.md) reminder 8) diff --git a/hyperdb-api-node/DEVELOPMENT.md b/hyperdb-api-node/DEVELOPMENT.md index 1bc370ed..931d0e44 100644 --- a/hyperdb-api-node/DEVELOPMENT.md +++ b/hyperdb-api-node/DEVELOPMENT.md @@ -13,8 +13,7 @@ bindings on top of the Rust `hyperdb-api` crate. ┌───────────────────────────────────────────────────────┐ │ JS extensions │ │ index.js — native loader + tagged templates, │ -│ parameterized queries, event hooks, │ -│ Symbol.asyncDispose, RowData.toJSON │ +│ Symbol.asyncDispose, async iteration │ │ pool.mjs — ConnectionPool (pure JS) │ │ arrow.mjs — Arrow convenience helpers (pure JS) │ ├───────────────────────────────────────────────────────┤ @@ -29,11 +28,11 @@ bindings on top of the Rust `hyperdb-api` crate. **Key architecture decisions:** -- **Sync Rust, async JS.** The Rust `Connection` is synchronous internally. - Each async JS method runs the blocking Rust code on a `tokio::task::spawn_blocking` - thread pool, returning a Promise to JavaScript. -- **Thread safety.** Connections are wrapped in `Arc>` for safe - concurrent access from the JS event loop. +- **Async Rust, async JS.** Each `#[napi] async fn` awaits the matching + `hyperdb_api::AsyncConnection` method on the napi tokio runtime and returns a + Promise. +- **Shared connection.** `Connection` holds an `Arc`, + so concurrent calls share it without a JS-side lock. - **Eager result collection.** `executeQuery()` collects all rows into memory before returning. For large result sets, use streaming or Arrow APIs. @@ -57,11 +56,11 @@ classDiagram +executeQueryStream(sql) QueryStream +executeQueryColumnar(sql) ColumnarStream +executeQueryToArrow(sql) Buffer - +executeQueryParams(sql, params) RowData[] + +prepare(sql) PreparedStatement + +queryObjects(sql) object[] +sql(strings, ...values) RowData[] +command(strings, ...values) number +querySchema(sql) ResultColumnInfo[] - +on(event, listener) this +close() void } @@ -116,7 +115,7 @@ classDiagram +toString() string } - class Inserter { + class RowInserter { +bufferedRowCount: number +addRow(values) void +addRows(rows) void @@ -145,8 +144,6 @@ classDiagram +getTimestampMs(i) number +getJson(i) string +getString(i) string - +setColumnNames(names) this - +toJSON() object } class ColumnarChunk { @@ -164,13 +161,6 @@ classDiagram +index: number } - class QueryEvent { - +sql: string - +durationMs: number - +rowCount: number - +type: string - } - HyperProcess ..> Connection : creates ConnectionBuilder ..> Connection : builds ConnectionPool o-- Connection : pools @@ -179,13 +169,12 @@ classDiagram Connection ..> ColumnarStream : returns Connection ..> RowData : returns Connection ..> ResultColumnInfo : returns - Connection ..> QueryEvent : emits Catalog --> Connection : uses Catalog ..> TableDefinition : manages - Inserter --> Connection : uses - Inserter --> TableDefinition : uses + RowInserter --> Connection : uses + RowInserter --> TableDefinition : uses TableDefinition --> SqlType : column types @@ -204,7 +193,7 @@ classDiagram | `lib.rs` | Module declarations | | `connection.rs` | `Connection`, `ConnectionBuilder` — napi-rs bindings | | `process.rs` | `HyperProcess` — manages the `hyperd` server process | -| `inserter.rs` | `Inserter` — COPY protocol bulk inserts | +| `inserter.rs` | `RowInserter` — COPY protocol bulk inserts | | `catalog.rs` | `Catalog` — DDL operations (create/drop tables/schemas) | | `result.rs` | `RowData`, `ResultColumnInfo` — query result types | | `query_stream.rs` | `QueryStream` — streaming row-oriented results | @@ -238,8 +227,8 @@ classDiagram |------|---------| | `__test__/smoke.mjs` | Smoke tests covering all major features | | `__test__/benchmark.mjs` | Insert and query performance benchmarks | -| `examples/complete-api-tour.mts` | Full TypeScript API tour (19 sections) | -| `examples/complete-api-tour.mjs` | Same tour in plain JavaScript | +| `examples/complete-api-tour.mts` | TypeScript tour of the main features | +| `examples/complete-api-tour.mjs` | JavaScript variant of the tour (prepared-statement flavor) | | `examples/typed-analytics.mts` | TypeScript analytics pipeline | | `examples/arrow-analytics.mjs` | Arrow integration deep-dive | | `examples/generate-demo-data.mjs` | Demo data generator | @@ -250,9 +239,10 @@ classDiagram ### Prerequisites 1. **Rust toolchain** — install via [rustup](https://rustup.rs/) -2. **Node.js** >= 21 -3. **npm** >= 9 -4. **`hyperd` binary** — set `HYPERD_PATH` environment variable +2. **`protoc`** — required by the `hyperdb-api-core` build script (`brew install protobuf`, `apt-get install protobuf-compiler`, or `choco install protoc`) +3. **Node.js** >= 21 +4. **npm** >= 9 +5. **`hyperd` binary** — run `make download-hyperd` from the repo root; `HyperProcess` finds `.hyperd/current` by walking up from the working directory. Set an absolute `HYPERD_PATH` to override it. ### Build steps @@ -289,14 +279,14 @@ Always build in release mode before running benchmarks. ## Running Tests ```bash -# Requires HYPERD_PATH to be set +# Requires hyperd: run `make download-hyperd` first (or set an absolute HYPERD_PATH) npm test ``` Tests use Node.js built-in `assert` module (no external test framework). The smoke test (`__test__/smoke.mjs`) covers: connections, queries, inserts, streams, columnar, pool, tagged templates, parameterized queries, BigInt, -dates, JSON, event hooks, and resource disposal. +dates, JSON, and resource disposal. Test artifacts are written to `test_results/` (gitignored). @@ -331,12 +321,11 @@ for the end-to-end flow. |----------|---------| | macOS ARM64 | `hyperdb-api-node-darwin-arm64` | | Linux x64 (glibc) | `hyperdb-api-node-linux-x64-gnu` | -| Linux x64 (musl) | `hyperdb-api-node-linux-x64-musl` | -| Linux ARM64 | `hyperdb-api-node-linux-arm64-gnu` | | Windows x64 | `hyperdb-api-node-win32-x64-msvc` | -macOS x64 (Intel) builds are currently disabled — see [`npm-build-publish.yml`](../.github/workflows/npm-build-publish.yml) -matrix; we'll re-enable once `macos-13` runner availability is reliable. +Only these three platforms are published; macOS x64 (Intel), Linux x64 (musl) and +Linux ARM64 have no prebuilt package. See the [`npm-build-publish.yml`](../.github/workflows/npm-build-publish.yml) +matrix. The build/publish pipeline lives in [`.github/workflows/npm-build-publish.yml`](../.github/workflows/npm-build-publish.yml); @@ -366,10 +355,11 @@ proper cross-platform release. Step-by-step guide for exposing a new Rust method to JavaScript: 1. **Implement in Rust** — add the method to the appropriate `src/*.rs` file - with the `#[napi]` attribute. Use `tokio::task::spawn_blocking` for blocking - operations. -2. **Add JS wrapper (if needed)** — if the method needs parameter escaping, - event hooks, or other JS-level behavior, add a wrapper in `index.js`. + with the `#[napi]` attribute. Await the `hyperdb_api::AsyncConnection` method + directly; use `tokio::task::spawn_blocking` only for CPU-heavy work (as + `RowInserter::execute` does for row encoding). +2. **Add JS wrapper (if needed)** — if the method needs template rewriting + or other JS-level behavior, add a wrapper in `index.js`. 3. **Update TypeScript declarations** — add the method signature and JSDoc comment to `index.d.ts`. The `.d.ts` is hand-written, not generated. 4. **Add tests** — add a test case to `__test__/smoke.mjs`. @@ -380,18 +370,18 @@ Step-by-step guide for exposing a new Rust method to JavaScript: ```rust // src/connection.rs #[napi] -impl JsConnection { - /// Does something useful. - /// @param input - Description of the parameter. - /// @returns The result description. +impl Connection { + /// Runs a command and reports the affected row count. + /// @param sql - The SQL command to run. + /// @returns A description of the affected rows. #[napi] - pub async fn my_new_method(&self, input: String) -> napi::Result { - let conn = self.conn.clone(); - tokio::task::spawn_blocking(move || { - let guard = conn.lock().unwrap(); - // ... use guard ... - Ok("result".to_string()) - }).await.unwrap() + pub async fn my_new_method(&self, sql: String) -> Result { + let affected = self + .inner + .execute_command(&sql) + .await + .map_err(|e| Error::from_reason(e.to_string()))?; + Ok(format!("{affected} rows affected")) } } ``` @@ -407,10 +397,8 @@ For functionality that lives entirely in JavaScript (no Rust changes): 4. **Update README** — if user-facing, add to `README.md`. JS extensions in `index.js` include: tagged template literals (`conn.sql`, -`conn.command`), parameterized queries (`executeQueryParams`, -`executeCommandParams`), query event hooks (`conn.on('query', ...)`), -`Symbol.asyncDispose`/`Symbol.dispose`, `RowData.toJSON()`, -`QueryStream[Symbol.asyncIterator]`, and `createExtractTable()`. +`conn.command`, `conn.sqlTyped`), `conn.queryObjects()`, `conn.transaction()`, `Symbol.asyncDispose`/`Symbol.dispose`, and +`QueryStream[Symbol.asyncIterator]`. ## Updating `index.d.ts` When the API Changes diff --git a/hyperdb-api-node/README.md b/hyperdb-api-node/README.md index a4a1075f..1427d13d 100644 --- a/hyperdb-api-node/README.md +++ b/hyperdb-api-node/README.md @@ -8,14 +8,13 @@ This package provides a native Node.js addon that gives JavaScript and TypeScrip - **Async-first** — all I/O methods return Promises - **Tagged template literals** — `conn.sql\`SELECT * FROM t WHERE id = ${id}\`` — safe by default -- **Apache Arrow integration** — query results as Arrow Tables at 15M+ rows/sec +- **Apache Arrow integration** — query results as Arrow Tables at 25M+ rows/sec - **Native performance** — compiled Rust, no FFI overhead at runtime - **Full TypeScript support** — hand-written `.d.ts` declarations with full IntelliSense - **Connection pooling** — configurable min/max, idle timeout, acquire timeout - **Multiple query APIs** — row-oriented, columnar, Arrow IPC, tagged templates - **Resource management** — `Symbol.asyncDispose` for `await using` (Node 22+) -- **Query tracing** — `conn.on('query', ...)` hooks for logging/metrics -- **Cross-platform** — macOS (ARM & x64), Linux, Windows +- **Cross-platform** — macOS (Apple Silicon), Linux x64 (glibc), Windows x64 ## Class Overview @@ -25,7 +24,7 @@ ConnectionBuilder ─builds─▶ Connection ──returns──▶ QueryStream ConnectionPool ──pools──▶ Connection ──returns──▶ ColumnarStream ──yields──▶ ColumnarChunk Catalog ──uses──▶ Connection TableDefinition ──uses──▶ SqlType -Inserter ──uses──▶ Connection + TableDefinition +RowInserter ──uses──▶ Connection + TableDefinition ``` For a detailed class diagram with all methods and properties, see @@ -50,13 +49,9 @@ npm will automatically install the correct prebuilt binary for your platform via |---|---|---| | macOS | ARM64 (Apple Silicon) | `hyperdb-api-node-darwin-arm64` | | Linux | x64 (glibc) | `hyperdb-api-node-linux-x64-gnu` | -| Linux | x64 (musl/Alpine) | `hyperdb-api-node-linux-x64-musl` | -| Linux | ARM64 (glibc) | `hyperdb-api-node-linux-arm64-gnu` | | Windows | x64 (MSVC) | `hyperdb-api-node-win32-x64-msvc` | -> **macOS x64 (Intel) is not currently published** while we wait for `macos-13` -> GitHub Actions runners to become reliably available. Build from source for -> Intel Macs in the meantime. +> **Only the three platforms above are published.** macOS x64 (Intel), Linux x64 (musl/Alpine) and Linux ARM64 have no prebuilt package, so `require('hyperdb-api-node')` fails there unless you [build from source](#build-from-source). `make download-hyperd` provides `hyperd` only for macOS (ARM64 and x64), Linux x64 (glibc) and Windows x64. ### Build from source @@ -92,7 +87,7 @@ import { createRequire } from 'module'; const require = createRequire(import.meta.url); const { HyperProcess, Connection, CreateMode, Catalog, - TableDefinition, SqlType, Inserter, + TableDefinition, SqlType, RowInserter, } = require('hyperdb-api-node') as typeof import('hyperdb-api-node'); import type { RowData } from 'hyperdb-api-node'; @@ -107,7 +102,7 @@ tableDef.addColumn('score', SqlType.double(), true); await new Catalog(conn).createTable(tableDef); // Insert data -const inserter = new Inserter(conn, tableDef); +const inserter = new RowInserter(conn, tableDef); inserter.addRows([[1, 'Alice', 95.5], [2, 'Bob', null], [3, 'Charlie', 88.0]]); console.log(`Inserted ${await inserter.execute()} rows`); @@ -283,12 +278,12 @@ await catalog.getTableNames('public'); // ['users', 'products'] await catalog.dropTable('public.users'); ``` -### `Inserter` +### `RowInserter` High-performance bulk data inserter using the COPY protocol. ```typescript -const inserter = new Inserter(conn, tableDef); +const inserter = new RowInserter(conn, tableDef); inserter.addRow([1, 'Widget', 19.99]); // one at a time inserter.addRows([ // or batch @@ -299,7 +294,7 @@ inserter.addRows([ // or batch console.log(inserter.bufferedRowCount); // 3 const count = await inserter.execute(); // send to server, returns row count -// Inserter can be reused after execute() +// RowInserter can be reused after execute() inserter.addRow([4, 'Another', 7.77]); await inserter.execute(); ``` @@ -343,7 +338,7 @@ for await (const row of conn.executeQueryStream('SELECT * FROM big_table')) { | JSON | `getJson(i)` -> `JSON.parse()` | `string` | | Any type | `getString(i)` | `string` (fallback) | -**Writing (JS -> Hyper) — via Inserter or tagged templates:** +**Writing (JS -> Hyper) — via RowInserter or tagged templates:** | JS Type | SQL Type | |---|---| @@ -351,13 +346,13 @@ for await (const row of conn.executeQueryStream('SELECT * FROM big_table')) { | `string` | TEXT | | `boolean` | BOOLEAN | | `null` / `undefined` | NULL | -| `Buffer` | BYTEA | +| `Buffer` | BYTEA (`RowInserter` only) | | `Date` | TIMESTAMP (via `toISOString()`) | | `{}` (plain object) | JSON (via `JSON.stringify()`) | ### Tagged Template Literals -The idiomatic way to write safe queries. Values are automatically escaped to SQL literals -- no SQL injection possible: +The idiomatic way to write safe queries. Each `${value}` becomes a `$n` parameter of a server-side prepared statement -- no SQL injection possible: ```js const rows = await conn.sql`SELECT * FROM users WHERE name = ${name} AND age > ${minAge}`; @@ -365,22 +360,29 @@ await conn.command`INSERT INTO users (id, name) VALUES (${id}, ${name})`; await conn.command`UPDATE users SET score = ${score} WHERE id = ${id}`; ``` -Supported value types: `number`, `string`, `boolean`, `null`, `Buffer`. +Supported value types: `number`, `bigint`, `string`, `boolean`, `null`, `Date`. -### Parameterized Queries +### Prepared Statements `$1`/`$2` placeholder syntax (alternative to tagged templates): ```typescript -const rows = await conn.executeQueryParams( - 'SELECT * FROM users WHERE name = $1 AND age > $2', ['Alice', 30] -); -await conn.executeCommandParams( - 'INSERT INTO users (id, name, score) VALUES ($1, $2, $3)', [1, 'Alice', 95.5] -); +const stmt = conn.prepare('SELECT * FROM users WHERE name = $1 AND age > $2'); +try { + const rows = await stmt.query(['Alice', 30]); +} finally { + await stmt.close(); +} + +const insert = conn.prepare('INSERT INTO users (id, name, score) VALUES ($1, $2, $3)'); +try { + await insert.execute([1, 'Alice', 95.5]); +} finally { + await insert.close(); +} ``` -Supported parameter types: `number`, `string`, `boolean`, `null`, `Buffer`. Values are escaped -- no SQL injection possible. +Supported parameter types: `number`, `bigint`, `string`, `boolean`, `null`. Values are bound to a server-side prepared statement, so no SQL injection is possible. ### `ConnectionPool` @@ -401,7 +403,7 @@ const users = await pool.queryParams('SELECT * FROM users WHERE age > $1', [18]) // Custom logic with auto acquire/release const result = await pool.use(async (conn) => { await conn.executeCommand('BEGIN'); - await conn.executeCommandParams('INSERT INTO logs VALUES ($1)', ['event']); + await conn.command`INSERT INTO logs VALUES (${'event'})`; await conn.executeCommand('COMMIT'); return 'ok'; }); @@ -418,24 +420,14 @@ await pool.close(); | `acquireTimeoutMs` | 30000 | Max ms to wait for a connection (0 = no limit) | | `createMode` | `CreateIfNotExists` | Database creation mode | -### `RowData` Extras +### Lossless integers and keyed rows ```js row.getBigInt(0); // lossless 64-bit as BigInt (e.g., 9007199254740993n) -row.toJSON(); // { "0": "1", "1": "Alice", "2": "95.5" } -// Schema-aware keys -row.setColumnNames(schema.map(c => c.name)); -row.toJSON(); // { id: "1", name: "Alice", score: "95.5" } -``` - -### Query Event Hooks - -```js -conn.on('query', ({ sql, durationMs, rowCount, type }) => { - console.log(`[${type}] ${sql} — ${durationMs}ms, ${rowCount} rows`); -}); -conn.off('query', listener); // unsubscribe +// Plain objects keyed by column name (values are text representations) +const objs = await conn.queryObjects('SELECT id, name FROM users'); +const typed = await conn.sqlTyped`SELECT id, name FROM users WHERE id = ${1}`; ``` ### Resource Management (`await using`) @@ -511,28 +503,17 @@ const count = await insertFromTable(conn, tableDef, arrowTable); ## Benchmark Results -Measured on **Apple M3 Max**, Node.js v24, release build, 1M rows. - -Table schema: `measurements(id INT NOT NULL, sensor_id INT, value DOUBLE, timestamp BIGINT)` — 24 bytes/row. - -### Query Performance +Measured on **Apple M3 Max**, Node.js v24.18.0, release build, hyperd `0.0.26359`, 10M rows for inserts and 1M rows for scans. Table schema: `measurements(id INT NOT NULL, sensor_id INT, value DOUBLE, timestamp BIGINT)` — 24 bytes/row. | Benchmark | Rows/sec | MB/s | Notes | |---|---|---|---| -| **Arrow Full Scan** | 15.5M | 355.4 | `executeQueryToArrow` + `tableFromIPC` — fastest | -| **Arrow Filtered** | 15.6M | 358.1 | Arrow IPC with `WHERE sensor_id = 5` | -| **Columnar Full Scan** | 7.9M | 180.9 | `executeQueryColumnar` — no Arrow dependency | -| **Columnar Filtered** | 5.5M | 125.3 | Columnar stream with `WHERE` filter | -| Full Scan (eager) | 1.0M | 22.8 | `executeQuery` — all rows in memory | -| Full Scan (stream) | 836K | 19.1 | `executeQueryStream` — row-level iteration | -| Full Scan (chunked) | 637K | 14.6 | `nextChunk()` loop with per-row access | -| Aggregation | 195M | 4464.8 | `GROUP BY` — server-side, 10 result rows | +| **Arrow Full Scan** | 28.6M | 685.7 | `executeQueryToArrow` — fastest read path | +| **Columnar Full Scan** | 13.2M | 315.8 | `executeQueryColumnar` — no Arrow dependency | +| Full Scan (eager) | 1.46M | 35.1 | `executeQuery` — all rows in memory | +| **Insert (Arrow COPY)** | 41.3M | 991.7 | `ArrowInserter` — fastest insert path | +| Insert (COPY) | 2.15M | 51.5 | `RowInserter.addRows()` | -### Insert Performance - -| Benchmark | Rows/sec | MB/s | Notes | -|---|---|---|---| -| **Insert (COPY)** | 1.4M | 31.2 | `Inserter.addRows()` batched in 50K chunks | +See [docs/BENCHMARK_GUIDE.md](https://github.com/tableau/hyper-api-rust/blob/main/docs/BENCHMARK_GUIDE.md#nodejs-bench--10m-rows-same-schema) for the full table, methodology and host details. ### Which Query API to Use @@ -549,10 +530,11 @@ The `examples/` directory contains runnable examples: | Example | Description | Run | |---|---|---| -| `complete-api-tour.mts` | Full 19-section tour in TypeScript | `npx tsx examples/complete-api-tour.mts` | -| `complete-api-tour.mjs` | Same tour in plain JavaScript | `node examples/complete-api-tour.mjs` | +| `complete-api-tour.mts` | Tour of the main features in TypeScript | `npx tsx examples/complete-api-tour.mts` | +| `complete-api-tour.mjs` | JavaScript variant of the tour (prepared-statement flavor) | `node examples/complete-api-tour.mjs` | | `typed-analytics.mts` | TypeScript analytics pipeline | `npx tsx examples/typed-analytics.mts` | | `arrow-analytics.mjs` | Arrow integration deep-dive | `node examples/arrow-analytics.mjs` | +| `generate-demo-data.mjs` | Demo data generator | `node examples/generate-demo-data.mjs` | | `hyper-explorer/` | Web-based database inspector & generator | See below | All examples require `HYPERD_PATH` to be set. @@ -571,4 +553,4 @@ A full-stack web application (React + Express) for inspecting and generating `.h ## License -Apache-2.0 +Licensed under either of [Apache License, Version 2.0](https://github.com/tableau/hyper-api-rust/blob/main/LICENSE-APACHE.txt) or [MIT license](https://github.com/tableau/hyper-api-rust/blob/main/LICENSE-MIT.txt), at your option. diff --git a/hyperdb-api-node/examples/hyper-explorer/README.md b/hyperdb-api-node/examples/hyper-explorer/README.md index bb442b7f..5e293673 100644 --- a/hyperdb-api-node/examples/hyper-explorer/README.md +++ b/hyperdb-api-node/examples/hyper-explorer/README.md @@ -7,11 +7,11 @@ The UI is a React SPA (Vite + Tailwind). The API is Express + TypeScript, backed ## Prerequisites - **Node.js** >= 21 -- **Rust toolchain** (stable) — required to build `hyperdb-api-node` from the workspace when using `file:../../` +- **Rust toolchain** (stable) and **`protoc`** — required to build `hyperdb-api-node` from the workspace when using `file:../../` - **`hyperdb-api-node` built** — from the `hyperdb-api-node/` directory: - Release: `npm run build` (faster runtime) - Debug: `npm run build:debug` (faster compile while iterating on Rust) -- **`hyperd`** — set the **`HYPERD_PATH`** environment variable to the `hyperd` executable (same as the rest of this repo) +- **`hyperd`** — run `make download-hyperd` (or `.\build.ps1 download-hyperd`) from the repo root; `HyperProcess` finds `.hyperd/current` by walking up from the working directory. Set an absolute **`HYPERD_PATH`** to override it. ## Quick start @@ -56,7 +56,6 @@ For a static deployment, run `npm run build`, host `dist/` behind any static fil ## System architecture ```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'lineColor': '#93c5fd', 'primaryColor': '#1e293b', 'primaryTextColor': '#e2e8f0', 'primaryBorderColor': '#475569', 'secondaryColor': '#172554', 'tertiaryColor': '#1e293b', 'edgeLabelBackground': '#0f172a', 'clusterBkg': '#0f172a', 'clusterBorder': '#475569' }}}%% graph TB subgraph Browser["Browser (localhost:5173)"] React["React SPA
Vite + TailwindCSS"] @@ -96,10 +95,10 @@ graph TB Proxy -->|"proxy"| Express Pool -->|"hyperdb-api-node
Connection, Catalog"| HyperProcess - style Browser fill:#1e293b,stroke:#475569,color:#e2e8f0 - style Express fill:#1e293b,stroke:#475569,color:#e2e8f0 - style HyperProcess fill:#172554,stroke:#60a5fa,color:#93c5fd - style Vite fill:#1e293b,stroke:#475569,color:#e2e8f0 + style Browser fill:none,stroke:#475569 + style Express fill:none,stroke:#475569 + style HyperProcess fill:none,stroke:#60a5fa + style Vite fill:none,stroke:#475569 ``` ## Backend: connection pool and query statistics @@ -114,7 +113,6 @@ If `hyperd.log` is unavailable, stats collection is skipped gracefully; the app ## Request flow: opening a database ```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'lineColor': '#93c5fd', 'signalColor': '#93c5fd', 'actorLineColor': '#93c5fd', 'actorBorder': '#60a5fa', 'actorBkg': '#1e293b', 'actorTextColor': '#e2e8f0', 'loopTextColor': '#cbd5e1', 'noteBkgColor': '#1e3a5f', 'noteTextColor': '#e2e8f0', 'noteBorderColor': '#60a5fa', 'activationBkgColor': '#334155', 'activationBorderColor': '#60a5fa', 'sequenceNumberColor': '#93c5fd' }}}%% sequenceDiagram participant U as User participant R as React app @@ -146,7 +144,6 @@ sequenceDiagram ## Request flow: column detail + Fourier analysis ```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'lineColor': '#93c5fd', 'signalColor': '#93c5fd', 'actorLineColor': '#93c5fd', 'actorBorder': '#60a5fa', 'actorBkg': '#1e293b', 'actorTextColor': '#e2e8f0', 'loopTextColor': '#cbd5e1', 'noteBkgColor': '#1e3a5f', 'noteTextColor': '#e2e8f0', 'noteBorderColor': '#60a5fa', 'activationBkgColor': '#334155', 'activationBorderColor': '#60a5fa', 'sequenceNumberColor': '#93c5fd' }}}%% sequenceDiagram participant U as User participant R as React app @@ -183,7 +180,6 @@ sequenceDiagram ## Request flow: generating a database ```mermaid -%%{init: {'theme': 'base', 'themeVariables': { 'lineColor': '#93c5fd', 'signalColor': '#93c5fd', 'actorLineColor': '#93c5fd', 'actorBorder': '#60a5fa', 'actorBkg': '#1e293b', 'actorTextColor': '#e2e8f0', 'loopTextColor': '#cbd5e1', 'noteBkgColor': '#1e3a5f', 'noteTextColor': '#e2e8f0', 'noteBorderColor': '#60a5fa', 'activationBkgColor': '#334155', 'activationBorderColor': '#60a5fa', 'sequenceNumberColor': '#93c5fd' }}}%% sequenceDiagram participant U as User participant R as React app @@ -202,7 +198,7 @@ sequenceDiagram Note over H: Hyper generates rows using random() and math functions E->>H: executeQuery(SELECT COUNT(*)) end - E-->>R: {dbPath, results: [{table, rowCount, durationMs}], _queries} + E-->>R: {dbPath, results: [{table, rowCount, durationMs}]} R-->>U: Success + "Open in Explorer" U->>R: Open generated file R->>E: POST /api/open {path} @@ -233,12 +229,14 @@ sequenceDiagram | Data type | Statistics | |-----------|--------------| -| **All types** | Row count, null count/%, distinct count, cardinality % | -| **Numeric** (INT, DOUBLE, …) | Min, max, mean, median, std dev, CV%, variance, sum, p10/p25/p75/p90 | -| **Text** | Min/max/avg length, top values | -| **Boolean** | True/false counts and percentages | +| **All types** | Row count, null count/%, distinct count | +| **Numeric** (INT, DOUBLE, …) | Min, max, mean, std dev, CV% | +| **Text** | Min/max/avg length, top 5 values | +| **Boolean** | True/false counts, true % | | **Date / timestamp** | Min, max | +Cardinality %, median, variance, sum, and percentiles appear in the column detail view. + ### Numeric column visualizations (column detail) - **Value distribution** — smooth area chart (200 buckets, monotone cubic spline). @@ -277,6 +275,7 @@ hyper-explorer/ │ ├── main.tsx │ ├── App.tsx # Layout, tabs, pool-driven loading, global drag-drop │ ├── api.ts # fetch helpers, _queries fan-out, types (incl. QueryStats) +│ ├── index.css │ ├── fourierClient.ts # Client DFT, reconstruction, R², term selection │ └── components/ │ ├── FileOpener.tsx @@ -299,24 +298,24 @@ hyper-explorer/ ## API reference -All data routes (except browse/open/close/generate-meta) require the **`db`** query parameter or body field set to the absolute path of the open `.hyper` file. +All data routes (except browse/open/close/generate-meta/generate) require the **`db`** query parameter or body field set to the absolute path of the open `.hyper` file. | Method | Path | Description | |--------|------|-------------| | `GET` | `/api/browse?dir=` | List directories and `.hyper` files (optional `dir`, default home); items include `size`, `lastModified` | | `POST` | `/api/open` | Body `{ path }` — schema tree; may include `_queries` | | `POST` | `/api/close` | Body `{ path }` — close pooled connections for that path | -| `GET` | `/api/preview/:schema/:table?db=&limit=&offset=&sortColumn=&sortDir=` | Paginated rows + `totalRowCount`; `limit` capped at **1000**; optional `sortColumn` / `sortDir` (`asc`/`desc`); `_queries` | +| `GET` | `/api/preview/:schema/:table?db=&limit=&offset=&sortColumn=&sortDir=&withCount=` | Paginated rows; `limit` capped at **1000**; `totalRowCount` is computed only when `withCount=1` (otherwise `null`); optional `sortColumn` / `sortDir` (`asc`/`desc`); `_queries` | | `GET` | `/api/stats/:schema/:table?db=` | Per-column statistics; `_queries` | | `GET` | `/api/column-detail/:schema/:table/:column?db=` | Detail payload + `_queries` | -| `POST` | `/api/query` | Body `{ db, sql }` — SELECT → rows + columns; otherwise command; returns `durationMs`, optional `queryStats`, `_queries` | +| `POST` | `/api/query` | Body `{ db, sql }` — SELECT → rows + columns; otherwise command; returns `durationMs` and optional `queryStats` | | `GET` | `/api/generate-meta` | Supported types and distributions | -| `POST` | `/api/generate` | Body `{ dbPath, tables: [...] }` — create DB; `_queries` | +| `POST` | `/api/generate` | Body `{ dbPath, tables: [...] }` — deletes any existing file at `dbPath`, then creates the DB; returns `{ dbPath, results }` | ## Tech stack - **Backend** — Express 4, TypeScript, **`tsx`** (watch in dev), **`hyperdb-api-node`** (N-API bindings to the Rust Hyper client). -- **Frontend** — React 18, **Vite 7**, TailwindCSS 3, **`sql-formatter`** (PostgreSQL dialect). +- **Frontend** — React 18, **Vite 8**, TailwindCSS 3, **`sql-formatter`** (PostgreSQL dialect). - **Math** — Radix-2 FFT on the server for spectrum data; client-side DFT / Fourier reconstruction / monotone cubic splines (Fritsch–Carlson). - **Concurrency** — `ConnectionPool` + tracked wrappers attach `queryStats` when the log path is known. @@ -325,7 +324,7 @@ All data routes (except browse/open/close/generate-meta) require the **`db`** qu The example is self-contained aside from: - **`hyperdb-api-node`** — `file:../../` in `package.json` (swap for a published package if you ship it). -- **`hyperd`** — pointed to by **`HYPERD_PATH`**. +- **`hyperd`** — found via `.hyperd/current` or **`HYPERD_PATH`**. ## Notes diff --git a/hyperdb-api-node/src/arrow_inserter.rs b/hyperdb-api-node/src/arrow_inserter.rs index 76652014..c367332f 100644 --- a/hyperdb-api-node/src/arrow_inserter.rs +++ b/hyperdb-api-node/src/arrow_inserter.rs @@ -85,15 +85,10 @@ impl ArrowInserter { let ins = guard.take().ok_or_else(already_executed)?; ins.execute() .await - .map(|n| { - #[expect( - clippy::cast_possible_wrap, - reason = "NAPI BigInt ↔ Hyper u64 bit-pattern reinterpret; JS consumers read the BigInt as an unsigned inserted-row count" - )] - let signed = n as i64; - signed - }) .map_err(|e| Error::from_reason(e.to_string())) + .and_then(|n| { + i64::try_from(n).map_err(|_| Error::from_reason("row count exceeds i64::MAX")) + }) } /// Cancels the COPY stream without committing any rows. diff --git a/hyperdb-api-node/src/connection.rs b/hyperdb-api-node/src/connection.rs index 5fc8646b..18e2bfca 100644 --- a/hyperdb-api-node/src/connection.rs +++ b/hyperdb-api-node/src/connection.rs @@ -104,15 +104,10 @@ impl Connection { self.inner .execute_command(&sql) .await - .map(|n| { - #[expect( - clippy::cast_possible_wrap, - reason = "NAPI BigInt ↔ Hyper u64 bit-pattern reinterpret; JS consumers read the BigInt as an unsigned affected-row count" - )] - let signed = n as i64; - signed - }) .map_err(|e| Error::from_reason(e.to_string())) + .and_then(|n| { + i64::try_from(n).map_err(|_| Error::from_reason("row count exceeds i64::MAX")) + }) } /// Executes a SQL query and returns all result rows. diff --git a/hyperdb-api-node/src/inserter.rs b/hyperdb-api-node/src/inserter.rs index af241a0f..e869d5d2 100644 --- a/hyperdb-api-node/src/inserter.rs +++ b/hyperdb-api-node/src/inserter.rs @@ -27,7 +27,7 @@ use crate::types::TableDefinition; /// /// Rows are buffered in memory; `execute()` encodes them into Hyper's /// binary COPY format and ships them to the server on a tokio task. -/// For Arrow IPC data, use [`ArrowInserter`](crate::arrow_inserter::ArrowInserter) +/// For Arrow IPC data, use `ArrowInserter` /// instead — it bypasses the per-row JS encoding loop entirely. #[napi] #[derive(Debug)] @@ -271,12 +271,7 @@ impl RowInserter { .finish() .await .map_err(|e| Error::from_reason(e.to_string()))?; - #[expect( - clippy::cast_possible_wrap, - reason = "NAPI BigInt ↔ Hyper u64 bit-pattern reinterpret; JS consumers read the BigInt as an unsigned inserted-row count" - )] - let signed = count as i64; - Ok(signed) + i64::try_from(count).map_err(|_| Error::from_reason("row count exceeds i64::MAX")) } } @@ -315,9 +310,7 @@ fn encode_rows( } } -/// Adds a value to the chunk using the column's SQL type for correct binary encoding. -/// -/// Narrows an integer to `i16` for a `SMALLINT` destination column, or fails. +/// Narrows an `i64` to `i16` for a `SMALLINT` destination column, or fails. /// /// Truncating with `as` would be silent data corruption: `add_i16` encodes two /// bytes, so a wrapped value is a *valid* encoding of the wrong number and @@ -407,7 +400,7 @@ fn add_value_typed( clippy::trivially_copy_pass_by_ref, reason = "signature kept for API consistency with the trait family that unifies Copy and non-Copy implementers" )] -/// Parses a JS object `{ 0: number[], 2: number[] }` into a `HashMap`>. +/// Parses a JS object `{ 0: number[], 2: number[] }` into a `HashMap>`. fn parse_number_columns(_env: &Env, obj: &Object) -> Result>> { let names = obj.get_property_names()?; let len = names.get_named_property::("length")?; @@ -447,8 +440,7 @@ fn js_value_to_insert_value(val: Unknown) -> Result { match val.get_type()? { ValueType::Null | ValueType::Undefined => Ok(InsertValue::Null), ValueType::Boolean => { - // napi-rs 3: `coerce_to_bool()` now returns `Result` directly; - // the old 2.x pattern of `.get_value()` on a `JsBoolean` wrapper is gone. + // The match arm guarantees a JS boolean; `coerce_to_bool` returns its value. let b = val.coerce_to_bool()?; Ok(InsertValue::Bool(b)) } @@ -489,13 +481,8 @@ fn js_value_to_insert_value(val: Unknown) -> Result { } } ValueType::String => { - // SAFETY: we just matched `ValueType::String` from `val.get_type()` - // above, so NAPI has validated that the JS value is a JsString. - // `JsUnknown::cast` simply reinterprets the wrapper to the - // type-specific handle and is sound under that invariant. - // napi-rs 3: `cast::()` now returns `Result` (previously `T`), - // so the `?` propagates the type-check failure before we call - // `into_utf8()`. + // SAFETY: `get_type()` returned `ValueType::String`, so the handle is a + // JS string; `Unknown::cast::` wraps it without a further type check. let s = unsafe { val.cast::() }? .into_utf8()? .as_str()? @@ -503,15 +490,9 @@ fn js_value_to_insert_value(val: Unknown) -> Result { Ok(InsertValue::String(s)) } ValueType::Object => { - // SAFETY: we matched `ValueType::Object` above, which covers both - // plain objects and Node `Buffer` instances. `JsUnknown::cast` - // only reinterprets the handle; if the underlying object is not - // actually a `Buffer`, `into_value()` fails with `Err(..)` and - // we fall back to the string-coercion branch. NAPI's value type - // tag guarantees soundness of the `cast` itself. - // napi-rs 3: `cast::()` now returns `Result` directly - // (previously it was `Buffer` + an `.into_value()` step that could - // fail). `Buffer` derefs to `[u8]`, so `.to_vec()` copies the bytes. + // SAFETY: `get_type()` returned `ValueType::Object`. `Unknown::cast::` + // calls `napi_get_buffer_info`, which returns `Err` for a non-Buffer object, + // so plain objects fall through to the string-coercion branch. if let Ok(buf) = unsafe { val.cast::() } { Ok(InsertValue::Bytes(buf.to_vec())) } else { diff --git a/hyperdb-api-node/src/prepared.rs b/hyperdb-api-node/src/prepared.rs index e8d555c1..79d8e56f 100644 --- a/hyperdb-api-node/src/prepared.rs +++ b/hyperdb-api-node/src/prepared.rs @@ -14,7 +14,7 @@ use crate::result::{RowData, extract_row}; // PreparedStatement // ============================================================================= -/// A server-side prepared statement bound to a [`Connection`]. +/// A server-side prepared statement bound to a `Connection`. /// /// Prepared statements are the supported way to run parameterized /// queries: parameter values are sent over the wire in native binary @@ -22,16 +22,16 @@ use crate::result::{RowData, extract_row}; /// /// ## Type inference /// -/// [`Connection.prepare(sql)`] defers server-side preparation until the +/// `Connection.prepare(sql)` defers server-side preparation until the /// first call to `query`/`execute` because Hyper's Parse message /// requires the exact set of parameter OIDs up front. The first call /// prepares using the OIDs inferred from the JS parameter values, then /// caches the prepared statement for subsequent calls. /// /// If you later call `query` with *different* types for the same -/// placeholders (e.g. `[42]` then `[42n]`), the statement re-prepares +/// placeholders (e.g. `[42]` then `[5_000_000_000n]`, INT then BIGINT; a `bigint` that fits `i32`, such as `42n`, binds as INT like `42`), the statement re-prepares /// transparently. For predictable, cache-hot behavior pass explicit -/// OIDs via [`Connection.prepareTyped(sql, oids)`]. +/// OIDs via `Connection.prepareTyped(sql, oids)`. /// /// @example /// ```js @@ -211,7 +211,7 @@ impl Connection { /// Parameter types are inferred from the JS values passed on the /// first call; if those types change between calls the statement /// re-prepares transparently. For explicit OID control, use - /// [`prepareTyped`](Self::prepare_typed). + /// `prepareTyped`. #[napi] pub fn prepare(&self, sql: String) -> PreparedStatement { let param_count = count_placeholders(&sql); @@ -357,15 +357,10 @@ async fn run_execute( .stmt .execute(&refs) .await - .map(|n| { - #[expect( - clippy::cast_possible_wrap, - reason = "NAPI BigInt ↔ Hyper u64 bit-pattern reinterpret; JS consumers read the BigInt as an unsigned affected-row count" - )] - let signed = n as i64; - signed - }) .map_err(|e| Error::from_reason(e.to_string())) + .and_then(|n| { + i64::try_from(n).map_err(|_| Error::from_reason("row count exceeds i64::MAX")) + }) } async fn run_fetch_one( diff --git a/hyperdb-api-node/src/result.rs b/hyperdb-api-node/src/result.rs index 5b374f51..bd80bb23 100644 --- a/hyperdb-api-node/src/result.rs +++ b/hyperdb-api-node/src/result.rs @@ -115,7 +115,7 @@ pub(crate) fn extract_row( /// A row from a query result. /// /// Use the typed accessor methods to get column values by index (0-based). -/// Returns `null` for NULL values or type mismatches. +/// Accessors return `null` for NULL values or type mismatches; `getInt32` throws when a BIGINT value does not fit. #[napi] #[derive(Debug)] pub struct RowData { @@ -270,9 +270,9 @@ impl RowData { } } - /// Gets a JSON column value as a parsed string (ready for JSON.parse in JS). + /// Gets a JSON column value as its raw JSON text. /// - /// Returns the raw JSON text. Use `JSON.parse(row.getJSON(idx))` in JS. + /// Use `JSON.parse(row.getJson(idx))` in JS to obtain an object. #[napi] pub fn get_json(&self, index: u32) -> Option { match self.values.get(index as usize)? { @@ -371,7 +371,7 @@ impl RowData { mod hex { #[expect( clippy::format_collect, - reason = "readable hex/string formatting loop; refactoring to fold! obscures intent" + reason = "readable hex/string formatting loop; per-byte `format!` keeps the encoder a one-liner; getString on a BYTEA cell is not a hot path" )] pub(super) fn encode(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() @@ -379,7 +379,7 @@ mod hex { } // ============================================================================= -// ResultSchemaInfo - Column metadata returned to JS +// ResultColumnInfo - Column metadata returned to JS // ============================================================================= /// Metadata about a column in a query result. diff --git a/hyperdb-api-salesforce/CHANGELOG.md b/hyperdb-api-salesforce/CHANGELOG.md index 61d82f72..fa9ab24c 100644 --- a/hyperdb-api-salesforce/CHANGELOG.md +++ b/hyperdb-api-salesforce/CHANGELOG.md @@ -9,13 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [1.0.0-rc.3] - 2026-09-07 +These changes first shipped in 1.0.0-rc.1. + ### Changed -- **BREAKING:** the `arrow` dependency moved from **58** to **59**, matching - `hyperdb-api`. Arrow IPC types cross this crate's API surface, so consumers - must move in lockstep. +- The `arrow` dev-dependency (used only by the example) moved from **58** to **59**, matching + `hyperdb-api`. This does not affect consumers. -- **BREAKING:** the minimum supported Rust version is now **1.88**, up from +- The minimum supported Rust version is now **1.88**, up from 1.81, and the crate is compiled with **edition 2024**. 1.88 is the version Red Hat Enterprise Linux 9.7 ships as `rust-toolset`. - **BREAKING:** the TLS crypto provider is now **ring** rather than AWS-LC. diff --git a/hyperdb-api-salesforce/DEVELOPMENT.md b/hyperdb-api-salesforce/DEVELOPMENT.md index dbcf7236..1f6335b2 100644 --- a/hyperdb-api-salesforce/DEVELOPMENT.md +++ b/hyperdb-api-salesforce/DEVELOPMENT.md @@ -44,24 +44,20 @@ The OAuth Access Token and DC JWT are cached independently in rotation that would invalidate tokens held by other connections sharing the same refresh token. -- **DC JWT** (~2-hour hard expiry): Refreshed proactively via `needs_refresh()`, - which checks two conditions: - 1. **Expiry threshold** -- fewer than 300 seconds (5 minutes) remaining - 2. **Max age** -- older than 900 seconds (15 minutes) - - The max-age check ensures the underlying OAuth Access Token is revalidated - regularly, catching server-side inactivity timeouts before the DC JWT's own - hard expiry. - -These constants live in `token.rs` (`DC_JWT_VALIDITY_BUFFER_SECS`) and are -passed as parameters to `needs_refresh()` by the caller. +- **DC JWT** (~2-hour hard expiry): the provider refreshes it when `is_valid()` + fails, meaning fewer than `DC_JWT_VALIDITY_BUFFER_SECS` (300 s) remain. The + stricter `needs_refresh(threshold, max_age)` check (expiry threshold *or* age + over 900 s) is applied by `AuthenticatedGrpcClient` in `hyperdb-api-core`, + which owns those constants (`DEFAULT_DC_JWT_EXPIRY_THRESHOLD_SECS`, + `DEFAULT_DC_JWT_MAX_AGE_SECS`). --- ## Testing -Unit tests cover token parsing, validity checks, JWT assertion generation, and -configuration validation: +Unit tests cover token parsing, validity checks, and JWT assertion generation. +`tests/crypto_provider_tests.rs` guards the `rustls-no-provider` choice by +constructing a provider (no network or credentials needed): ```bash cargo test -p hyperdb-api-salesforce diff --git a/hyperdb-api-salesforce/README.md b/hyperdb-api-salesforce/README.md index db56f7eb..9f5d6501 100644 --- a/hyperdb-api-salesforce/README.md +++ b/hyperdb-api-salesforce/README.md @@ -88,8 +88,9 @@ let config = SalesforceAuthConfig::new(login_url, client_id)? `DataCloudTokenProvider` caches both the OAuth Access Token and the DC JWT independently. The OAuth Access Token (~2-hour lifetime) is only refreshed when genuinely expired, avoiding unnecessary refresh token rotation. The DC JWT is -refreshed proactively based on both expiry and age. `get_token()` returns a +refreshed when it is within 5 minutes of expiry. `get_token()` returns a cached token when valid, or transparently performs the full flow when needed. +`AuthenticatedGrpcClient` additionally refreshes it once it is older than 15 minutes. ### SharedTokenProvider @@ -105,6 +106,7 @@ let provider_clone = provider.clone(); tokio::spawn(async move { let token = provider_clone.get_token().await?; // use token.bearer_token() as the Authorization header + Ok::<(), hyperdb_api_salesforce::SalesforceAuthError>(()) }); ``` @@ -233,15 +235,15 @@ cargo run -p hyperdb-api-salesforce --example salesforce_auth_example | Error | Cause and Solution | |-------|-------------------| -| `invalid_grant: user hasn't approved this consumer` | Pre-authorize the user (Step 3 above) | -| `invalid_client_id` | Verify Consumer Key matches exactly | -| `Private key error: failed to parse private key` | Convert to PKCS#8: `openssl pkcs8 -topk8 -nocrypt -in keypair.key -out private.key` | -| `invalid_grant: authentication failure` | Check username, login URL, and certificate match | -| `DC JWT exchange failed` | Verify Data Cloud license and `cdp_query_api` scope | -| `client_secret is required for Password and RefreshToken auth modes` | Add `.client_secret(...)` to your config | +| `authorization failed: invalid_grant - user hasn't approved this consumer` | Pre-authorize the user (Step 3 above) | +| `authorization failed: invalid_client_id - ...` | Verify Consumer Key matches exactly | +| `private key error: failed to parse private key (expected PKCS#8 PEM format): ...` | Convert to PKCS#8: `openssl pkcs8 -topk8 -nocrypt -in keypair.key -out private.key` | +| `authorization failed: invalid_grant - authentication failure` | Check username, login URL, and certificate match | +| `authorization failed: ...` or `HTTP error: ...` after a `DC JWT exchange failed` warning in the logs | Verify Data Cloud license and `cdp_query_api` scope | +| `configuration error: client_secret is required for Password and RefreshToken auth modes` | Add `.client_secret(...)` to your config | --- ## License -Apache-2.0 — see [LICENSE-APACHE](../LICENSE-APACHE). +Licensed under either of [Apache License, Version 2.0](https://github.com/tableau/hyper-api-rust/blob/main/LICENSE-APACHE.txt) or [MIT license](https://github.com/tableau/hyper-api-rust/blob/main/LICENSE-MIT.txt), at your option. diff --git a/hyperdb-api-salesforce/src/lib.rs b/hyperdb-api-salesforce/src/lib.rs index fd9067f3..4190e0a8 100644 --- a/hyperdb-api-salesforce/src/lib.rs +++ b/hyperdb-api-salesforce/src/lib.rs @@ -24,8 +24,9 @@ //! Both the OAuth Access Token and the DC JWT are cached independently. //! The OAuth Access Token is only refreshed when genuinely expired, to avoid //! unnecessary OAuth Refresh Token rotation that would invalidate tokens -//! held by other connections. The DC JWT is refreshed proactively based -//! on both its expiry time and its age (maxAge check). +//! held by other connections. The DC JWT is refreshed when fewer than 5 +//! minutes of its lifetime remain; [`DataCloudToken::needs_refresh`] adds an +//! age check for callers that want earlier refresh. //! //! # Example: JWT Bearer Token Flow //! diff --git a/hyperdb-api-salesforce/src/token.rs b/hyperdb-api-salesforce/src/token.rs index 64eb1430..a5ff46b6 100644 --- a/hyperdb-api-salesforce/src/token.rs +++ b/hyperdb-api-salesforce/src/token.rs @@ -192,10 +192,11 @@ impl DataCloudTokenResponse { /// Sent as the `Authorization: Bearer ` header with every gRPC call /// to the Hyper query engine. /// -/// The DC JWT has a ~2-hour lifetime (`exp` claim), but is proactively -/// refreshed much earlier (every ~15 minutes by default) so that the -/// underlying OAuth Access Token is revalidated before Salesforce's -/// server-side inactivity timeout can invalidate it. +/// The DC JWT has a ~2-hour lifetime (`exp` claim). +/// [`DataCloudTokenProvider`](crate::DataCloudTokenProvider) +/// refreshes it within 5 minutes of expiry; callers that also want age-based +/// refresh, as `AuthenticatedGrpcClient` does every ~15 minutes, use +/// [`Self::needs_refresh`]. #[derive(Debug, Clone)] pub struct DataCloudToken { /// Token type (e.g., "Bearer") diff --git a/hyperdb-api/README.md b/hyperdb-api/README.md index 1c3a1084..51698ce9 100644 --- a/hyperdb-api/README.md +++ b/hyperdb-api/README.md @@ -107,7 +107,7 @@ async fn main() -> Result<()> { let hyper = HyperProcess::new(None, None)?; let endpoint = hyper.require_endpoint()?; - let config = PoolConfig::new(&endpoint, "pooled.hyper") + let config = PoolConfig::new(endpoint, "pooled.hyper") .create_mode(CreateMode::CreateIfNotExists) .max_size(10); @@ -364,7 +364,7 @@ if catalog.has_table("public.users")? { Type-safe SQL identifier handling with automatic escaping: ```rust -use hyperdb_api::{Name, TableName}; +use hyperdb_api::{table_name, Name, TableName}; // Simple construction let name = Name::try_new("users")?; @@ -444,7 +444,7 @@ ASCII `A-Z a-z 0-9 _ . -`; anything else returns `Error::InvalidName`. Thread-safe cancellation from another thread: ```rust -let conn = Arc::new(Connection::create_or_open(&hyper, "test.hyper")?); +let conn = Arc::new(Connection::new(&hyper, "test.hyper", CreateMode::CreateIfNotExists)?); // ... in another thread: conn.cancel()?; // Cancels running query (SQLSTATE 57014) ``` @@ -453,7 +453,7 @@ conn.cancel()?; // Cancels running query (SQLSTATE 57014) ```rust conn.set_notice_receiver(Some(Box::new(|notice| { - println!("Notice: {} ({})", notice.message, notice.severity.as_deref().unwrap_or("")); + println!("Notice: {} ({})", notice.message(), notice.severity().unwrap_or("")); }))); ``` @@ -462,7 +462,7 @@ conn.set_notice_receiver(Some(Box::new(|notice| { Per-query performance metrics from Hyper's internal log: ```rust -use hyperdb_api::query_stats::LogFileStatsProvider; +use hyperdb_api::LogFileStatsProvider; conn.enable_query_stats(LogFileStatsProvider::from_process(&hyper)); conn.execute_command("SELECT * FROM users")?; @@ -517,14 +517,21 @@ window functions, CTEs, complex JOINs, and type-safe query composition. ```toml [dependencies] -sea-query = "0.32" +sea-query = "1.0" sea-query-hyperdb = "1.0" ``` ```rust -use sea_query::{Query, Expr, Iden}; +use sea_query::{Expr, ExprTrait, Iden, Query}; use sea_query_hyperdb::HyperQueryBuilder; +#[derive(Iden)] +enum Users { + Table, + Name, + Age, +} + let sql = Query::select() .column(Users::Name) .from(Users::Table) @@ -559,4 +566,4 @@ crates.io-friendly.) ## License -Apache-2.0 +Licensed under either of [Apache License, Version 2.0](https://github.com/tableau/hyper-api-rust/blob/main/LICENSE-APACHE.txt) or [MIT license](https://github.com/tableau/hyper-api-rust/blob/main/LICENSE-MIT.txt), at your option. diff --git a/hyperdb-api/benches/benchmark.rs b/hyperdb-api/benches/benchmark.rs index 998d7ba6..73365691 100644 --- a/hyperdb-api/benches/benchmark.rs +++ b/hyperdb-api/benches/benchmark.rs @@ -835,7 +835,7 @@ fn tcp_vs_grpc_query(row_count: i64) -> String { TCP_VS_GRPC_QUERY_TEMPLATE.replace("{row_count}", &row_count.to_string()) } -/// Runs a TCP-vs-gRPC query comparison on a synthesized 100M-row 4-column +/// Runs a TCP-vs-gRPC query comparison on a synthesized row_count-row 4-column /// result that matches the `measurements` schema. One Hyper process in /// `ListenMode::Both` serves both transports, and the query has no /// attached-database dependency, so the comparison isolates @@ -843,8 +843,8 @@ fn tcp_vs_grpc_query(row_count: i64) -> String { /// /// Produces three rows: /// - TCP (streaming) -/// - gRPC streaming (the new `Connection::execute_query` → chunk stream path) -/// - gRPC buffered (the existing `execute_query_to_arrow` path — materializes +/// - gRPC streaming (streaming via `Connection::execute_query`) +/// - gRPC buffered (buffered via `execute_query_to_arrow` — materializes /// the whole Arrow IPC payload in client memory before decoding) fn run_tcp_vs_grpc_query_benchmark(row_count: i64, _db_path: &str) -> Result<()> { use hyperdb_api::Parameters; @@ -892,9 +892,7 @@ fn run_tcp_vs_grpc_query_benchmark(row_count: i64, _db_path: &str) -> Result<()> // Use SYNC transfer mode for a fair TCP-vs-gRPC comparison: in SYNC // the server streams the whole result as one server-streaming RPC, // which mirrors TCP's COPY TO STDOUT shape and avoids the - // per-ADAPTIVE-chunk round-trips whose row-count cap ("the server - // stops after one chunk and the client has to ask for more") would - // under-report row counts if the client doesn't poll repeatedly. + // per-ADAPTIVE-chunk round-trips. let transfer_mode = hyperdb_api::grpc::TransferMode::Sync; // Test buffered first to verify the server can send all rows before @@ -1141,9 +1139,8 @@ fn main() -> Result<()> { run_query_benchmarks(&connection, row_count)?; } - // Phase 5b: TCP vs gRPC query comparison on the same populated - // `measurements` table (reuses the single-threaded insert's output, so - // no extra INSERT cost beyond the query work itself). + // Phase 5b: TCP vs gRPC on a generate_series source with the + // `measurements` shape (gRPC cannot attach the local .hyper file). run_tcp_vs_grpc_query_benchmark(row_count, db_path)?; // Print comparison summary diff --git a/hyperdb-api/examples/additional_examples/async_usage.rs b/hyperdb-api/examples/additional_examples/async_usage.rs index 35f13f69..85fe1e24 100644 --- a/hyperdb-api/examples/additional_examples/async_usage.rs +++ b/hyperdb-api/examples/additional_examples/async_usage.rs @@ -11,6 +11,9 @@ //! //! # Running this example //! +//! Requires the pinned `hyperd`: run `make download-hyperd` once, or set an +//! absolute `HYPERD_PATH`. +//! //! ```bash //! cargo run -p hyperdb-api --example async_usage //! ``` @@ -351,8 +354,9 @@ fn print_best_practices() { println!("2. Use spawn_blocking for synchronous Connection"); println!(" Wrap all blocking database calls to avoid blocking the executor.\n"); - println!("3. Create Connections inside spawn_blocking"); - println!(" Connection is not Send/Sync - create it inside the blocking task.\n"); + println!("3. Give each blocking task its own Connection"); + println!(" A Connection runs one statement at a time; concurrent tasks need separate"); + println!(" connections (or a pool::ConnectionPool).\n"); println!("4. Share HyperProcess with Arc"); println!(" HyperProcess can be wrapped in Arc and shared across tasks.\n"); diff --git a/hyperdb-api/examples/additional_examples/threaded_inserter.rs b/hyperdb-api/examples/additional_examples/threaded_inserter.rs index e7d2d15d..00ff442f 100644 --- a/hyperdb-api/examples/additional_examples/threaded_inserter.rs +++ b/hyperdb-api/examples/additional_examples/threaded_inserter.rs @@ -12,7 +12,10 @@ //! The pattern separates data encoding (CPU-bound) from network I/O, allowing //! multiple cores to prepare data while a single thread handles transmission. //! -//! Run with: cargo run -p hyperdb-api --example `threaded_inserter` +//! Run with: `cargo run -p hyperdb-api --example threaded_inserter -- [--workers N] [--rows N] [--chunk-size N]` +//! +//! Requires the pinned `hyperd`: run `make download-hyperd` once, or set an +//! absolute `HYPERD_PATH`. #![allow( clippy::cast_precision_loss, diff --git a/hyperdb-api/examples/additional_examples/transactions.rs b/hyperdb-api/examples/additional_examples/transactions.rs index f4a2cf65..2fef1d8a 100644 --- a/hyperdb-api/examples/additional_examples/transactions.rs +++ b/hyperdb-api/examples/additional_examples/transactions.rs @@ -11,20 +11,16 @@ //! - Multi-table reconnect semantics: only committed cross-table data is visible after reconnect //! - Auto-rollback safety net when the guard is dropped without commit //! - DDL inside transactions and known restrictions -//! - Legacy raw transaction methods (`begin_transaction`, `commit`, -//! `rollback`) — included for completeness only; deprecated and -//! slated for removal. New code should use the RAII guard. +//! - Unguarded transaction control (`begin_transaction_unguarded`, +//! `commit_unguarded`, `rollback_unguarded`) for code that cannot take +//! the guard's `&mut` borrow, such as a helper holding `&Connection`. +//! Prefer the RAII guard everywhere else: you must pair every unguarded +//! `BEGIN` with a commit or rollback yourself, on every path. //! -//! cargo run -p hyperdb-api --example transactions - -// The `example_raw_transaction` and `example_raw_error_recovery` -// helpers below intentionally exercise the deprecated raw transaction -// API for documentation purposes. New code should use the RAII guard -// shown in `example_transaction_guard` instead. -#![allow( - deprecated, - reason = "example intentionally demonstrates the deprecated raw transaction API alongside the RAII guard" -)] +//! Run with: `cargo run -p hyperdb-api --example transactions` +//! +//! Requires the pinned `hyperd`: run `make download-hyperd` once, or set an +//! absolute `HYPERD_PATH`. use hyperdb_api::{ Catalog, Connection, CreateMode, HyperProcess, Parameters, Result, SqlType, TableDefinition, @@ -42,7 +38,7 @@ fn main() -> Result<()> { let mut connection = Connection::new(&hyper, db_path, CreateMode::CreateAndReplace)?; println!("Created database: {db_path}\n"); - // Example 1: Raw transaction methods + // Example 1: Unguarded transaction control example_raw_transaction(&connection)?; // Example 2: RAII Transaction guard @@ -58,16 +54,13 @@ fn main() -> Result<()> { example_multi_table_rollback(&mut connection)?; // Example 6: Multi-table reconnect semantics - example_multi_table_reconnect(&hyper, db_path)?; - // Reconnect after the previous example closed the connection - let mut connection = Connection::new(&hyper, db_path, CreateMode::DoNotCreate)?; + example_multi_table_reconnect(&hyper)?; // Example 7: Auto-rollback on drop (the safety net) example_auto_rollback_on_drop(&mut connection)?; // Example 8: DDL in transactions and known restrictions - // (Last because the DDL-after-DML error leaves the connection in a state - // where subsequent queries may not work reliably.) + // (Last because it deliberately triggers a server error inside a transaction.) example_ddl_in_transactions(&mut connection)?; println!("\nAll transaction examples completed successfully!"); @@ -108,10 +101,10 @@ fn print_table(connection: &Connection, table_name: &str) -> Result<()> { } // --------------------------------------------------------------------------- -// Example 1: Raw transaction methods (begin / commit / rollback) +// Example 1: Unguarded transaction control (&self helpers) // --------------------------------------------------------------------------- fn example_raw_transaction(connection: &Connection) -> Result<()> { - println!("=== Example 1: Raw Transaction Methods ==="); + println!("=== Example 1: Unguarded Transaction Control ==="); create_accounts_table(connection, "raw_txn")?; @@ -332,7 +325,7 @@ fn example_multi_table_rollback(connection: &mut Connection) -> Result<()> { // connection. This demonstrates the atomicity/isolation guarantees across // reconnects on the same `HyperProcess`. // --------------------------------------------------------------------------- -fn example_multi_table_reconnect(hyper: &HyperProcess, _db_path: &str) -> Result<()> { +fn example_multi_table_reconnect(hyper: &HyperProcess) -> Result<()> { println!("=== Example 6: Multi-Table Reconnect Semantics ==="); // Use a fresh database for this example diff --git a/hyperdb-api/examples/create_hyper_file_from_csv.rs b/hyperdb-api/examples/create_hyper_file_from_csv.rs index bee6c01a..b5686f7b 100644 --- a/hyperdb-api/examples/create_hyper_file_from_csv.rs +++ b/hyperdb-api/examples/create_hyper_file_from_csv.rs @@ -71,8 +71,8 @@ fn run_create_hyper_file_from_csv() -> Result<()> { // Also be aware that the `header` option is used in this example: // It treats the first line of the csv file as a header and does not import it. // - // The parameters of the COPY command are documented in the Tableau Hyper SQL documentation - // (https://tableau.github.io/hyper-db/docs/sql/command/copy_from). + // The parameters of the COPY command are documented in the Data Cloud SQL reference (the SQL dialect Hyper uses) + // (https://developer.salesforce.com/docs/data/data-cloud-query-guide/references/dc-sql-reference/copy-from.html). println!( "Issuing the SQL COPY command to load the csv file into the table. Since the first line" ); diff --git a/hyperdb-api/src/arrow_inserter.rs b/hyperdb-api/src/arrow_inserter.rs index fccc4035..7b6d9634 100644 --- a/hyperdb-api/src/arrow_inserter.rs +++ b/hyperdb-api/src/arrow_inserter.rs @@ -116,7 +116,7 @@ pub struct ArrowInserter<'conn> { chunk_count: usize, /// Start time for timing the insert operation. start_time: Instant, - /// Flush threshold in bytes. Data is buffered until this threshold is reached. + /// Flush threshold in bytes. Controls how often an explicit flush happens. flush_threshold: usize, /// Bytes buffered since the last flush. buffered_bytes: usize, @@ -240,7 +240,8 @@ impl<'conn> ArrowInserter<'conn> { /// Sets a custom flush threshold in bytes. /// - /// Data is buffered until the threshold is reached, then flushed to the server. + /// Data is written to the socket as each chunk is inserted; the threshold + /// only controls how often an explicit flush and progress log happen. /// Default is 16 MB (matching `HyperBinary` Inserter). /// /// # Example @@ -441,8 +442,8 @@ impl<'conn> ArrowInserter<'conn> { /// raw IPC and `RecordBatch` paths cannot be mixed. /// - Returns [`Error::FeatureNotSupported`] / [`Error::Server`] if the lazy COPY /// session fails to open. - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub fn insert_raw(&mut self, data: &[u8]) -> Result<()> { if data.is_empty() { return Ok(()); @@ -493,7 +494,6 @@ impl<'conn> ArrowInserter<'conn> { /// # Ok(()) /// # } /// ``` - /// println!("Inserted {} rows", rows); pub fn execute(mut self) -> Result { // Finalize the IPC StreamWriter if insert_batch() was used. // `into_inner()` calls `finish()` internally (writing the EOS marker) @@ -513,17 +513,12 @@ impl<'conn> ArrowInserter<'conn> { } } - if self.writer.is_none() { + let Some(writer) = self.writer.take() else { // No data was sent return Ok(0); - } + }; - let rows = self - .writer - .take() - .map(hyperdb_api_core::client::CopyInWriter::finish) - .transpose()? - .unwrap_or(0); + let rows = writer.finish()?; let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX); info!( @@ -619,15 +614,8 @@ impl<'conn> ArrowInserter<'conn> { /// - Returns [`Error::Conversion`] wrapping the underlying Arrow IPC /// writer error if the schema or batch cannot be serialized (e.g. /// dictionary misalignment, encoding failure). - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. - /// - /// # Panics - /// - /// Panics internally only if the IPC writer state is corrupted — - /// callers cannot trigger this from the public API. The `batch_ipc_writer` - /// is consulted via `as_mut().unwrap()` after it has just been set to - /// `Some`, so the unwrap is unreachable. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub fn insert_batch(&mut self, batch: &arrow::record_batch::RecordBatch) -> Result<()> { if self.insert_mode == Some(InsertMode::RawIpc) { return Err(Error::invalid_operation( @@ -655,7 +643,7 @@ impl<'conn> ArrowInserter<'conn> { // Write the record batch — this appends batch IPC bytes to the internal Vec self.batch_ipc_writer .as_mut() - .expect("IPC writer must exist") + .ok_or_else(|| Error::internal("IPC writer must exist"))? .write(batch) .map_err(|e| Error::conversion(format!("Failed to write Arrow batch: {e}")))?; diff --git a/hyperdb-api/src/arrow_reader.rs b/hyperdb-api/src/arrow_reader.rs index 4053c9d3..c21718f3 100644 --- a/hyperdb-api/src/arrow_reader.rs +++ b/hyperdb-api/src/arrow_reader.rs @@ -107,7 +107,7 @@ impl<'conn> ArrowReader<'conn> { /// transport (ArrowReader wraps `COPY TO STDOUT`, which is TCP-only). /// - Returns [`crate::Error::Server`] if the server rejects the /// `COPY () TO STDOUT WITH (format arrowstream)` statement. - /// - Returns [`crate::Error::Io`] on transport-level I/O failures. + /// - Returns [`crate::Error::Connection`] on transport-level I/O failures. pub fn query_to_arrow(&self, select_query: &str) -> Result> { let copy_query = format!("COPY ({select_query}) TO STDOUT WITH (format arrowstream)"); let client = self.connection.tcp_client().ok_or_else(|| { diff --git a/hyperdb-api/src/arrow_result.rs b/hyperdb-api/src/arrow_result.rs index f3900551..73ecccff 100644 --- a/hyperdb-api/src/arrow_result.rs +++ b/hyperdb-api/src/arrow_result.rs @@ -723,11 +723,11 @@ impl ArrowRowset { } } - /// Returns the total number of rows across all batches. + /// Returns the number of rows in the batches the rowset currently holds. /// - /// For streaming rowsets this reflects only batches decoded **so far** — - /// until [`next_chunk`](Self::next_chunk) has pulled everything from the - /// source, the total is not yet known. + /// For buffered rowsets this is the total row count of the result. For + /// streaming rowsets it counts only batches decoded but not yet returned + /// by [`next_chunk`](Self::next_chunk), so it is not the result total. #[must_use] pub fn total_rows(&self) -> usize { match &self.inner { @@ -742,12 +742,12 @@ impl ArrowRowset { } } - /// Returns true if there are no rows available **right now**. + /// Returns `true` if the rowset has no rows. /// - /// For streaming rowsets this only reflects the currently-decoded - /// batches, not the full result — a streaming rowset that has not been - /// iterated will usually report `is_empty() == true` even if the server - /// will send more data on `next_chunk`. + /// For buffered rowsets this describes the whole result. For streaming + /// rowsets it returns `true` only after the source is exhausted and every + /// decoded batch has been returned by [`next_chunk`](Self::next_chunk); + /// until then it returns `false`, even if the result turns out to be empty. #[must_use] pub fn is_empty(&self) -> bool { match &self.inner { diff --git a/hyperdb-api/src/async_arrow_inserter.rs b/hyperdb-api/src/async_arrow_inserter.rs index 49711d43..6646035a 100644 --- a/hyperdb-api/src/async_arrow_inserter.rs +++ b/hyperdb-api/src/async_arrow_inserter.rs @@ -69,7 +69,7 @@ pub struct AsyncArrowInserter<'conn> { chunk_count: usize, /// Start time for timing the insert operation. start_time: Instant, - /// Flush threshold in bytes. Data is buffered until this threshold is reached. + /// Flush threshold in bytes. Controls how often an explicit flush happens. flush_threshold: usize, /// Bytes buffered since the last flush. buffered_bytes: usize, @@ -130,7 +130,8 @@ impl<'conn> AsyncArrowInserter<'conn> { /// Sets a custom flush threshold in bytes. /// - /// Data is buffered until the threshold is reached, then flushed to the server. + /// Data is written to the socket as each chunk is inserted; the threshold + /// only controls how often an explicit flush and progress log happen. /// Default is 16 MB (matching `HyperBinary` Inserter). #[must_use] pub fn with_flush_threshold(mut self, threshold: usize) -> Self { @@ -151,8 +152,8 @@ impl<'conn> AsyncArrowInserter<'conn> { /// subsequent chunks instead). /// - Returns [`Error::FeatureNotSupported`] / [`Error::Server`] if the lazy COPY /// session cannot be opened. - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub async fn insert_data(&mut self, arrow_ipc_data: &[u8]) -> Result<()> { if arrow_ipc_data.is_empty() { return Ok(()); @@ -198,8 +199,8 @@ impl<'conn> AsyncArrowInserter<'conn> { /// /// - Returns [`Error::Internal`] if no schema has been sent yet (call /// [`insert_data`](Self::insert_data) first). - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub async fn insert_record_batches(&mut self, arrow_batch_data: &[u8]) -> Result<()> { if arrow_batch_data.is_empty() { return Ok(()); @@ -243,8 +244,8 @@ impl<'conn> AsyncArrowInserter<'conn> { /// /// - Returns [`Error::FeatureNotSupported`] / [`Error::Server`] if the lazy COPY /// session cannot be opened. - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub async fn insert_raw(&mut self, data: &[u8]) -> Result<()> { if data.is_empty() { return Ok(()); @@ -272,19 +273,16 @@ impl<'conn> AsyncArrowInserter<'conn> { /// /// # Errors /// - /// Returns [`Error::Server`] or [`Error::Io`] if the `CommandComplete` + /// Returns [`Error::Server`] or [`Error::Connection`] if the `CommandComplete` /// round-trip fails (server rejected some buffered batch, or the socket /// closed mid-flush). If no data was ever written, returns `Ok(0)`. pub async fn execute(mut self) -> Result { - if self.writer.is_none() { + let Some(writer) = self.writer.take() else { return Ok(0); - } - - let rows = match self.writer.take() { - Some(w) => w.finish().await?, - None => 0, }; + let rows = writer.finish().await?; + let duration_ms = u64::try_from(self.start_time.elapsed().as_millis()).unwrap_or(u64::MAX); info!( target: "hyperdb_api", @@ -490,8 +488,8 @@ impl AsyncArrowInserterOwned { /// - Returns [`Error::Internal`] if a schema was already sent. /// - Returns [`Error::FeatureNotSupported`] / [`Error::Server`] if the lazy COPY /// session cannot be opened. - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub async fn insert_data(&mut self, arrow_ipc_data: &[u8]) -> Result<()> { if arrow_ipc_data.is_empty() { return Ok(()); @@ -520,8 +518,8 @@ impl AsyncArrowInserterOwned { /// # Errors /// /// - Returns [`Error::Internal`] if no schema has been sent yet. - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub async fn insert_record_batches(&mut self, arrow_batch_data: &[u8]) -> Result<()> { if arrow_batch_data.is_empty() { return Ok(()); @@ -549,8 +547,8 @@ impl AsyncArrowInserterOwned { /// /// - Returns [`Error::FeatureNotSupported`] / [`Error::Server`] if the lazy COPY /// session cannot be opened. - /// - Returns [`Error::Server`] / [`Error::Io`] if the server rejects - /// the data or the socket write fails. + /// - Returns [`Error::Server`] if the server rejects the data, or + /// [`Error::Connection`] if the socket write fails. pub async fn insert_raw(&mut self, data: &[u8]) -> Result<()> { if data.is_empty() { return Ok(()); @@ -574,7 +572,7 @@ impl AsyncArrowInserterOwned { /// - Returns [`Error::Internal`] with message /// `"No data was inserted before execute()"` if no COPY session was /// ever opened. - /// - Returns [`Error::Server`] / [`Error::Io`] if the `CommandComplete` + /// - Returns [`Error::Server`] / [`Error::Connection`] if the `CommandComplete` /// round-trip fails. pub async fn execute(mut self) -> Result { let elapsed = self.start_time.elapsed(); diff --git a/hyperdb-api/src/async_connection_builder.rs b/hyperdb-api/src/async_connection_builder.rs index dc4e9049..759d2468 100644 --- a/hyperdb-api/src/async_connection_builder.rs +++ b/hyperdb-api/src/async_connection_builder.rs @@ -75,8 +75,8 @@ impl AsyncConnectionBuilder { } } - #[must_use] /// Sets the database path. + #[must_use] pub fn database(mut self, path: impl AsRef) -> Self { self.database = Some(path.as_ref().to_path_buf()); self @@ -89,15 +89,15 @@ impl AsyncConnectionBuilder { self } - #[must_use] /// Sets the username for authentication. + #[must_use] pub fn user(mut self, user: impl Into) -> Self { self.user = Some(user.into()); self } - #[must_use] /// Sets the password for authentication. + #[must_use] pub fn password(mut self, password: impl Into) -> Self { self.password = Some(password.into()); self @@ -111,45 +111,48 @@ impl AsyncConnectionBuilder { } /// Sets the query timeout. + /// + /// The value is recorded on the builder but no transport applies it, so + /// queries run until completion regardless of this setting. #[must_use] pub fn query_timeout(mut self, timeout: Duration) -> Self { self.query_timeout = Some(timeout); self } - #[must_use] /// Sets the application name sent to the server. + #[must_use] pub fn application_name(mut self, name: impl Into) -> Self { self.application_name = Some(name.into()); self } - #[must_use] /// Convenience method to set user and password at once. + #[must_use] pub fn auth(mut self, user: impl Into, password: impl Into) -> Self { self.user = Some(user.into()); self.password = Some(password.into()); self } - #[must_use] /// Convenience method to create a new database. + #[must_use] pub fn create_new_database(mut self, database_path: impl AsRef) -> Self { self.database = Some(database_path.as_ref().to_path_buf()); self.create_mode = CreateMode::Create; self } - #[must_use] /// Convenience method to create database if it doesn't exist. + #[must_use] pub fn create_or_open_database(mut self, database_path: impl AsRef) -> Self { self.database = Some(database_path.as_ref().to_path_buf()); self.create_mode = CreateMode::CreateIfNotExists; self } - #[must_use] /// Convenience method to open an existing database. + #[must_use] pub fn open_database(mut self, database_path: impl AsRef) -> Self { self.database = Some(database_path.as_ref().to_path_buf()); self.create_mode = CreateMode::DoNotCreate; @@ -165,10 +168,11 @@ impl AsyncConnectionBuilder { /// Builds and establishes the connection (async). /// - /// Transport is auto-detected from the endpoint URL: + /// Transport is auto-detected from the endpoint, with the same rules as + /// [`ConnectionBuilder`](crate::ConnectionBuilder): /// - `https://` / `http://` → gRPC - /// - `tab.domain://` → Unix domain socket (Unix only) - /// - `tab.pipe://` → named pipe (Windows only) + /// - `tab.domain://` or an absolute socket path → Unix domain socket (Unix only) + /// - `tab.pipe://` or a `\\host\pipe\name` path → named pipe (Windows only) /// - otherwise → TCP /// /// # Errors diff --git a/hyperdb-api/src/async_transaction.rs b/hyperdb-api/src/async_transaction.rs index 90a1abc5..966539c5 100644 --- a/hyperdb-api/src/async_transaction.rs +++ b/hyperdb-api/src/async_transaction.rs @@ -37,10 +37,9 @@ pub struct AsyncTransaction<'conn> { impl<'conn> AsyncTransaction<'conn> { /// Creates a new async transaction by issuing `BEGIN TRANSACTION`. pub(crate) async fn new(connection: &'conn mut AsyncConnection) -> Result { - // Use the crate-internal `_raw` family. The matching `pub` - // methods on `AsyncConnection` are `#[deprecated]` for - // downstream consumers; this guard is the recommended - // replacement. + // Delegate to the public `*_unguarded` primitive; commit/rollback + // consume `self`. Drop cannot roll back (no async Drop), so it only + // warns. connection.begin_transaction_unguarded().await?; Ok(Self { connection, diff --git a/hyperdb-api/src/catalog.rs b/hyperdb-api/src/catalog.rs index f22ed20b..b363d347 100644 --- a/hyperdb-api/src/catalog.rs +++ b/hyperdb-api/src/catalog.rs @@ -905,9 +905,9 @@ impl<'conn> Catalog<'conn> { // Metadata Helpers // ============================================================ - /// Returns the approximate row count for a table. + /// Returns the row count of a table. /// - /// This executes `SELECT COUNT(*) FROM table_name`. + /// This executes `SELECT COUNT(*) FROM table_name`, so the count is exact. /// /// # Example /// diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index 41112827..c951c68c 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -4,11 +4,13 @@ //! Database connection management. //! //! The [`Connection`] type provides a unified interface for connecting to Hyper -//! databases via either TCP (`PostgreSQL` wire protocol) or gRPC transport. +//! databases via the `PostgreSQL` wire protocol (TCP, Unix domain socket, or named pipe) or gRPC transport. //! The transport is automatically detected from the endpoint URL: //! -//! - `https://` or `http://` → gRPC transport -//! - Otherwise → TCP transport (e.g., `localhost:7483`) +//! - `https://` or `http://` → gRPC (read-only) +//! - `tab.domain://...` or an absolute socket path → Unix domain socket (Unix only) +//! - `tab.pipe://...` or a `\\host\pipe\name` path → named pipe (Windows only) +//! - otherwise `host:port` → TCP (e.g., `localhost:7483`) use std::path::Path; @@ -107,8 +109,10 @@ pub enum CreateMode { /// # Transport Auto-Detection /// /// The transport is automatically detected from the endpoint URL: -/// - `https://` or `http://` → gRPC transport (read-only until server supports writes) -/// - Otherwise → TCP transport (full read/write support) +/// - `https://` or `http://` → gRPC (read-only) +/// - `tab.domain://...` or an absolute socket path → Unix domain socket (Unix only) +/// - `tab.pipe://...` or a `\\host\pipe\name` path → named pipe (Windows only) +/// - otherwise `host:port` → TCP; all wire-protocol transports have full read/write support /// /// # CSV / Text Import & Export /// @@ -237,7 +241,7 @@ impl Connection { /// /// # Arguments /// - /// * `endpoint` - The server endpoint (host:port). + /// * `endpoint` - The server endpoint; see [`ConnectionBuilder`](crate::ConnectionBuilder) for the accepted forms. /// * `database_path` - Path to the database file. /// * `create_mode` - How to handle database creation. /// @@ -291,8 +295,7 @@ impl Connection { /// Returns true if this connection supports write operations. /// - /// Currently, only TCP connections support writes. gRPC connections are - /// read-only until the server supports write operations over gRPC. + /// Only TCP connections support writes; gRPC connections are read-only. pub fn supports_writes(&self) -> bool { self.transport.supports_writes() } @@ -361,7 +364,7 @@ impl Connection { /// /// # Arguments /// - /// * `endpoint` - The server endpoint (host:port). + /// * `endpoint` - The server endpoint; see [`ConnectionBuilder`](crate::ConnectionBuilder) for the accepted forms. /// * `database_path` - Path to the database file. /// * `create_mode` - How to handle database creation. /// * `user` - Username for authentication. @@ -411,7 +414,7 @@ impl Connection { /// # Errors /// /// Returns an error if: - /// - The connection is using gRPC transport (write operations not yet supported) + /// - The connection is using gRPC transport (write operations are unsupported over gRPC) /// - The command fails to execute pub fn execute_command(&self, command: &str) -> Result { let token = self.stats_before_query(command); @@ -427,8 +430,8 @@ impl Connection { /// Executes a SQL query and returns a streaming result set. /// - /// Results are streamed in chunks (default 64K rows), keeping memory usage - /// constant regardless of result set size. This makes it safe for any + /// Results are streamed in chunks (up to 65,536 rows over TCP), keeping memory + /// usage constant regardless of result set size. This makes it safe for any /// result size, from a single row to billions of rows. /// /// # Example @@ -454,7 +457,7 @@ impl Connection { /// /// # Memory Behavior /// - /// - Only one chunk is held in memory at a time (~few MB for 64K rows) + /// - Only one chunk is held in memory at a time (~few MB for a 64K-row TCP chunk) /// - Safe for result sets of any size (millions/billions of rows) /// - Memory usage is `O(chunk_size)`, not `O(total_rows)` /// @@ -463,7 +466,11 @@ impl Connection { /// - Returns [`Error::Server`] wrapping a `hyperdb_api_core::client::Error` if the /// SQL fails to parse, execute, or if the server reports an error /// while streaming. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] if reading from or writing to the + /// transport fails, or [`Error::Closed`] if the server closes the + /// connection. + /// - Over gRPC, returns [`Error::Cancelled`] or [`Error::Timeout`] if the + /// call is cancelled or its deadline expires. pub fn execute_query(&self, query: &str) -> Result> { let token = self.stats_before_query(query); @@ -476,9 +483,9 @@ impl Connection { } Transport::Grpc(grpc) => { // gRPC streaming: pull chunks lazily so peak memory is - // bounded by one gRPC message (tonic default 64 MB), not - // by the full result size. Matches TCP's - // constant-memory streaming shape. + // bounded by one gRPC message (the `GrpcConfig` max message + // size, 64 MB by default), not by the full result size. + // Matches TCP's constant-memory streaming shape. // // The transport module already creates a fresh gRPC client // per query (gRPC client needs &mut self to execute), so we @@ -652,7 +659,7 @@ impl Connection { /// let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; /// if let Some(row) = conn.fetch_optional("SELECT * FROM users WHERE id = 999")? { /// let name: Option = row.get(1); - /// println!("Found user: {:?}", name); + /// println!("Found user: {name:?}"); /// } /// Ok(()) /// } @@ -685,7 +692,7 @@ impl Connection { /// for row in rows { /// let id: Option = row.get(0); /// let name: Option = row.get(1); - /// println!("User {}: {:?}", id.unwrap_or(-1), name); + /// println!("User {}: {name:?}", id.unwrap_or(-1)); /// } /// Ok(()) /// } @@ -786,11 +793,10 @@ impl Connection { .collect() } - /// Returns a lazy iterator over rows, mapping each to `T` via - /// [`FromRow`]. + /// Returns a lazy iterator mapping each row to `T` via [`FromRow`]. /// /// This is the streaming variant of [`fetch_all_as`](Self::fetch_all_as): - /// memory usage is bounded by the chunk size (default 64K rows), not by + /// memory usage is bounded by the chunk size (up to 65,536 rows over TCP), not by /// the total row count. Use this for large result sets where collecting /// all rows into a `Vec` would exceed memory limits. /// @@ -847,15 +853,14 @@ impl Connection { Ok(crate::result::TypedRowIterator::::new(rowset)) } - /// Fetches a single row from a **parameterized** query and maps it to a - /// struct using [`FromRow`](crate::FromRow). + /// Fetches one row of a **parameterized** query as a [`FromRow`](crate::FromRow) struct. /// /// This is the parameterized counterpart to /// [`fetch_one_as`](Self::fetch_one_as): it binds `$1`, `$2`, … placeholders /// from `params` (via [`ToSqlParam`](crate::params::ToSqlParam), exactly as /// [`query_params`](Self::query_params) does) and maps the first result row /// into `T`. Use it when a parameterized `SELECT` should yield a typed - /// struct rather than a raw [`Row`](crate::Row). + /// struct rather than a raw [`Row`]. /// /// # Example /// @@ -900,8 +905,7 @@ impl Connection { T::from_row(crate::RowAccessor::new(&row, &indices)) } - /// Fetches all rows from a **parameterized** query and maps them to structs - /// using [`FromRow`](crate::FromRow). + /// Fetches all rows of a **parameterized** query as [`FromRow`](crate::FromRow) structs. /// /// This is the parameterized counterpart to /// [`fetch_all_as`](Self::fetch_all_as): it binds `$1`, `$2`, … placeholders @@ -953,8 +957,7 @@ impl Connection { .collect() } - /// Returns a lazy iterator over the rows of a **parameterized** query, - /// mapping each to `T` via [`FromRow`]. + /// Returns a lazy iterator over a **parameterized** query, mapping rows via [`FromRow`]. /// /// This is the parameterized counterpart to /// [`stream_as`](Self::stream_as): it binds `$1`, `$2`, … placeholders from @@ -1027,7 +1030,7 @@ impl Connection { /// fn main() -> Result<()> { /// let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; /// let count: i64 = conn.fetch_scalar("SELECT COUNT(*) FROM users")?; - /// println!("User count: {}", count); + /// println!("User count: {count}"); /// Ok(()) /// } /// ``` @@ -1062,7 +1065,7 @@ impl Connection { /// fn main() -> Result<()> { /// let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; /// let max_id: Option = conn.fetch_optional_scalar("SELECT MAX(id) FROM users")?; - /// println!("Max ID: {:?}", max_id); + /// println!("Max ID: {max_id:?}"); /// Ok(()) /// } /// ``` @@ -1113,7 +1116,7 @@ impl Connection { /// fn main() -> Result<()> { /// let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; /// let count = conn.query_count("SELECT COUNT(*) FROM users WHERE active = true")?; - /// println!("Active users: {}", count); + /// println!("Active users: {count}"); /// Ok(()) /// } /// ``` @@ -1151,7 +1154,8 @@ impl Connection { /// the Parse round-trip on every call. Note that a prepared statement /// fixes its parameter OIDs up front, so it cannot accept both whole and /// scaled `NUMERIC` values; `query_params` re-parses per call and can. - /// See [`Numeric::sql_oid`](crate::ToSqlParam::sql_oid) for the detail. + /// See the [`ToSqlParam`](crate::ToSqlParam) implementation for + /// [`Numeric`](crate::Numeric) for the detail. /// /// Under the hood, `query_params` is a one-shot /// prepare+execute+close: it prepares an unnamed statement, binds @@ -1182,7 +1186,7 @@ impl Connection { /// for row in &chunk { /// let id: Option = row.get(0); /// let name: Option = row.get(1); - /// println!("Found: {:?} - {:?}", id, name); + /// println!("Found: {id:?} - {name:?}"); /// } /// } /// Ok(()) @@ -1213,7 +1217,9 @@ impl Connection { /// - Returns [`Error::Server`] if the server rejects the statement at /// `Parse`, `Bind`, or `Execute` time, including on type-mismatch /// between `params` and the inferred OIDs. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] if reading from or writing to the + /// transport fails, or [`Error::Closed`] if the server closes the + /// connection. pub fn query_params( &self, query: &str, @@ -1271,7 +1277,9 @@ impl Connection { /// - Returns [`Error::FeatureNotSupported`] if the connection is using gRPC transport. /// - Returns [`Error::Server`] if the server rejects the statement at /// `Parse`, `Bind`, or `Execute` time. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] if reading from or writing to the + /// transport fails, or [`Error::Closed`] if the server closes the + /// connection. pub fn command_params( &self, query: &str, @@ -1314,7 +1322,7 @@ impl Connection { /// "INSERT INTO users VALUES (1, 'Alice')", /// "INSERT INTO users VALUES (2, 'Bob')", /// ])?; - /// println!("Total affected: {}", total); + /// println!("Total affected: {total}"); /// Ok(()) /// } /// ``` @@ -1590,7 +1598,7 @@ impl Connection { /// let hyper = HyperProcess::new(None, None)?; /// let conn = Connection::new(&hyper, "test.hyper", CreateMode::CreateIfNotExists)?; /// if let Some(version) = conn.server_version() { - /// println!("Hyper {}", version); + /// println!("Hyper {version}"); /// if version >= ServerVersion::new(0, 1, 0) { /// println!("Has feature X"); /// } @@ -1645,7 +1653,7 @@ impl Connection { /// fn main() -> Result<()> { /// let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; /// let plan = conn.explain("SELECT * FROM users WHERE id = 1")?; - /// println!("{}", plan); + /// println!("{plan}"); /// Ok(()) /// } /// ``` @@ -1688,11 +1696,9 @@ impl Connection { Ok(lines.join("\n")) } - /// Returns a reference to the underlying TCP client. + /// Returns the underlying wire-protocol client, or `None` for gRPC connections. /// - /// # Panics - /// - /// This method returns `None` if the connection is using gRPC transport. + /// TCP, Unix-socket and named-pipe connections all return `Some`. pub fn tcp_client(&self) -> Option<&Client> { match &self.transport { Transport::Tcp(tcp) => Some(&tcp.client), @@ -1707,26 +1713,25 @@ impl Connection { &self.transport } - /// Prepares a SQL statement with automatic parameter type inference. + /// Prepares a SQL statement that takes no parameters. /// /// The returned [`PreparedStatement`](crate::PreparedStatement) can - /// be executed many times with different parameter values; the - /// server caches the parsed plan. This is the preferred way to - /// execute a statement repeatedly inside a loop. + /// be executed many times; the server caches the parsed plan. This + /// is the preferred way to execute a statement repeatedly inside a + /// loop. /// - /// For explicit parameter types (necessary when `$N` placeholders - /// would otherwise be ambiguous), use - /// [`prepare_typed`](Self::prepare_typed). + /// Hyper does not infer parameter types, so statements with `$N` + /// placeholders must use [`prepare_typed`](Self::prepare_typed). /// /// # Example /// /// ```no_run /// # use hyperdb_api::{Connection, CreateMode, Result}; /// # fn example(conn: &Connection) -> Result<()> { - /// let stmt = conn.prepare("SELECT name FROM users WHERE id = $1")?; - /// for id in [1_i32, 2, 3] { - /// let name: String = stmt.fetch_scalar(&[&id])?; - /// println!("{id}: {name}"); + /// let stmt = conn.prepare("SELECT count(*) FROM users")?; + /// for _ in 0..3 { + /// let n: i64 = stmt.fetch_scalar(&[])?; + /// println!("{n} users"); /// } /// # Ok(()) /// # } @@ -1742,9 +1747,8 @@ impl Connection { /// Prepares a SQL statement with explicit parameter type OIDs. /// - /// Use this when the server cannot infer parameter types from the - /// SQL alone (e.g. a bare `$1` in a `WHERE v > $1` clause with no - /// other context). Constants for common types live in + /// Declares the type of each `$N` placeholder up front (e.g. for a + /// `WHERE v > $1` clause). Constants for common types live in /// [`hyperdb_api_core::types::oids`]. /// /// In practice this is the *only* way to prepare a statement that binds @@ -1770,7 +1774,9 @@ impl Connection { /// (prepared statements are TCP-only). /// - Returns [`Error::Server`] if the server rejects the `Parse` /// message, e.g. SQL syntax error or unknown OID. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] if reading from or writing to the + /// transport fails, or [`Error::Closed`] if the server closes the + /// connection. pub fn prepare_typed( &self, query: &str, @@ -1869,7 +1875,7 @@ impl Connection { /// let conn = Connection::new(&hyper, "test.hyper", CreateMode::CreateIfNotExists)?; /// /// if let Some(version) = conn.parameter_status("server_version") { - /// println!("Connected to Hyper version: {}", version); + /// println!("Connected to Hyper version: {version}"); /// } /// Ok(()) /// } @@ -1899,8 +1905,8 @@ impl Connection { /// /// # Errors /// - /// - Returns [`Error::FeatureNotSupported`] on gRPC connections — cancellation is not - /// yet implemented for gRPC transport. + /// - Returns [`Error::FeatureNotSupported`] on gRPC connections, which do not + /// support query cancellation. /// - Returns [`Error::Connection`] or [`Error::Io`] if the separate /// cancel-request connection to the server fails. pub fn cancel(&self) -> Result<()> { @@ -1973,17 +1979,17 @@ impl Connection { /// fn main() -> Result<()> { /// let hyper = HyperProcess::new(None, None)?; /// let conn = Connection::new(&hyper, "test.hyper", CreateMode::Create)?; - /// + /// /// // Do some work with the database /// conn.execute_command("CREATE TABLE test (id INT)")?; - /// + /// /// // Unload from memory (but keep connection) /// conn.unload_database()?; - /// + /// /// // Database can still be accessed (will be reloaded automatically) /// let count: i64 = conn.fetch_scalar("SELECT COUNT(*) FROM test")?; - /// println!("Count: {}", count); - /// + /// println!("Count: {count}"); + /// /// Ok(()) /// } /// ``` @@ -2018,16 +2024,16 @@ impl Connection { /// fn main() -> Result<()> { /// let hyper = HyperProcess::new(None, None)?; /// let conn = Connection::new(&hyper, "test.hyper", CreateMode::Create)?; - /// + /// /// // Do some work with the database /// conn.execute_command("CREATE TABLE test (id INT)")?; - /// + /// /// // Release database completely from session /// conn.unload_release()?; - /// + /// /// // Database cannot be accessed after this point without new connection /// // conn.execute_command("SELECT * FROM test")?; // This would fail - /// + /// /// Ok(()) /// } /// ``` @@ -2054,7 +2060,7 @@ impl Connection { /// /// The provider determines how stats are collected. Use /// [`LogFileStatsProvider`](crate::LogFileStatsProvider) to parse Hyper's log file (requires local - /// `hyperd.log`), or implement a custom [`QueryStatsProvider`](crate::QueryStatsProvider). + /// `hyperd.log`), or implement a custom [`QueryStatsProvider`]. /// /// # Example /// @@ -2164,9 +2170,7 @@ impl Connection { // (rather than the guard's `&mut self`) delegate to these. // // They are public because a `&self` helper cannot use the guard at all, - // and `hyperdb-mcp`'s engine is exactly that case. They replaced the - // `#[doc(hidden)] #[deprecated]` `begin_transaction`/`commit`/`rollback` - // wrappers, which were removed in 1.0.0. + // and `hyperdb-mcp`'s engine is exactly that case. /// Issues `BEGIN TRANSACTION` without returning a guard. /// @@ -2192,7 +2196,9 @@ impl Connection { Ok(()) } - /// Issues `COMMIT` for a transaction opened with + /// Issues `COMMIT` for an unguarded transaction. + /// + /// The transaction is opened with /// [`begin_transaction_unguarded`](Self::begin_transaction_unguarded). /// /// **Prefer [`Transaction::commit`](crate::Transaction::commit)** on the @@ -2206,7 +2212,9 @@ impl Connection { Ok(()) } - /// Issues `ROLLBACK` for a transaction opened with + /// Issues `ROLLBACK` for an unguarded transaction. + /// + /// The transaction is opened with /// [`begin_transaction_unguarded`](Self::begin_transaction_unguarded). /// /// **Prefer [`Transaction::rollback`](crate::Transaction::rollback)** on diff --git a/hyperdb-api/src/connection_builder.rs b/hyperdb-api/src/connection_builder.rs index 895a0b3e..5c01dcf7 100644 --- a/hyperdb-api/src/connection_builder.rs +++ b/hyperdb-api/src/connection_builder.rs @@ -17,8 +17,10 @@ use hyperdb_api_core::client::Config; /// # Transport Auto-Detection /// /// The transport is automatically detected from the endpoint URL: -/// - `https://` or `http://` → gRPC transport (read-only) -/// - Otherwise → TCP transport (e.g., `localhost:7483`) +/// - `https://` or `http://` → gRPC (read-only) +/// - `tab.domain://...` or an absolute socket path → Unix domain socket (Unix only) +/// - `tab.pipe://...` or a `\\host\pipe\name` path → named pipe (Windows only) +/// - otherwise `host:port` → TCP (e.g., `localhost:7483`) /// /// # Example /// @@ -55,7 +57,8 @@ pub struct ConnectionBuilder { user: Option, password: Option, login_timeout: Option, - /// Query timeout — cancel queries that exceed this duration. + /// Query timeout; recorded but not applied by any transport (see the + /// `query_timeout` setter). query_timeout: Option, /// Application name sent to the server during connection startup. application_name: Option, @@ -74,7 +77,7 @@ impl ConnectionBuilder { /// /// # Arguments /// - /// * `endpoint` - The server endpoint (host:port). + /// * `endpoint` - The server endpoint; see [`ConnectionBuilder`] for the accepted forms. pub fn new(endpoint: impl Into) -> Self { Self { endpoint: endpoint.into(), @@ -89,8 +92,8 @@ impl ConnectionBuilder { } } - #[must_use] /// Sets the database path. + #[must_use] pub fn database(mut self, path: impl AsRef) -> Self { self.database = Some(path.as_ref().to_path_buf()); self @@ -105,23 +108,23 @@ impl ConnectionBuilder { self } - #[must_use] /// Sets the username for authentication. /// /// Default is "`tableau_internal_user`". + #[must_use] pub fn user(mut self, user: impl Into) -> Self { self.user = Some(user.into()); self } - #[must_use] /// Sets the password for authentication. + #[must_use] pub fn password(mut self, password: impl Into) -> Self { self.password = Some(password.into()); self } - /// Sets the login timeout. + /// Sets the login timeout (TCP connections only). #[must_use] pub fn login_timeout(mut self, timeout: Duration) -> Self { self.login_timeout = Some(timeout); @@ -130,49 +133,50 @@ impl ConnectionBuilder { /// Sets the query timeout. /// - /// Queries that exceed this duration will be cancelled automatically. - /// Default is no timeout (queries run until completion). + /// The value is recorded on the builder but no transport applies it, so + /// queries run until completion regardless of this setting. #[must_use] pub fn query_timeout(mut self, timeout: Duration) -> Self { self.query_timeout = Some(timeout); self } - #[must_use] /// Sets the application name sent to the server. /// - /// This appears in server logs and can be used for monitoring. + /// This appears in server logs and can be used for monitoring. It applies + /// to TCP connections only. + #[must_use] pub fn application_name(mut self, name: impl Into) -> Self { self.application_name = Some(name.into()); self } - #[must_use] /// Convenience method to set user and password at once. + #[must_use] pub fn auth(mut self, user: impl Into, password: impl Into) -> Self { self.user = Some(user.into()); self.password = Some(password.into()); self } - #[must_use] /// Convenience method to create a new database. + #[must_use] pub fn create_new_database(mut self, database_path: impl AsRef) -> Self { self.database = Some(database_path.as_ref().to_path_buf()); self.create_mode = CreateMode::Create; self } - #[must_use] /// Convenience method to create database if it doesn't exist. + #[must_use] pub fn create_or_open_database(mut self, database_path: impl AsRef) -> Self { self.database = Some(database_path.as_ref().to_path_buf()); self.create_mode = CreateMode::CreateIfNotExists; self } - #[must_use] /// Convenience method to open an existing database. + #[must_use] pub fn open_database(mut self, database_path: impl AsRef) -> Self { self.database = Some(database_path.as_ref().to_path_buf()); self.create_mode = CreateMode::DoNotCreate; @@ -210,12 +214,20 @@ impl ConnectionBuilder { /// Builds and establishes the connection. /// /// The transport is automatically detected from the endpoint URL: - /// - `https://` or `http://` → gRPC transport - /// - Otherwise → TCP transport + /// - `https://` or `http://` → gRPC (read-only) + /// - `tab.domain://...` or an absolute socket path → Unix domain socket (Unix only) + /// - `tab.pipe://...` or a `\\host\pipe\name` path → named pipe (Windows only) + /// - otherwise `host:port` → TCP + /// + /// `application_name` and `login_timeout` apply to TCP connections only. /// /// # Errors /// - /// Returns an error if the connection fails or if database creation fails. + /// - Returns [`Error::Config`] if the endpoint cannot be parsed. + /// - Returns [`Error::FeatureNotSupported`] for a gRPC endpoint with a create mode other than [`CreateMode::DoNotCreate`]. + /// - Returns [`Error::Connection`] or [`Error::Timeout`] if the transport handshake fails. + /// - Returns [`Error::Authentication`] if authentication is rejected. + /// - Returns [`Error::Server`] or [`Error::Internal`] if the database creation or attach SQL fails. pub fn build(self) -> Result { let transport_type = detect_transport_type(&self.endpoint); diff --git a/hyperdb-api/src/copy.rs b/hyperdb-api/src/copy.rs index 20582e55..2fc91a7f 100644 --- a/hyperdb-api/src/copy.rs +++ b/hyperdb-api/src/copy.rs @@ -36,7 +36,7 @@ //! fn main() -> Result<()> { //! let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; //! -//! // Import from a file +//! // Import from an in-memory byte slice (any `impl Read` works) //! let csv_data = b"1,Alice\n2,Bob\n"; //! let rows = conn.import_csv("my_table", &csv_data[..])?; //! println!("Imported {} rows", rows); @@ -143,8 +143,8 @@ impl CopyOptions { self } - #[must_use] /// Sets the string used to represent NULL values. + #[must_use] pub fn with_null(mut self, null_string: impl Into) -> Self { self.null_string = Some(null_string.into()); self @@ -272,7 +272,7 @@ impl Connection { /// (COPY is TCP-only). /// - Returns [`Error::Server`] if the server rejects the /// `COPY () TO STDOUT` statement. - /// - Returns [`Error::Io`] if writing to `writer` fails. + /// - Returns [`Error::Connection`] if writing to `writer` fails. pub fn export_csv(&self, select_query: &str, writer: &mut dyn std::io::Write) -> Result { let opts = CopyOptions::csv().with_header(true); self.export_text(select_query, &opts, writer) @@ -302,7 +302,7 @@ impl Connection { /// // Pipe-separated with custom NULL /// let opts = CopyOptions::csv() /// .with_delimiter(b'|') - /// .with_null("\\N".to_string()) + /// .with_null("\\N") /// .with_header(true); /// conn.export_text("SELECT * FROM users", &opts, &mut std::io::stdout())?; /// Ok(()) @@ -311,12 +311,12 @@ impl Connection { /// /// # Errors /// - /// - Returns [`Error::Config`] if `options` fail validation (e.g. an - /// illegal delimiter/quote combination), or + /// - Returns [`Error::Config`] if `options` set a QUOTE or ESCAPE + /// character on a text/TSV format, or /// [`Error::FeatureNotSupported`] if the connection is on gRPC. /// - Returns [`Error::Server`] if the server rejects the /// `COPY TO STDOUT` statement. - /// - Returns [`Error::Io`] if writing to `writer` fails. + /// - Returns [`Error::Connection`] if writing to `writer` fails. pub fn export_text( &self, select_query: &str, @@ -350,7 +350,7 @@ impl Connection { /// fn main() -> Result<()> { /// let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::DoNotCreate)?; /// let csv = conn.export_csv_string("SELECT id, name FROM users")?; - /// println!("{}", csv); + /// println!("{csv}"); /// Ok(()) /// } /// ``` @@ -456,7 +456,7 @@ impl Connection { /// [`Error::FeatureNotSupported`] if the connection is on gRPC. /// - Returns [`Error::Server`] if the server rejects the /// `COPY FROM STDIN` statement or a row during import. - /// - Returns [`Error::Io`] if reading from `reader` fails. + /// - Returns [`Error::Connection`] if reading from `reader` fails. pub fn import_text( &self, table_name: &str, diff --git a/hyperdb-api/src/error.rs b/hyperdb-api/src/error.rs index a40d701e..8122a787 100644 --- a/hyperdb-api/src/error.rs +++ b/hyperdb-api/src/error.rs @@ -190,7 +190,7 @@ pub enum Error { // ---- Column / row mapping ------------------------------------------ /// Structured error for named-column access in row decoding. Used - /// by `FromRow` impls and `Row::try_get` / `Row::get_by_name` to + /// by `FromRow` impls (through `RowAccessor`) and `Row::get_by_name` to /// signal which column failed and why. #[error("column {name}: {kind}")] Column { diff --git a/hyperdb-api/src/grpc_connection.rs b/hyperdb-api/src/grpc_connection.rs index 5aa1ef83..2f20f975 100644 --- a/hyperdb-api/src/grpc_connection.rs +++ b/hyperdb-api/src/grpc_connection.rs @@ -73,7 +73,7 @@ use crate::error::Result; -// Re-export types from hyperdb_api_core::client::grpc for convenience +// Internal imports from hyperdb_api_core::client::grpc; the public re-exports live in `crate::grpc`. pub(crate) use hyperdb_api_core::client::grpc::{ GrpcClient, GrpcClientSync, GrpcConfig, GrpcQueryResult, }; @@ -95,12 +95,9 @@ pub(crate) use hyperdb_api_core::client::grpc::{ /// /// # Async vs Sync /// -/// This struct provides both async and sync APIs: -/// -/// - `connect()` / `execute_query()` - blocking (uses internal tokio runtime) -/// - `connect_async()` / `execute_query_async()` - async (requires tokio runtime) -/// -/// For applications already using tokio, the async methods are preferred. +/// This struct is blocking: `connect()` and `execute_query()` use an internal +/// tokio runtime. Applications already using tokio should use the async +/// counterpart, [`GrpcConnectionAsync`]. #[derive(Debug)] pub struct GrpcConnection { /// The underlying gRPC client (sync wrapper) @@ -195,7 +192,7 @@ impl GrpcConnection { /// # Errors /// /// Returns [`crate::Error::Server`] if the gRPC server rejects the query - /// or if the HTTP/2 channel fails mid-stream. + /// or [`crate::Error::Connection`] if the HTTP/2 channel fails mid-stream. pub fn execute_query_to_arrow(&mut self, sql: &str) -> Result { Ok(self.client.execute_query_to_arrow(sql)?) } @@ -221,7 +218,7 @@ impl GrpcConnection { /// # Errors /// /// Returns [`crate::Error::Server`] if the gRPC server rejects the query - /// or the HTTP/2 channel fails. + /// or [`crate::Error::Connection`] if the HTTP/2 channel fails. pub fn execute_query(&mut self, sql: &str) -> Result { Ok(self.client.execute_query(sql)?) } @@ -248,21 +245,12 @@ impl GrpcConnection { /// (channel closed, network error, auth expired) — useful for /// metrics, retry logic, or "cancel failed" UX. /// - /// # Fallible by design - /// - /// The `Result<()>` return is the **explicit user-facing cancel - /// API** and is distinct from the - /// [`Cancellable`](hyperdb_api_core::client::Cancellable) trait, which requires - /// an infallible `cancel(&self)` method with no arguments. A - /// `GrpcConnection` cannot implement `Cancellable` directly: the - /// trait's signature has nowhere to pass a `query_id`, and gRPC - /// connections can carry many concurrent queries (so there is no - /// unambiguous "the" query to cancel the way there is on a PG wire - /// connection). If you need `Cancellable`-style fire-and-forget - /// cancel for a future gRPC streaming result type, it will live - /// on a per-query handle that wraps this method and swallows - /// errors — mirroring - /// [`impl Cancellable for hyperdb_api_core::client::Client`](hyperdb_api_core::client::Cancellable). + /// # Why it takes a `query_id` + /// + /// A gRPC channel can carry many concurrent queries, so there is no + /// single query to cancel implicitly; the caller names the query to + /// cancel, and the call returns `Result<()>` so transport failures are + /// visible. /// /// # Example /// @@ -375,8 +363,8 @@ impl GrpcConnectionAsync { /// /// # Errors /// - /// Returns [`crate::Error::Server`] if the server rejects the query or the - /// HTTP/2 channel fails mid-stream. + /// Returns [`crate::Error::Server`] if the server rejects the query or + /// [`crate::Error::Connection`] if the HTTP/2 channel fails mid-stream. pub async fn execute_query_to_arrow(&mut self, sql: &str) -> Result { Ok(self.client.execute_query_to_arrow(sql).await?) } @@ -385,18 +373,15 @@ impl GrpcConnectionAsync { /// /// # Errors /// - /// Returns [`crate::Error::Server`] if the server rejects the query or the - /// HTTP/2 channel fails. + /// Returns [`crate::Error::Server`] if the server rejects the query or + /// [`crate::Error::Connection`] if the HTTP/2 channel fails. pub async fn execute_query(&mut self, sql: &str) -> Result { Ok(self.client.execute_query(sql).await?) } /// Cancels an in-flight gRPC query by its `query_id` (async). /// - /// See [`GrpcConnection::cancel_query`] for full semantics, including - /// the "Fallible by design" discussion of why this returns - /// `Result<()>` and why it is *not* an implementation of the - /// [`Cancellable`](hyperdb_api_core::client::Cancellable) trait. The async + /// See [`GrpcConnection::cancel_query`] for full semantics. The async /// variant avoids blocking the current thread; both variants route /// the cancel over the same channel used for queries, carrying this /// connection's database routing and custom headers. diff --git a/hyperdb-api/src/lib.rs b/hyperdb-api/src/lib.rs index 77f3f6b1..a8a23433 100644 --- a/hyperdb-api/src/lib.rs +++ b/hyperdb-api/src/lib.rs @@ -9,16 +9,17 @@ //! //! # Architecture //! -//! This is a layered API built from four crates: -//! - `hyper-types` — Type definitions with `LittleEndian` encoding -//! - `hyper-protocol` — Wire protocol with `HyperBinary` COPY support -//! - `hyper-client` — Sync/async TCP and gRPC clients -//! - `hyperdb-api` — High-level API (this crate) +//! This crate is the user-facing layer over `hyperdb-api-core`, an internal +//! implementation crate (not a public API) with three modules: `types` +//! (`LittleEndian` encoding), `protocol` (wire protocol, `HyperBinary` COPY) +//! and `client` (sync/async TCP, UDS, named-pipe and gRPC clients). //! //! Optional companion crates: //! - `sea-query-hyperdb` — `HyperDB` SQL dialect backend for `sea-query` //! - `hyperdb-api-salesforce` — Salesforce Data Cloud OAuth authentication -//! - `hyperdb-api-derive` — Proc-macro `#[derive(FromRow)]` (re-exported by this crate) +//! - `hyperdb-api-derive` — `#[derive(FromRow)]`, `#[derive(Table)]`, +//! `query_as!` and `query_scalar!`; add it as a direct dependency (it is not +//! re-exported here) //! //! # Quick Start //! @@ -37,7 +38,7 @@ //! for row in &chunk { //! let id: Option = row.get(0); //! let name: Option = row.get(1); -//! println!("id: {:?}, name: {:?}", id, name); +//! println!("id: {id:?}, name: {name:?}"); //! } //! } //! Ok(()) @@ -54,7 +55,6 @@ //! ```text //! Connection (owns underlying client) //! ├── Inserter<'conn> -//! │ └── CopyInWriter<'conn> //! ├── Catalog<'conn> //! ├── KvStore<'conn> //! ├── Rowset<'conn> @@ -63,7 +63,8 @@ //! //! This is a **simple hierarchical design**, not a complex lifetime web: //! - **Single root owner**: `Connection` owns the underlying client -//! - **Simple borrows**: All dependent types borrow `&'conn Connection` +//! - **Simple borrows**: Dependent types borrow the `Connection`; a +//! [`Transaction`] borrows it exclusively //! - **No circular references**: `Inserter` doesn't reference `Catalog`, etc. //! - **Single lifetime parameter**: Just one `'conn` — no multi-lifetime bounds //! @@ -71,11 +72,13 @@ //! while any dependent type holds a reference to it: //! //! ```compile_fail -//! # use hyperdb_api::{Connection, Inserter, CreateMode}; +//! # use hyperdb_api::{Connection, Inserter, CreateMode, SqlType, TableDefinition}; //! # fn example() -> hyperdb_api::Result<()> { //! let conn = Connection::connect("localhost:7483", "test.hyper", CreateMode::CreateIfNotExists)?; -//! let inserter = Inserter::new(&conn, /* ... */)?; -//! drop(conn); // ERROR: cannot move `conn` because it is borrowed by `inserter` +//! let def = TableDefinition::new("t").add_required_column("a", SqlType::int()); +//! let mut inserter = Inserter::new(&conn, &def)?; +//! drop(conn); // ERROR: cannot move out of `conn` because it is borrowed +//! inserter.execute()?; //! # Ok(()) //! # } //! ``` @@ -94,9 +97,8 @@ //! # } //! ``` //! -//! The `execute(self)` method on [`Inserter`] takes ownership (`self`), which -//! automatically ends the borrow when the insert completes — no manual cleanup -//! needed. +//! [`Inserter::execute`] takes `&mut self`; the borrow of the [`Connection`] +//! ends when the [`Inserter`] is dropped. //! //! # Key Types //! @@ -153,10 +155,10 @@ #![warn(clippy::must_use_candidate)] mod arrow_inserter; -/// Semantic version of this crate, resolved at compile time from -/// `Cargo.toml`. Used by downstream tools (notably `hyperdb-mcp`) to -/// surface the library version in their own status output without -/// duplicating the version string. +/// Semantic version of this crate, resolved at compile time from `Cargo.toml`. +/// +/// Used by downstream tools (notably `hyperdb-mcp`) to surface the library +/// version in their own status output without duplicating the version string. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); mod arrow_reader; @@ -211,6 +213,7 @@ pub use async_inserter::AsyncInserter; pub use async_kv_store::AsyncKvStore; pub use async_prepared::{AsyncPreparedStatement, AsyncPreparedStatementOwned}; pub use async_result::AsyncRowset; +pub use async_transaction::AsyncTransaction; pub use catalog::Catalog; pub use connection::{Connection, CreateMode, ScalarValue}; pub use connection_builder::ConnectionBuilder; @@ -220,7 +223,6 @@ pub use prepared::PreparedStatement; // Re-export Notice for callback registrants. `hyperdb-api-core`'s // `client::Error` is intentionally NOT re-exported — callers match // directly on the flat `Error` enum this crate defines. -pub use async_transaction::AsyncTransaction; pub use hyperdb_api_core::client::{Notice, NoticeReceiver}; pub use inserter::{ChunkSender, ColumnMapping, InsertChunk, Inserter, IntoValue, MappedInserter}; pub use kv_store::{BatchGuardOutcome, BatchSetOutcome, KvStore, SetOutcome}; @@ -249,10 +251,10 @@ pub use hyperdb_api_core::types::{ Timestamp, Type, }; -/// Re-export of `GeoError` from hyperdb-api-core::types. +/// Error type for geography conversions (WKT parsing, WKB encoding and decoding). pub use hyperdb_api_core::types::GeoError; -/// Re-export of the PostgreSQL OID constants. Access as `hyperdb_api::oids::INT4` etc. +/// Re-export of the PostgreSQL OID constants, e.g. `hyperdb_api::oids::INT`. pub use hyperdb_api_core::types::oids; // Re-export gRPC types (always available) diff --git a/hyperdb-api/src/process.rs b/hyperdb-api/src/process.rs index 4b06ee18..79b1f90d 100644 --- a/hyperdb-api/src/process.rs +++ b/hyperdb-api/src/process.rs @@ -828,10 +828,10 @@ impl HyperProcess { /// Parses a connection descriptor to extract host:port, socket path, or pipe path. /// /// Input formats: - /// - "tab.tcp://host:port" → "host:port" - /// - "tab.domain:///domain/" → "/domain/" (socket path) - /// - "tab.pipe:///pipe/" → "/pipe/" (named pipe) - /// - "tcp.grpc://host:port" → "host:port" + /// - `tab.tcp://host:port` → `host:port` + /// - `tab.domain:///domain/` → `/domain/` (socket path) + /// - `tab.pipe:///pipe/` → `/pipe/` (named pipe) + /// - `tcp.grpc://host:port` → `host:port` fn parse_connection_descriptor(descriptor: &str) -> Result { // Handle domain socket format if let Some(rest) = descriptor.strip_prefix("tab.domain://") { @@ -1204,6 +1204,23 @@ pub(crate) const NO_DEFAULT_PARAMETERS: &str = "no_default_parameters"; /// Default log configuration for hyperd: file-based JSON logging. const DEFAULT_LOG_CONFIG: &str = "file,json,all,hyperd,0"; +/// Transport used for client connections to a [`HyperProcess`]. +/// +/// `HyperProcess` uses [`TransportMode::Tcp`] unless [`Parameters::set_transport_mode`] +/// selects otherwise. [`TransportMode::Ipc`] (Unix domain sockets on Unix, named pipes +/// on Windows) has lower connect and round-trip latency but slower large streamed +/// reads than TCP. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TransportMode { + /// Use IPC (Unix domain sockets on Unix, named pipes on Windows). + #[default] + Ipc, + + /// Use TCP/IP connections. + /// Required when connecting from remote clients or when IPC is not available. + Tcp, +} + /// Parameters for configuring the Hyper server. /// /// When starting a [`HyperProcess`], a set of default parameters are automatically applied @@ -1222,7 +1239,7 @@ const DEFAULT_LOG_CONFIG: &str = "file,json,all,hyperd,0"; /// | `default_database_version` | `3` | File format version for newly created `.hyper` databases (v3 adds 128-bit NUMERIC support, required for DECIMAL128 parquet columns) | /// /// To disable these defaults, add the `no_default_parameters` key (for example -/// `params.set("no_default_parameters", "")` via [`Parameters::set`]. +/// `params.set("no_default_parameters", "")`) via [`Parameters::set`]. /// /// # Listen Modes /// @@ -1253,52 +1270,13 @@ const DEFAULT_LOG_CONFIG: &str = "file,json,all,hyperd,0"; /// # Transport Modes /// /// Use [`set_transport_mode`](Parameters::set_transport_mode) to control whether Hyper uses -/// TCP or IPC (Unix Domain Sockets): -/// -/// ``` -/// use hyperdb_api::{Parameters, TransportMode}; -/// -/// let mut params = Parameters::new(); -/// params.set_transport_mode(TransportMode::Tcp); // Force TCP instead of IPC -/// ``` -/// -/// Transport mode for `HyperProcess` connections. -/// -/// Controls whether the server uses TCP or Unix Domain Sockets (IPC) for connections. -/// On Unix systems, IPC is the default for better local performance. -/// On Windows, TCP is always used. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum TransportMode { - /// Use IPC (Unix Domain Sockets on Unix, Named Pipes on Windows). - /// This is the default mode and provides better performance for local connections. - #[default] - Ipc, - - /// Use TCP/IP connections. - /// Required when connecting from remote clients or when IPC is not available. - Tcp, -} - -/// Parameters for configuring the Hyper server. -/// -/// When starting a [`HyperProcess`], a set of default parameters are automatically applied -/// (matching the C++ `HyperProcess` behavior). You can override these defaults or disable -/// them entirely by adding the `no_default_parameters` key (for example -/// `params.set("no_default_parameters", "")` via [`Parameters::set`]. -/// -/// # Transport Modes -/// -/// Use [`set_transport_mode`](Self::set_transport_mode) to control whether Hyper uses -/// TCP or IPC (Unix Domain Sockets on Unix systems). -/// -/// # Example +/// TCP or IPC (Unix domain sockets on Unix, named pipes on Windows). TCP is the default: /// /// ``` /// use hyperdb_api::{Parameters, TransportMode}; /// /// let mut params = Parameters::new(); -/// params.set("log_file_size_limit", "100k"); -/// params.set_transport_mode(TransportMode::Tcp); // Force TCP instead of IPC +/// params.set_transport_mode(TransportMode::Ipc); // opt into Unix domain sockets / named pipes /// ``` #[derive(Debug, Clone, Default)] pub struct Parameters { @@ -1327,8 +1305,8 @@ impl Parameters { /// Sets the transport mode (TCP or IPC/UDS). /// - /// By default, `HyperProcess` uses IPC (Unix Domain Sockets on Unix) for better - /// performance. Use `TransportMode::Tcp` if you need TCP connections. + /// By default, `HyperProcess` uses TCP. Pass `TransportMode::Ipc` to use Unix domain + /// sockets (Unix) or named pipes (Windows). /// /// # Example /// @@ -1336,7 +1314,7 @@ impl Parameters { /// use hyperdb_api::{Parameters, TransportMode}; /// /// let mut params = Parameters::new(); - /// params.set_transport_mode(TransportMode::Tcp); // Use TCP instead of IPC + /// params.set_transport_mode(TransportMode::Ipc); // opt into IPC /// ``` pub fn set_transport_mode(&mut self, mode: TransportMode) -> &mut Self { self.transport_mode = Some(mode); diff --git a/hyperdb-api/src/result.rs b/hyperdb-api/src/result.rs index 917e021e..42fb0a7a 100644 --- a/hyperdb-api/src/result.rs +++ b/hyperdb-api/src/result.rs @@ -8,74 +8,8 @@ //! - [`RowIterator`] — C++-like iterator for simple row-by-row processing //! - [`ResultSchema`] — Column metadata (names and types) //! -//! # Streaming Design -//! -//! Query results are streamed from the server in chunks of up to -//! [`DEFAULT_BINARY_CHUNK_SIZE`] rows (64K). Only one chunk is held in memory -//! at a time, so memory usage is `O(chunk_size)` regardless of total result -//! size — safe for billion-row results. -//! -//! # Iteration Patterns -//! -//! Two patterns are available, both streaming with constant memory: -//! -//! ## Pattern 1: Chunked (`next_chunk()`) — batch processing -//! -//! Best for high-throughput scenarios. Error checking happens once per chunk -//! (~64K rows), and you get direct `Vec` iteration with good cache -//! locality. Natural for batch operations, vectorized processing, or -//! parallelizing across chunks. -//! -//! ```no_run -//! # use hyperdb_api::{Connection, CreateMode, Result}; -//! # fn example(conn: &Connection) -> Result<()> { -//! let mut result = conn.execute_query("SELECT * FROM table")?; -//! while let Some(chunk) = result.next_chunk()? { -//! for row in &chunk { -//! let id: Option = row.get(0); -//! let value: Option = row.get(1); -//! } -//! } -//! # Ok(()) -//! # } -//! ``` -//! -//! ## Pattern 2: Iterator (`rows()`) — simple row-by-row -//! -//! Best for simple iteration where you process one row at a time. Each item -//! is `Result` since chunk fetches can fail, so error checking happens -//! per-row. The extra iterator wrapper adds slight overhead compared to -//! `next_chunk()`. -//! -//! ```no_run -//! # use hyperdb_api::{Connection, Result}; -//! # fn example(conn: &Connection) -> Result<()> { -//! let result = conn.execute_query("SELECT * FROM table")?; -//! for row in result.rows() { -//! let row = row?; // Handle potential errors -//! let id: Option = row.get(0); -//! let value: Option = row.get(1); -//! } -//! # Ok(()) -//! # } -//! ``` -//! -//! **When to use which:** -//! - `rows()` — simple iteration, one row at a time, small overhead acceptable -//! - `next_chunk()` — maximum performance, large result sets, batch operations -//! -//! # Type Coercion -//! -//! The generic `row.get::()` method supports automatic widening coercion: -//! -//! | Request Type | Coerces From | -//! |---|---| -//! | `i32` | `i16` | -//! | `i64` | `i32`, `i16` | -//! | `f64` | `f32` | -//! -//! Direct accessors (`row.get_i32()`, `row.get_f64()`) skip coercion for -//! slightly better performance when the exact type is known. +//! See [`Rowset`] for streaming and iteration guidance and [`RowValue`] for +//! the type coercions applied by `row.get::()`. use std::sync::Arc; @@ -88,7 +22,7 @@ use hyperdb_api_core::types::SqlType; use crate::arrow_result::{ArrowRowset, FromArrowValue}; use crate::error::Result; -/// Default chunk size for streaming queries (64K rows). +/// Default chunk size for TCP streaming queries (64K rows). pub(crate) const DEFAULT_BINARY_CHUNK_SIZE: usize = 65536; // ============================================================================= @@ -123,9 +57,7 @@ pub struct Row { /// from an `Arc`) so that metadata-dependent decoders like /// [`Self::get_numeric`] can look up `SqlType` per column without /// the caller plumbing scale through manually. `None` only in the - /// unusual case a row is constructed outside `next_chunk` (no such - /// path exists in-tree today; the field is `Option` so future - /// schemas-unavailable paths remain compilable). + /// unusual case a row is constructed outside `next_chunk`. schema: Option>, } @@ -221,8 +153,9 @@ impl Row { /// Gets a typed value at the given column index, returning a `Result` /// with a descriptive error on failure. /// - /// Use this in [`FromRow`] implementations for better error messages - /// than bare `row.get(idx).ok_or(...)`. + /// Prefer this over [`Row::get`] when a NULL or type mismatch should be an + /// error. Inside [`FromRow`] implementations use + /// [`RowAccessor::position`](crate::RowAccessor::position) instead. /// /// # Example /// @@ -403,8 +336,11 @@ impl Row { /// Gets raw bytes at the given column index. /// - /// For TCP rows, returns the raw binary data. For Arrow rows, this method - /// is not available and returns None. + /// Returns `None` if the value is NULL or `idx` is out of range. + /// + /// For TCP rows this is the raw HyperBinary encoding of any column type. + /// For Arrow rows only `BYTEA` (Arrow `Binary`/`LargeBinary`) columns + /// yield bytes; other types return `None`. #[inline] pub fn get_bytes(&self, idx: usize) -> Option> { match &self.inner { @@ -628,6 +564,19 @@ impl Row { } /// Trait for types that can be extracted from a Row. +/// +/// # Type Coercion +/// +/// The generic `row.get::()` method supports automatic widening coercion: +/// +/// | Request Type | Coerces From | +/// |---|---| +/// | `i32` | `i16` | +/// | `i64` | `i32`, `i16` | +/// | `f64` | `f32` | +/// +/// Direct accessors (`row.get_i32()`, `row.get_f64()`) skip coercion for +/// slightly better performance when the exact type is known. pub trait RowValue: Sized { /// Extract a value from a Row at the given column index. fn from_row(row: &Row, idx: usize) -> Option; @@ -957,10 +906,21 @@ impl ResultSchema { /// # } /// ``` /// +/// # Iteration Patterns +/// +/// Two streaming patterns are available: +/// +/// - [`next_chunk`](Self::next_chunk): best for high-throughput and batch +/// work. Errors are checked once per chunk and each chunk is a plain +/// `Vec`. +/// - [`rows`](Self::rows): best for simple row-by-row processing. Each item +/// is a `Result` because fetching a later chunk can fail. +/// /// # Memory Behavior /// /// - Only one chunk is held in memory at a time -/// - Default chunk size is 64K rows (~few MB depending on row width) +/// - Over TCP a chunk holds up to 65,536 rows (~few MB depending on row +/// width); over gRPC a chunk is one Arrow record batch as sent by the server /// - Memory usage is `O(chunk_size)`, not `O(total_rows)` /// - Safe for billion-row results pub struct Rowset<'conn> { @@ -974,12 +934,13 @@ pub struct Rowset<'conn> { /// the column's `SqlType` without the caller plumbing scale /// through manually. schema_cache: Option>, - /// For one-shot prepared statements (the internal + /// For one-shot prepared statements (the /// [`crate::Connection::query_params`] path), hold the statement /// handle here so its `Drop`-time `close_statement` fires *after* /// the rowset releases its connection lock. Dropping the statement /// before the rowset would deadlock because the inner stream owns - /// the connection's `MutexGuard`. + /// the connection's `MutexGuard`. Field order matters: `inner` must be + /// declared before this field so it drops first. _statement_guard: Option, } @@ -995,7 +956,7 @@ impl std::fmt::Debug for Rowset<'_> { enum RowsetInner<'conn> { /// TCP streaming result (uses `QueryStream`). Tcp(QueryStream<'conn>), - /// Arrow-based result from gRPC (all data loaded). + /// Arrow result streamed from gRPC, one `RecordBatch` per chunk. Arrow(ArrowRowset), /// TCP streaming result from a prepared-statement execute. Prepared(hyperdb_api_core::client::PreparedQueryStream<'conn>), @@ -1033,7 +994,7 @@ impl<'conn> Rowset<'conn> { #[expect( clippy::used_underscore_binding, - reason = "underscore-prefixed parameter retained for trait-method signature compatibility" + reason = "the `_statement_guard` field is underscore-prefixed because it is held only for its drop order" )] /// Attaches a `OwnedPreparedStatement` that should be dropped /// **after** this rowset is consumed. Used by the one-shot @@ -1165,7 +1126,8 @@ impl<'conn> Rowset<'conn> { /// Returns the next chunk of rows from the result set. /// - /// Each chunk contains up to `chunk_size` rows (default 64K). + /// Over TCP each chunk holds up to 65,536 rows; over gRPC each chunk is one + /// Arrow record batch as sent by the server. /// Returns `Ok(None)` when all rows have been consumed. /// /// # Example @@ -1186,8 +1148,14 @@ impl<'conn> Rowset<'conn> { /// # Errors /// /// - Returns [`crate::Error::Server`] if the server sends an `ErrorResponse` - /// while streaming the result set. - /// - Returns [`crate::Error::Io`] on transport-level I/O failures. + /// while streaming the result set (over TCP this includes a query + /// cancelled with SQLSTATE `57014`). + /// - Returns [`crate::Error::Connection`] if reading from the transport + /// fails, or [`crate::Error::Closed`] if the server closes the + /// connection mid-stream. + /// - Over gRPC, returns [`crate::Error::Cancelled`] or + /// [`crate::Error::Timeout`] if the call is cancelled or its deadline + /// expires. /// - Returns [`crate::Error::Conversion`] if an Arrow IPC chunk cannot be decoded. pub fn next_chunk(&mut self) -> Result>> { // Pull the next raw chunk from the underlying transport first; @@ -1255,8 +1223,8 @@ impl<'conn> Rowset<'conn> { /// # Error Handling /// /// Unlike C++ which uses exceptions, Rust requires explicit error handling. - /// Each item in the iterator is a `Result` to handle - /// potential network or protocol errors during streaming. + /// Each item is a `Result<`[`Row`]`>` so that errors fetching later chunks + /// surface during iteration. /// /// # Comparison with `next_chunk()` /// @@ -1276,6 +1244,9 @@ impl<'conn> Rowset<'conn> { /// Collects all rows into a Vec. /// + /// Buffers the entire result in memory. For large results, iterate with + /// [`next_chunk`](Self::next_chunk) or [`rows`](Self::rows) instead. + /// /// This is a convenience method that handles error collection more elegantly /// than the standard `collect::, _>>()` pattern. /// @@ -1307,6 +1278,9 @@ impl<'conn> Rowset<'conn> { /// Collects the first column of each row into a Vec. /// + /// Buffers the entire result in memory. For large results, iterate with + /// [`next_chunk`](Self::next_chunk) or [`rows`](Self::rows) instead. + /// /// This is useful for single-column queries or when you only need one column. /// /// # Example @@ -1341,6 +1315,9 @@ impl<'conn> Rowset<'conn> { /// Collects the first column, filtering out NULL values. /// + /// Buffers the entire result in memory. For large results, iterate with + /// [`next_chunk`](Self::next_chunk) or [`rows`](Self::rows) instead. + /// /// This is useful when you know the column doesn't contain NULLs or want to ignore them. /// /// # Example @@ -1492,7 +1469,7 @@ impl<'conn> Rowset<'conn> { /// # Memory Behavior /// /// Memory usage remains constant regardless of result set size: -/// - Internally fetches 64K rows at a time +/// - Internally fetches one chunk at a time (up to 65,536 rows over TCP) /// - Previous chunks are dropped when exhausted /// - Safe for billion-row results /// diff --git a/hyperdb-api/src/row_accessor.rs b/hyperdb-api/src/row_accessor.rs index 0a27f855..b003745e 100644 --- a/hyperdb-api/src/row_accessor.rs +++ b/hyperdb-api/src/row_accessor.rs @@ -1,7 +1,7 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! [`RowAccessor`] — name-based column access for [`FromRow`] impls +//! [`RowAccessor`] — name-based column access for [`FromRow`](crate::FromRow) impls //! with cached column-name → index resolution. //! //! When a typed query is consumed via [`Connection::fetch_one_as`] / diff --git a/hyperdb-api/src/table_copy.rs b/hyperdb-api/src/table_copy.rs index 4674434c..b641abcf 100644 --- a/hyperdb-api/src/table_copy.rs +++ b/hyperdb-api/src/table_copy.rs @@ -24,8 +24,10 @@ //! with `Index support is disabled`, and `CHECK` with `check constraints not //! implemented yet`. A Hyper table therefore never *has* one of those to lose. //! What it can have is `NOT NULL`, `DEFAULT`, `COLLATE`, and the assumed key -//! forms ([`TableConstraint`](crate::TableConstraint)) — and all five survive -//! a copy. +//! forms ([`TableConstraint`](crate::TableConstraint)) — and all five are +//! reproduced by a copy, except that a `DEFAULT` expression that is not +//! portable to another database is dropped and listed in +//! [`CopyTableReport::unpreserved`]. /// Why a piece of the source schema could not be reproduced. #[derive(Debug, Clone, PartialEq, Eq)] @@ -109,7 +111,7 @@ impl std::fmt::Display for UnpreservedItem { /// and what it dropped. /// /// A copy that returns `Ok` has moved every row. It has **not** necessarily -/// reproduced every constraint — check [`is_fully_preserved`](Self::is_fully_preserved). +/// reproduced every constraint and default — check [`is_fully_preserved`](Self::is_fully_preserved). #[derive(Debug, Clone, Default, PartialEq, Eq)] #[non_exhaustive] pub struct CopyTableReport { @@ -130,7 +132,7 @@ pub struct CopyTableReport { } impl CopyTableReport { - /// Returns `true` if every constraint on the source was reproduced. + /// Returns `true` if every constraint and default on the source was reproduced. #[must_use] pub fn is_fully_preserved(&self) -> bool { self.unpreserved.is_empty() diff --git a/hyperdb-api/src/transaction.rs b/hyperdb-api/src/transaction.rs index ce298647..e90bc0b2 100644 --- a/hyperdb-api/src/transaction.rs +++ b/hyperdb-api/src/transaction.rs @@ -36,9 +36,8 @@ pub struct Transaction<'conn> { impl<'conn> Transaction<'conn> { /// Creates a new transaction by issuing `BEGIN TRANSACTION`. pub(crate) fn new(connection: &'conn mut Connection) -> Result { - // Use the crate-internal `_raw` family. The matching `pub` - // methods on `Connection` are `#[deprecated]` for downstream - // consumers; this guard is the recommended replacement. + // Delegate to the public `*_unguarded` primitive; the guard adds the + // pairing guarantee (commit/rollback consume `self`, drop rolls back). connection.begin_transaction_unguarded()?; Ok(Self { connection, diff --git a/hyperdb-api/src/transport.rs b/hyperdb-api/src/transport.rs index 68d71d10..f6c1bde3 100644 --- a/hyperdb-api/src/transport.rs +++ b/hyperdb-api/src/transport.rs @@ -12,7 +12,7 @@ use crate::error::{Error, Result}; -/// Transport type indicator (public for introspection). +/// Transport type indicator (crate-internal). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum TransportType { /// TCP connection using `PostgreSQL` wire protocol. @@ -85,12 +85,12 @@ pub(crate) enum Transport { Grpc(Box), } -/// TCP transport using hyper-client. +/// TCP transport using `hyperdb_api_core::client`. pub(crate) struct TcpTransport { pub(crate) client: hyperdb_api_core::client::Client, } -/// gRPC transport using hyper-client's gRPC module. +/// gRPC transport using the `hyperdb_api_core::client::grpc` module. pub(crate) struct GrpcTransport { pub(crate) client: hyperdb_api_core::client::grpc::GrpcClientSync, pub(crate) config: hyperdb_api_core::client::grpc::GrpcConfig, @@ -121,7 +121,7 @@ impl Transport { pub(crate) fn supports_writes(&self) -> bool { match self { Transport::Tcp(_) => true, - Transport::Grpc(_) => false, // TODO: Server capability check in future + Transport::Grpc(_) => false, } } diff --git a/hyperdb-api/tests/arrow_inserter_tests.rs b/hyperdb-api/tests/arrow_inserter_tests.rs index 8f9d90a9..92eab705 100644 --- a/hyperdb-api/tests/arrow_inserter_tests.rs +++ b/hyperdb-api/tests/arrow_inserter_tests.rs @@ -3,9 +3,10 @@ //! Tests for the `ArrowInserter` API. //! -//! Note: These tests focus on the API behavior and error handling. -//! Full integration tests with real Arrow data would require the `arrow` crate -//! as a dev-dependency or pre-generated Arrow IPC test data. +//! These tests cover construction and its error paths, status accessors, +//! cancellation and drop behavior, and inserting real Arrow IPC streams +//! through `insert_data` and `insert_raw`, including with a small flush +//! threshold. use hyperdb_api::{ArrowInserter, Catalog, SqlType, TableDefinition}; @@ -171,31 +172,6 @@ fn test_arrow_inserter_execute_without_data() { assert_eq!(count, 0); } -// ============================================================================= -// ArrowInserter Status Methods Tests -// ============================================================================= - -#[test] -fn test_arrow_inserter_status_methods() { - let test = TestConnection::new().expect("Failed to create test connection"); - - let table_def = TableDefinition::new("test_status").add_required_column("id", SqlType::int()); - - Catalog::new(&test.connection) - .create_table(&table_def) - .expect("Failed to create table"); - - let inserter = - ArrowInserter::new(&test.connection, &table_def).expect("Failed to create inserter"); - - // Initial state - assert!(!inserter.has_data()); - assert_eq!(inserter.total_bytes(), 0); - assert_eq!(inserter.chunk_count(), 0); - - inserter.cancel(); -} - // ============================================================================= // ArrowInserter Cancel Tests // ============================================================================= @@ -248,51 +224,122 @@ fn test_arrow_inserter_drop_without_execute() { } // ============================================================================= -// Note on Integration Tests with Real Arrow Data +// ArrowInserter IPC Insert Tests // ============================================================================= -// -// To test with real Arrow data, you would need to either: -// 1. Add `arrow` crate as a dev-dependency and generate Arrow IPC streams -// 2. Include pre-generated Arrow IPC test files -// -// Example with arrow crate (if added as dev-dependency): -// ```rust -// #[test] -// fn test_arrow_inserter_with_real_data() { -// use arrow::array::Int32Array; -// use arrow::datatypes::{DataType, Field, Schema}; -// use arrow::ipc::writer::StreamWriter; -// use arrow::record_batch::RecordBatch; -// use std::sync::Arc; -// -// let test = TestConnection::new().expect("Failed to create test connection"); -// -// // Create matching table -// let table_def = TableDefinition::new("arrow_test") -// .add_required_column("id", SqlType::int()); -// Catalog::new(&test.connection).create_table(&table_def).unwrap(); -// -// // Generate Arrow IPC data -// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]); -// let batch = RecordBatch::try_new( -// Arc::new(schema.clone()), -// vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))], -// ).unwrap(); -// -// let mut buffer = Vec::new(); -// { -// let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap(); -// writer.write(&batch).unwrap(); -// writer.finish().unwrap(); -// } -// -// // Insert Arrow data -// let mut inserter = ArrowInserter::new(&test.connection, &table_def).unwrap(); -// inserter.insert_data(&buffer).unwrap(); -// let rows = inserter.execute().unwrap(); -// -// assert_eq!(rows, 5); -// let count = test.execute_scalar_i64("SELECT COUNT(*) FROM arrow_test").unwrap(); -// assert_eq!(count, 5); -// } -// ``` + +#[test] +fn test_arrow_inserter_insert_data_ipc_stream() { + use arrow::array::Int32Array; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::ipc::writer::StreamWriter; + use arrow::record_batch::RecordBatch; + use std::sync::Arc; + + let test = TestConnection::new().expect("Failed to create test connection"); + + let table_def = TableDefinition::new("arrow_ipc").add_required_column("id", SqlType::int()); + Catalog::new(&test.connection) + .create_table(&table_def) + .expect("Failed to create table"); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))], + ) + .expect("Failed to build batch"); + + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &schema).expect("stream writer"); + writer.write(&batch).expect("write batch"); + writer.finish().expect("finish stream"); + } + + let mut inserter = + ArrowInserter::new(&test.connection, &table_def).expect("Failed to create inserter"); + inserter.insert_data(&buffer).expect("insert_data"); + let rows = inserter.execute().expect("execute"); + assert_eq!(rows, 5); + + let count = test + .execute_scalar_i64("SELECT COUNT(*) FROM arrow_ipc") + .expect("count"); + assert_eq!(count, 5); + let sum = test + .execute_scalar_i64("SELECT SUM(id) FROM arrow_ipc") + .expect("sum"); + assert_eq!(sum, 15); +} + +/// Serializes one Arrow IPC stream with an `id: Int32` schema and one batch per entry. +fn int_ipc_stream(batches: &[Vec]) -> Vec { + use arrow::array::Int32Array; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::ipc::writer::StreamWriter; + use arrow::record_batch::RecordBatch; + use std::sync::Arc; + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &schema).expect("stream writer"); + for values in batches { + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(values.clone()))], + ) + .expect("Failed to build batch"); + writer.write(&batch).expect("write batch"); + } + writer.finish().expect("finish stream"); + } + buffer +} + +#[test] +fn test_arrow_inserter_insert_raw_ipc_stream() { + let test = TestConnection::new().expect("Failed to create test connection"); + + let table_def = TableDefinition::new("arrow_raw").add_required_column("id", SqlType::int()); + Catalog::new(&test.connection) + .create_table(&table_def) + .expect("Failed to create table"); + + let buffer = int_ipc_stream(&[vec![1, 2, 3]]); + let mut inserter = + ArrowInserter::new(&test.connection, &table_def).expect("Failed to create inserter"); + inserter.insert_raw(&buffer).expect("insert_raw"); + assert_eq!(inserter.execute().expect("execute"), 3); + + let sum = test + .execute_scalar_i64("SELECT SUM(id) FROM arrow_raw") + .expect("sum"); + assert_eq!(sum, 6); +} + +#[test] +fn test_arrow_inserter_small_flush_threshold_multiple_batches() { + let test = TestConnection::new().expect("Failed to create test connection"); + + let table_def = TableDefinition::new("arrow_flush").add_required_column("id", SqlType::int()); + Catalog::new(&test.connection) + .create_table(&table_def) + .expect("Failed to create table"); + + let buffer = int_ipc_stream(&[vec![1, 2], vec![3, 4], vec![5, 6]]); + let mut inserter = ArrowInserter::new(&test.connection, &table_def) + .expect("Failed to create inserter") + .with_flush_threshold(1); + inserter.insert_data(&buffer).expect("insert_data"); + assert_eq!(inserter.execute().expect("execute"), 6); + + let count = test + .execute_scalar_i64("SELECT COUNT(*) FROM arrow_flush") + .expect("count"); + assert_eq!(count, 6); + let sum = test + .execute_scalar_i64("SELECT SUM(id) FROM arrow_flush") + .expect("sum"); + assert_eq!(sum, 21); +} diff --git a/hyperdb-api/tests/grpc_streaming_tests.rs b/hyperdb-api/tests/grpc_streaming_tests.rs index d092c899..f72d1776 100644 --- a/hyperdb-api/tests/grpc_streaming_tests.rs +++ b/hyperdb-api/tests/grpc_streaming_tests.rs @@ -3,20 +3,20 @@ //! Integration tests for the gRPC streaming chunk API. //! -//! These tests verify that [`GrpcClient::execute_query_stream`] and the +//! These tests verify that `GrpcClientSync::execute_query_stream` and the //! streaming [`ArrowRowset::from_stream`] path wire up correctly against a //! live `hyperd`: //! -//! - the raw client's streaming API yields one or more `Bytes` chunks that -//! together reproduce the query result, and -//! - the high-level `Connection::execute_query` on a gRPC transport uses -//! the streaming path without regressing row counts or schema. +//! - the raw `GrpcClientSync` streaming API yields one or more `Bytes` +//! chunks, and the streaming `ArrowRowset::from_stream` path decodes them +//! to the full row count. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use hyperdb_api::{HyperProcess, ListenMode, Parameters, Result}; -/// Spins up `hyperd` in gRPC-only mode and returns (process, grpc url, db path). -/// The table `streaming_probe` is populated over gRPC-less server-side -/// `generate_series`, so no TCP INSERT is required for these tests. +/// Spins up `hyperd` in gRPC-only mode and returns the process and its gRPC URL. fn grpc_hyperd() -> Result<(HyperProcess, String)> { let mut params = Parameters::new(); params.set("log_dir", "test_results"); @@ -37,19 +37,19 @@ fn test_grpc_stream_chunks_decode_correctly() -> Result<()> { let config = hyperdb_api_core::client::grpc::GrpcConfig::new(&url); let mut client = hyperdb_api_core::client::grpc::GrpcClientSync::connect(config)?; - // A generate_series result is cheap and predictable — 50k i64 rows. + // A generate_series result is cheap and predictable — 50k rows. let stream = client.execute_query_stream("SELECT i FROM generate_series(1, 50000) AS s(i)")?; // Count how many raw Bytes chunks the server produced. struct Counter { inner: hyperdb_api_core::client::grpc::GrpcChunkStreamSync, - seen: usize, + seen: Arc, } impl hyperdb_api::ChunkSource for Counter { fn next_chunk(&mut self) -> hyperdb_api::Result> { match self.inner.next_chunk()? { Some(b) => { - self.seen += 1; + self.seen.fetch_add(1, Ordering::Relaxed); Ok(Some(b)) } None => Ok(None), @@ -57,9 +57,10 @@ fn test_grpc_stream_chunks_decode_correctly() -> Result<()> { } } + let chunks = Arc::new(AtomicUsize::new(0)); let source = Box::new(Counter { inner: stream, - seen: 0, + seen: Arc::clone(&chunks), }); let mut rowset = hyperdb_api::ArrowRowset::from_stream(source)?; @@ -68,6 +69,10 @@ fn test_grpc_stream_chunks_decode_correctly() -> Result<()> { total_rows += chunk.len(); } assert_eq!(total_rows, 50_000, "streamed row count must match query"); + assert!( + chunks.load(Ordering::Relaxed) >= 1, + "server produced no chunks" + ); Ok(()) } diff --git a/hyperdb-api/tests/remaining_features_tests.rs b/hyperdb-api/tests/remaining_features_tests.rs index 4cc1eb2b..fc34c940 100644 --- a/hyperdb-api/tests/remaining_features_tests.rs +++ b/hyperdb-api/tests/remaining_features_tests.rs @@ -15,7 +15,6 @@ mod common; use common::TestConnection; use hyperdb_api::{Catalog, FromRow, ServerVersion}; -// The derive macro is no longer re-exported from hyperdb_api; import directly. use hyperdb_api_derive::FromRow; // ============================================================================= @@ -308,12 +307,6 @@ fn test_fetch_all_as() { assert_eq!(users[2].name, "Carol"); } -// `test_from_row_tuple` was removed in v0.3.0 along with the blanket -// `(Option,)` … `(Option, Option, Option, Option)` -// `FromRow` impls. For ad-hoc tuple-shaped destructuring, callers -// should now use `Row::get(idx)` directly on each row, or define a -// struct with `#[derive(FromRow)]`. - // ============================================================================= // #17 cont: #[derive(FromRow)] parity with hand-written impl // ============================================================================= @@ -541,8 +534,9 @@ fn test_stream_as_multi_chunk() { .collect::>>() .expect("collect"); - let last = usize::try_from(ROWS).expect("row count fits usize") - 1; - assert_eq!(users.len(), ROWS as usize); + let rows = usize::try_from(ROWS).expect("row count fits usize"); + let last = rows - 1; + assert_eq!(users.len(), rows); assert_eq!(users[0].id, 1); assert_eq!(users[0].name, "name1"); assert!((users[0].score - 1.0).abs() < 0.001); diff --git a/hyperdb-bootstrap/CHANGELOG.md b/hyperdb-bootstrap/CHANGELOG.md index 2e1f9aec..94f2e1a9 100644 --- a/hyperdb-bootstrap/CHANGELOG.md +++ b/hyperdb-bootstrap/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [1.0.0-rc.3] - 2026-09-07 +Entries accumulated under `[Unreleased]` from 0.1.1 until this section was +dated, so several first shipped in earlier releases. The root `CHANGELOG.md` +records where each shipped: the `0.0.26359` pin in 0.7.1, and the PyPI wheels +with the `0.0.26479` pin in 1.0.0-rc.2. + ### Changed - **BREAKING: `hyperd` is now downloaded from the PyPI `tableauhyperapi` @@ -61,7 +66,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). the four download URLs. It therefore validates the exact pinned bytes rather than merely that the CDN serves *something* at that path. -- **BREAKING:** the minimum supported Rust version is now **1.88**, up from +- The minimum supported Rust version is now **1.88**, up from 1.81, and the crate is compiled with **edition 2024**. 1.88 is the version Red Hat Enterprise Linux 9.7 ships as `rust-toolset`. - **BREAKING:** the TLS crypto provider is now **ring** rather than AWS-LC. diff --git a/hyperdb-bootstrap/README.md b/hyperdb-bootstrap/README.md index 1bfc0c94..cb2f363e 100644 --- a/hyperdb-bootstrap/README.md +++ b/hyperdb-bootstrap/README.md @@ -11,32 +11,17 @@ Windows). This crate automates the "download the right wheel for your platform, extract `hyperd` out of it, put it somewhere useful" step so contributors and CI can bootstrap with a single command. -> **Why the PyPI wheels?** Two properties that no other distribution -> channel offers together. First, **the download URL is constructible** from -> nothing but the version and the platform's wheel tag: +> **Why the PyPI wheels?** The download URL is constructible from the version +> and the platform's wheel tag alone: > > ```text > https://files.pythonhosted.org/packages/py3/t/tableauhyperapi/tableauhyperapi-{version}-py3-none-{wheel_tag}.whl > ``` > -> Tableau's own download filenames embed an opaque build id (`r07abb490`) -> that cannot be derived from the version, so every bump previously required -> scraping an HTML page to discover it. Second, **PyPI publishes a sha256 per -> file**, so the pinned digests are now *read off an API* rather than produced -> by downloading four ~80 MB archives and hashing them by hand. -> -> The bytes are the same either way: the `hyperd` inside the -> `macosx_13_0_arm64` wheel is bit-identical (sha256 -> `aef5c819…6450e478`, 277,836,448 bytes) to the one this crate used to pull -> from Tableau. Same build, different envelope — and the wheels are ~3.6–4.5% -> smaller. Both report `minos 13.0`, so the `macosx_13_0` wheel tag is not a -> raised support floor and no contributor loses support. -> -> *Historical note, since older revisions of this file argued it at length:* -> Tableau's C++ `macos-arm64` zip did once ship an **x86_64** `hyperd`, which -> is why this crate used to take the Java bundle specifically. Tableau fixed -> that in `0.0.26225`; from that release the C++ and Java binaries are -> byte-identical, so the distinction no longer explains anything. +> and PyPI publishes a sha256 for every file at +> `https://pypi.org/pypi/tableauhyperapi//json`, which is where the +> digests pinned in `hyperd-version.toml` come from. The `macos-arm64` build is +> a native arm64 binary that requires macOS 13 or later. ## Install @@ -56,7 +41,7 @@ hyperdb-bootstrap download --dest /opt/hyperd # Force a re-download even if the version is already cached hyperdb-bootstrap download --force -# Install a specific release ad-hoc — just the version, no build id needed +# Install a specific release ad-hoc (version only) hyperdb-bootstrap download --version 0.0.26359 # Use an external pinned-version TOML instead of the baked-in default @@ -66,8 +51,11 @@ hyperdb-bootstrap download --version-file ./my-hyperd.toml # catch yanks/renames early. Exits non-zero on any failure. hyperdb-bootstrap verify +# Check a candidate bump before merging +hyperdb-bootstrap verify --version-file ./candidate.toml + # Print the installed binary's path -hyperdb-bootstrap which +hyperdb-bootstrap which # add --dest DIR for a custom root # Print the pinned release metadata hyperdb-bootstrap version @@ -79,32 +67,32 @@ four wheel tags are unchanged from `0.0.19484` through `0.0.26479`, so this work for any realistic ad-hoc pin or benchmark baseline. For a release whose wheel tags differ, write a full pin file and pass `--version-file`. -**Version-source precedence (highest → lowest):** - -1. `--version X` -2. `--version-file PATH` -3. `./hyperd-version.toml` (auto-discovered in current dir) -4. Compiled-in default shipped with this crate +**Version source** — `--version X` and `--version-file PATH` are mutually +exclusive. With neither, `download` uses `./hyperd-version.toml` if it exists in +the current directory, otherwise the pin compiled into this crate. ## Library ```rust -use hyperd_bootstrap::{install, InstallOptions, VersionSource}; - -let installed = install(InstallOptions { - dest_root: "/opt/hyperd".into(), - version_source: VersionSource::Builtin, - platform: None, // auto-detect - force: false, -})?; - -println!("hyperd: {}", installed.binary_path.display()); -# Ok::<(), hyperd_bootstrap::Error>(()) +use hyperdb_bootstrap::{install, InstallOptions, VersionSource}; + +fn main() -> Result<(), hyperdb_bootstrap::Error> { + let installed = install(InstallOptions { + dest_root: "/opt/hyperd".into(), + version_source: VersionSource::Builtin, + platform: None, // auto-detect + force: false, + })?; + + println!("hyperd: {}", installed.binary_path.display()); + Ok(()) +} ``` The library is blocking (no async runtime) and has no `tokio` dependency, so it can be dropped into build scripts, `postinstall` hooks, or -synchronous applications. +synchronous applications. Downloads and `verify` shell out to `curl`, which +must be on `PATH`; minimal container images often need it installed. ## Build-time guarantees @@ -146,8 +134,7 @@ tag produces a **silent 404** rather than a clear error. Keeping them in ## Install layout -The versioned cache directory and `current/VERSION` are keyed on the version -alone — there is no build id in the path any more. +The versioned cache directory and `current/VERSION` are keyed on the version alone. ```text / diff --git a/hyperdb-bootstrap/src/install.rs b/hyperdb-bootstrap/src/install.rs index 131e367f..92045486 100644 --- a/hyperdb-bootstrap/src/install.rs +++ b/hyperdb-bootstrap/src/install.rs @@ -40,7 +40,7 @@ pub enum VersionSource { /// Configuration passed to [`install`]. #[derive(Debug, Clone)] pub struct InstallOptions { - /// Root directory under which `/` and `current/` are created. + /// Root directory under which `/` and `current/` are created. pub dest_root: PathBuf, /// Which release to resolve. See [`VersionSource`]. pub version_source: VersionSource, diff --git a/hyperdb-compile-check/CHANGELOG.md b/hyperdb-compile-check/CHANGELOG.md index 5e16fe04..8c3f1484 100644 --- a/hyperdb-compile-check/CHANGELOG.md +++ b/hyperdb-compile-check/CHANGELOG.md @@ -32,8 +32,6 @@ surface, and consumers should depend on `hyperdb-api-derive` instead. ### Changed -- **BREAKING:** the minimum supported Rust version is now **1.88**, up from +- The minimum supported Rust version is now **1.88**, up from 1.81, and the crate is compiled with **edition 2024**. 1.88 is the version Red Hat Enterprise Linux 9.7 ships as `rust-toolset`. -- The `arrow` dependency moved from **58** to **59**, in lockstep with - `hyperdb-api`. diff --git a/hyperdb-compile-check/src/db.rs b/hyperdb-compile-check/src/db.rs index 92ed4136..8cc1b4a9 100644 --- a/hyperdb-compile-check/src/db.rs +++ b/hyperdb-compile-check/src/db.rs @@ -25,26 +25,24 @@ pub struct CompileTimeDb { // // The `parking_lot::Mutex` is what makes this safe: it ensures only one // proc-macro expansion thread touches the connection at a time. Each `query_as!` -// site locks, runs one dry-run (~7ms), unlocks. They serialize on the one +// site locks, runs one dry-run, unlocks. They serialize on the one // connection rather than each getting their own (a connection-pool approach -// would work too but adds startup cost for negligible gain at v1 scale). +// would work too but adds startup cost for negligible gain). // // Neither `HyperProcess` nor `Connection` is `Send`/`Sync` in the public API. // We implement both here because `OnceLock` requires `T: Send + Sync`. // The `Mutex` upholds the invariant that only one thread ever accesses the // fields — making the `Send`/`Sync` impls sound. // -// REVISIT: if `HyperProcess`/`Connection` are made `Send` upstream, remove -// these impls and let the compiler derive them. +// If `HyperProcess`/`Connection` become `Send` upstream, these impls can be +// removed and derived by the compiler. // // # Why `parking_lot::Mutex` instead of `std::sync::Mutex` // -// Proc-macros routinely call `panic!` to emit a `compile_error!`. A -// `std::sync::Mutex` poisons on the first panic, causing every subsequent -// macro invocation in the same crate to receive `PoisonError` regardless of -// whether they have anything to do with the failing site. `parking_lot::Mutex` -// never poisons — lock acquisition always succeeds after the panicking thread -// releases the lock, so a bad `query_as!` site doesn't cascade. +// A panic inside a dry-run must not poison the lock for every later macro +// invocation; `parking_lot::Mutex` never poisons, so lock acquisition always +// succeeds after the panicking thread releases the lock and a bad `query_as!` +// site doesn't cascade. // SAFETY: `OnceLock` requires `Send`; safe because the `Mutex` guarantees // exclusive access — `CompileTimeDb` is never touched without holding the lock. @@ -101,9 +99,10 @@ impl CompileTimeDb { let db_path = log_dir.path().join("compile_check.hyper"); let conn = Connection::new(&process, &db_path, CreateMode::CreateAndReplace)?; - // Keep `log_dir` alive as long as the process — drop it with the struct. - // We leak the TempDir intentionally: `CompileTimeDb` is `'static` (stored - // in a static); the OS will clean up the temp dir on process exit. + // `CompileTimeDb` lives in a `static` and is never dropped, so the + // directory (hyperd logs plus the scratch database) outlives the process + // and is left to the system's temp-file reaper. Forgetting the `TempDir` + // keeps it from being removed while hyperd is still using it. std::mem::forget(log_dir); Ok(Self { diff --git a/hyperdb-compile-check/src/dry_run.rs b/hyperdb-compile-check/src/dry_run.rs index b27a43cb..db30ccdd 100644 --- a/hyperdb-compile-check/src/dry_run.rs +++ b/hyperdb-compile-check/src/dry_run.rs @@ -6,7 +6,7 @@ //! Wraps arbitrary user SQL in a CTE and runs it against the shared //! `CompileTimeDb`, returning the `ResultSchema` without touching any rows. //! -//! # Critical: query execution is lazy (Phase 0 S6) +//! # Query execution is lazy //! //! `Connection::execute_query()` does NOT run the query on the TCP transport. //! The query only executes — and server errors / the `RowDescription` (schema) @@ -32,7 +32,7 @@ pub fn dry_run(db: &mut CompileTimeDb, user_sql: &str) -> Result { let wrapped = format!("WITH __hdb_q AS ({user_sql}) SELECT * FROM __hdb_q LIMIT 0"); let mut rowset = db.conn.execute_query(&wrapped)?; - // Force execution (Phase 0 S6): LIMIT 0 returns Ok(None) from next_chunk + // Force execution: LIMIT 0 returns Ok(None) from next_chunk // but populates the schema cache first. rowset.next_chunk()?; diff --git a/hyperdb-compile-check/src/error_extract.rs b/hyperdb-compile-check/src/error_extract.rs index 5fee330e..a7d2fe2d 100644 --- a/hyperdb-compile-check/src/error_extract.rs +++ b/hyperdb-compile-check/src/error_extract.rs @@ -3,9 +3,9 @@ //! SQLSTATE-based Hyper error classification. //! -//! Phase 0 spike S5 confirmed that Hyper returns PostgreSQL SQLSTATE codes as a -//! structured field on `Error::Server`. We branch on the **stable code**, not -//! fragile message text, so Hyper message wording changes don't break us. +//! Hyper reports PostgreSQL SQLSTATE codes as a structured field on +//! `Error::Server`. Classification branches on the **stable code**, not +//! fragile message text, so Hyper message wording changes don't break it. //! //! Relevant codes: //! - `42P01` — undefined_table → extract the table name and seed-and-retry @@ -46,8 +46,8 @@ pub fn classify(err: &Error) -> ErrorClass { /// Extract a double-quoted or single-quoted identifier from a Hyper error /// message, with a fallback to the full message. /// -/// Phase 0 output for 42P01: `ERROR: table "ghosts" does not exist (42P01)` -/// Phase 0 output for 42703: `ERROR: unknown column 'ema1l' (42703)` +/// Hyper formats 42P01 as `ERROR: table "ghosts" does not exist (42P01)` and +/// 42703 as `ERROR: unknown column 'ema1l' (42703)`. fn extract_quoted_identifier(message: &str, _kind: &str) -> String { // Try double-quoted first ("ghosts"), then single-quoted ('ema1l'). if let Some(name) = extract_between(message, '"', '"') { diff --git a/hyperdb-mcp/README.md b/hyperdb-mcp/README.md index 344c381a..d2c548d2 100644 --- a/hyperdb-mcp/README.md +++ b/hyperdb-mcp/README.md @@ -372,15 +372,16 @@ query_data(data: '[{"region":"West","revenue":1200},...]', sql: 'SELECT region, Ingest a file and run a SQL query in a single call. Streams from disk — handles files of any size. ```text -query_file(path: '/tmp/sales.parquet', sql: 'SELECT TOP 10 * FROM sales ORDER BY amount DESC') +query_file(path: '/tmp/sales.parquet', sql: 'SELECT TOP 10 * FROM data ORDER BY amount DESC') ``` | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `path` | string | yes | Path to CSV / JSON / JSONL / Parquet / Arrow IPC file | | `sql` | string | yes | SQL query to run | -| `table_name` | string | no | Table name — defaults to filename stem | +| `table_name` | string | no | Table name for use in SQL — defaults to `"data"` | | `schema` | object | no | Partial column-name → type map (see [Schema Overrides](#schema-overrides)) | +| `json_extract_path` | string | no | Dot-separated path to a nested array in a JSON wrapper file (numeric segments index arrays) | ### Database Tools @@ -399,6 +400,8 @@ load_data(table: 'customers', data: '[{"id":1,"name":"Alice"},...]') | `format` | string | no | `"json"` or `"csv"` — auto-detected | | `mode` | string | no | `"replace"` (default) or `"append"` | | `schema` | object | no | Partial column-name → type map (see [Schema Overrides](#schema-overrides)) | +| `database` | string | no | Target database: `"local"` (default), `"persistent"`, or a writable attached alias | +| `persist` | boolean | no | Shorthand for `database: "persistent"`; `database` wins if both are set | #### `load_file` @@ -412,8 +415,12 @@ load_file(table: 'orders', path: '/tmp/orders.csv') |-----------|------|----------|-------------| | `table` | string | yes | Table name | | `path` | string | yes | Path to CSV / JSON / JSONL / Parquet / Arrow IPC file | -| `mode` | string | no | `"replace"` (default) or `"append"` | +| `mode` | string | no | `"replace"` (default), `"append"`, or `"merge"` (upsert by `merge_key`; new columns auto-added) | +| `merge_key` | string or array | for merge | Column(s) matching incoming rows to existing rows; rejected for other modes | | `schema` | object | no | Partial column-name → type map (see [Schema Overrides](#schema-overrides)) | +| `json_extract_path` | string | no | Dot-separated path to a nested array in a JSON wrapper file | +| `database` | string | no | Target database: `"local"` (default), `"persistent"`, or a writable attached alias | +| `persist` | boolean | no | Shorthand for `database: "persistent"`; `database` wins if both are set | When you're unsure of the right types — or recovering from a previous `SCHEMA_MISMATCH` — call [`inspect_file`](#inspect_file) first. It reports the @@ -442,9 +449,63 @@ load_iceberg(table: 'sales', path: '/lake/warehouse/db/sales') Schema overrides are not accepted — hyperd derives the schema from the Iceberg table metadata. +#### `load_files` + +Load several files in parallel, each into its own table. One entry's failure does not abort the others. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `files` | array | yes | Entries with `table`, `path`, and optional `mode` (`"replace"` or `"append"`; `"merge"` and `merge_key` are rejected), `schema`, and `json_extract_path` | +| `concurrency` | integer | no | Maximum concurrent ingests; defaults to `min(files, 8)`, capped at 16 | +| `database` | string | no | Target database for every entry: `"local"` (default), `"persistent"`, or a writable attached alias | +| `persist` | boolean | no | Shorthand for `database: "persistent"`; `database` wins if both are set | + +#### `attach_database` / `detach_database` / `list_attached_databases` + +Attach another `.hyper` file for the session and reference its tables as `.public.
`. `detach_database` takes only `alias`; `list_attached_databases` lists the current attachments. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `alias` | string | yes | SQL identifier; cannot be `local` | +| `path` | string | yes | Absolute path to the `.hyper` file | +| `kind` | string | no | Attachment kind; defaults to `"local_file"`, the only kind supported | +| `writable` | boolean | no | Default `false`; required for writes and `copy_query` targets | +| `on_missing` | string | no | `"error"` (default) or `"create"` (needs `writable: true`; the parent directory must exist) | + +#### `copy_query` + +Run a read-only `SELECT` / `WITH` / `VALUES` statement and land the result in a target table, possibly in another database. + +```text +copy_query(sql: 'SELECT * FROM src.public.customers', target_table: 'customers_copy', mode: 'create', temp_attach: [...]) +``` + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `sql` | string | yes | Read-only statement; reference attached tables as `.public.
` | +| `target_table` | string | yes | Unqualified destination table in the `public` schema | +| `mode` | string | yes | `"create"` (error if it exists), `"append"` (error if missing), or `"replace"` | +| `target_database` | string | no | Destination alias; defaults to local. Attached aliases must be writable | +| `temp_attach` | array | no | Databases attached for this call only and detached afterwards | + +#### `set_table_metadata` + +Record prose metadata for a table that already has a catalog entry. Unset fields stay unchanged; an empty string clears a field. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `table` | string | yes | Table name | +| `source_url` | string | no | Where the data came from | +| `source_description` | string | no | What the dataset contains | +| `purpose` | string | no | Why the data is loaded | +| `license` | string | no | License or attribution requirements | +| `notes` | string | no | Refresh instructions, gotchas, caveats | +| `data_url` | string | no | Machine-actionable download URL for the raw data | +| `database` | string | no | Catalog to write: local and persistent share one catalog; a writable attached alias uses its own | + #### `query` -Run a **read-only** SQL query against local (default), persistent, or an attached database. Accepts `SELECT`, `WITH`, `EXPLAIN`, `SHOW`, `VALUES`. For DDL/DML use `execute`. +Run a **read-only** SQL query against local (default), persistent, or an attached database. Accepts `SELECT`, `WITH`, `EXPLAIN`, `SHOW`, `VALUES`. For DDL/DML use `execute`. Results are capped at 10,000 rows; larger results return `truncated: true` plus `total_rows` — add `LIMIT`/`GROUP BY` or use `export` for the full set. ```text query(sql: 'SELECT c.name, SUM(o.amount) FROM orders o JOIN customers c ON o.customer_id = c.id GROUP BY c.name') @@ -495,7 +556,7 @@ sample(table: 'orders', n: 10) #### `inspect_file` -Dry-run schema inference on a CSV, Parquet, or Arrow IPC file **without ingesting +Dry-run schema inference on a CSV, JSON, JSONL, Parquet, or Arrow IPC file **without ingesting it**. Returns the exact schema `load_file` / `query_file` would use (including the full-file numeric widening pass) plus per-column `min`, `max`, `null_count`, and `sample_values`. Nothing is written to Hyper and `hyperd` is not even @@ -712,6 +773,8 @@ watch_directory(path: '/tmp/inbox', table: 'events') unwatch_directory(path: '/tmp/inbox') ``` +`watch_directory` also accepts `max_concurrent` (files ingested in parallel; default 4, capped at 32), `database`, and `persist`. + **Producer protocol (`.ready` sentinel):** 1. Write data file (e.g. `foo.csv`) and close it. @@ -729,6 +792,10 @@ Key properties: ### Utility Tools +#### `get_readme` + +Returns the LLM-facing tool index, parameter rules, and usage examples. It takes no parameters and is allowed in read-only mode. + #### `status` Returns MCP/native/API installation identity, daemon and Hyper connection facts, @@ -932,7 +999,7 @@ execute(sql: [ Both statements run inside a single Hyper transaction — they commit together or both roll back. No race window between them. -> **Tip:** For file-based upserts (merging updated data from a CSV/JSON file into an existing table), use `load_file` with `mode: "merge"` and a `merge_key` instead of writing manual SQL — it handles the UPDATE/INSERT logic automatically and also auto-adds new columns. +> **Tip:** For file-based upserts (merging updated data from a CSV/JSON file into an existing table), use `load_file` with `mode: "merge"` and a `merge_key` instead of writing manual SQL — it handles the delete-and-reinsert by key automatically and also auto-adds new columns. ### Transactions @@ -1011,6 +1078,10 @@ Errors include a machine-readable code and a suggestion: | `READ_ONLY_VIOLATION` | Mutating op in read-only mode | Use `query_*` / `inspect_file`, or restart without `--read-only` | | `RESOURCE_BUSY` | The reserved persistent attachment hit file contention (SQLSTATE 55006) | Run `hyperdb-mcp doctor`; compare client/daemon identities; close the possible owner (Hyper, Tableau, or another process), or copy/select another `.hyper` file; retry | | `CONNECTION_LOST` | `hyperd` crashed or wire protocol desynchronized | Retry — the server tears down the engine and reconnects on the next call | +| `INVALID_ARGUMENT` | Bad parameter (relative path, duplicate saved-query name, invalid alias) | Fix the argument per the message | +| `EMPTY_DATA` | Input has zero rows or columns | Check the source data | +| `PERMISSION_DENIED` | Filesystem permission denied, or `export` with `overwrite: false` hit an existing file | Choose another path or fix permissions | +| `INTERNAL_ERROR` | Unexpected server failure | Retry; report with `hyperdb-mcp doctor` output | Server-returned errors include a machine-readable `code`, a `message`, and a `suggestion` with concrete retry guidance. The `SCHEMA_MISMATCH` suggestion for @@ -1044,4 +1115,3 @@ the executable or its containing directory, or install it under an ancestor's - **[hyperdb-api](../hyperdb-api/)** — Core Rust API (sync/async connections, inserter, query) - **[DEVELOPMENT.md](DEVELOPMENT.md)** — Internal architecture, design decisions, contributor guide - **[ROADMAP.md](ROADMAP.md)** — Forward-looking design sketches for features that aren't built yet -- **[Design Spec](../docs/specs/hyperdb-mcp-design.md)** — Full design document diff --git a/hyperdb-mcp/build.rs b/hyperdb-mcp/build.rs index 758f1d35..56be8fd6 100644 --- a/hyperdb-mcp/build.rs +++ b/hyperdb-mcp/build.rs @@ -27,7 +27,7 @@ //! identifies them, so no timestamp is appended. //! //! Falls back to `unknown` on any failure (no git binary, not a repo, -//! detached state, etc.) so the crate still builds for consumers who +//! etc.) so the crate still builds for consumers who //! obtained the source as a tarball. //! //! `Cargo.lock` is excluded from the dirty check via a git pathspec — @@ -41,9 +41,9 @@ fn main() { // one directory into the workspace `.git`. println!("cargo:rerun-if-changed=../.git/HEAD"); println!("cargo:rerun-if-changed=../.git/refs/heads"); - // Changes in the git index also affect dirty-ness (staging/unstaging - // files flips the tree's dirty state without HEAD moving). Touching - // `.git/index` reruns us so the `-dirty` marker stays honest. + // Staging or unstaging rewrites `.git/index`, which reruns this script. + // Unstaged edits to tracked files do not, so the `-dirty` marker is + // best-effort until HEAD, a branch ref, or the index next changes. println!("cargo:rerun-if-changed=../.git/index"); let hash = Command::new("git") @@ -60,18 +60,12 @@ fn main() { // permanently mark every build dirty even though the source tree // itself matches HEAD. // - // We also exclude `Cargo.lock` from the dirty check via a pathspec. - // Rationale: release-please bumps `[workspace.package].version` in - // `Cargo.toml` but does NOT update `Cargo.lock`. When CI checks - // out the release tag and runs `cargo build --release`, cargo - // silently reconciles the lockfile in-place, dirtying the - // worktree. Without the exclude, every release binary gets - // stamped `-dirty-` even though the source is - // pristine. Cargo.lock churn at build time is not a logical - // source change and shouldn't trip the marker. Any *other* - // modified tracked file (including a deliberately-edited - // Cargo.lock alongside a code change) still trips it because - // `git status --porcelain` will list it. + // `Cargo.lock` is excluded from the dirty check via a pathspec. The + // release-please workflow syncs `Cargo.lock` in the release PR; this + // exclusion is a safety net for a release commit whose lockfile is still + // stale, which `cargo build` would rewrite in place. Any other modified + // tracked file, including a `Cargo.lock` edited alongside a code change, + // still trips the marker. // // `:(exclude,top)` is git's portable long-form pathspec magic: // `top` anchors at the repo root (so a hypothetical sub-crate diff --git a/hyperdb-mcp/src/chart.rs b/hyperdb-mcp/src/chart.rs index cca1af6e..49e3dec8 100644 --- a/hyperdb-mcp/src/chart.rs +++ b/hyperdb-mcp/src/chart.rs @@ -1097,8 +1097,8 @@ enum TemporalKind { /// How to interpret the x column when extracting f64 axis positions. /// -/// Drives [`group_series`] and the corresponding rendering branch in -/// [`line_or_scatter`] / [`draw_bar`]. `Temporal` is the new mode added +/// Drives `group_series` and the corresponding rendering branch in +/// [`line_or_scatter`] / [`draw_bar`]. `Temporal` is the mode /// for proportional time-axis rendering: x positions are real Unix /// epoch seconds (so 6 hours apart on the wire are 6 hours apart on /// the chart), and tick labels are formatted via chrono. diff --git a/hyperdb-mcp/src/engine.rs b/hyperdb-mcp/src/engine.rs index daad923d..4354e3f6 100644 --- a/hyperdb-mcp/src/engine.rs +++ b/hyperdb-mcp/src/engine.rs @@ -19,7 +19,7 @@ //! detects the [`crate::error::ErrorCode::ConnectionLost`] error, drops the //! engine, and transparently re-creates it on the next call. This auto-reconnect //! path covers both transport-level failures and the `"desynchronized"` state -//! surfaced by the `hyper-client` layer's bounded drain. +//! surfaced by the bounded drain in `hyperdb_api_core::client`. //! //! # Workspace Model //! @@ -329,8 +329,7 @@ fn create_table_statements( /// Closures receive this rather than `&Engine` for a reason: the guard /// holds `&mut Connection`, so the borrow checker will not let the same /// connection be driven around the transaction. That statically rules out -/// the "statement escaped the transaction" bug class that the previous -/// `&self` + `*_unguarded` shape could only address by convention. +/// the "statement escaped the transaction" bug class. #[derive(Debug)] pub struct EngineTransaction<'conn> { txn: Transaction<'conn>, @@ -429,10 +428,8 @@ pub struct Engine { /// User-data persistent database. Attached under alias `"persistent"` /// during [`Engine::new`]. `None` in `--ephemeral-only` mode. persistent_path: Option, - /// `true` when the persistent `.hyper` file was just created during - /// engine construction (so the catalog-seed step should fire). Reset - /// to `false` after the server consumes it via - /// [`Self::take_persistent_was_created`]. + /// `true` when engine construction created the persistent `.hyper` + /// file. Read via [`Self::persistent_was_just_created`]; never reset. persistent_was_created: bool, /// Cached "_table_catalog exists in ``" probes, keyed by /// canonical alias (lowercase). Populated on first call to @@ -857,9 +854,9 @@ impl Engine { let set_sql = format!("SET schema_search_path = '{}'", alias.replace('\'', "''")); self.execute_command(&set_sql)?; - // `AssertUnwindSafe` is sound for the same reason it is in - // `ScopedSearchPath`'s `Drop`: the only state that outlives the - // unwind is a session variable we are about to overwrite anyway. + // `AssertUnwindSafe` is sound here: after a panic in `f`, the only + // work done on `self` is the `SET` that restores the search path, + // and the panic is then re-raised unchanged. let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(self))); let restore_sql = format!( @@ -922,10 +919,9 @@ impl Engine { } /// `true` when this engine just created the persistent `.hyper` file - /// during construction. The server consumes this signal once to - /// decide whether to seed `_table_catalog`; subsequent reads stay - /// `true` (the flag isn't reset — it's a fact about the engine's - /// startup, not a one-shot signal). + /// during construction. + /// + /// The flag is a fact about the engine's startup and is never reset. pub fn persistent_was_just_created(&self) -> bool { self.persistent_was_created } @@ -1033,7 +1029,7 @@ impl Engine { /// functionally equivalent to a successful rollback). /// /// This is the correctness primitive for ingest operations: it lets - /// per-row `INSERT` loops (Parquet, Arrow, JSON) leave zero partial data + /// the per-row `INSERT` loop of JSON ingest leave zero partial data /// on failure. The CSV `COPY FROM` path is already atomic at the /// statement level, but wrapping it in a transaction costs nothing and /// makes per-row INSERT loops atomic across the whole batch. @@ -1560,10 +1556,9 @@ impl Engine { /// /// Avoids the `Catalog::has_table` probe entirely — we just run the sample /// SELECT first and translate a Hyper "table does not exist" error into - /// our own [`ErrorCode::TableNotFound`]. This sidesteps the old pattern - /// where a racy `has_table` silently returning `Err` would be rewritten - /// to `false` and surface as a spurious `TableNotFound` for tables that - /// actually exist. + /// our own [`ErrorCode::TableNotFound`]. A racy `has_table` that silently + /// returned `Err` would otherwise be rewritten to `false` and surface as a + /// spurious `TableNotFound` for tables that actually exist. /// /// # Errors /// @@ -1892,9 +1887,8 @@ fn row_value_to_json(row: &hyperdb_api::Row, idx: usize, sql_type: &SqlType) -> .unwrap_or(Value::Null); } if oid_val == oids::NUMERIC.0 { - // `Row` is schema-aware as of the upstream NUMERIC fix — it - // carries an `Arc` and `row.get::()` - // reads the scale from the column's + // `Row` is schema-aware: it carries an `Arc` and + // `row.get::()` reads the scale from the column's // `SqlType::Numeric { precision, scale }` descriptor before // dispatching on the buffer length. That covers all three // NUMERIC wire shapes the server can send on a query result: @@ -1903,12 +1897,8 @@ fn row_value_to_json(row: &hyperdb_api::Row, idx: usize, sql_type: &SqlType) -> // * 16-byte `BigNumeric` (precision > 18) // * Arrow `Decimal128`/`Decimal256` (gRPC transport) // - // Prior to the upstream fix, `type_modifier` was being dropped - // during `RowDescription` parsing so the scale presented here - // was always `0`, the 8-byte form wasn't decodable at all, and - // `AVG` results fell through to `Null`. All of that is now - // handled inside `hyperdb-api`; this function only needs to pick - // the JSON shape. + // Scale and width decoding are handled inside `hyperdb-api`; this + // function only needs to pick the JSON shape. // // `Numeric::to_string()` uses the decoded scale and is exact. // Ordinary query results retain their established compact JSON @@ -2121,11 +2111,6 @@ pub fn resolve_log_dir(persistent_db_path: Option<&str>) -> PathBuf { } } -/// Build the `{name, columns, row_count}` JSON for a single table, shared -/// between [`Engine::describe_tables`] (bulk) and [`Engine::describe_table`] -/// (single) so both paths emit byte-identical shapes. A missing table -/// surfaces as the underlying Hyper "relation does not exist" error; single- -/// table callers should run it through `translate_table_missing`. /// Describe columns of `table_name` in attached database `db_alias` by /// querying that database's `pg_catalog.pg_attribute` directly. Used when /// the connection-bound `Catalog` API can't see the target database. @@ -2163,6 +2148,10 @@ fn describe_columns_via_pg_catalog( .collect()) } +/// Build the `{name, columns, row_count}` JSON for a single table, shared +/// between [`Engine::describe_tables`] (bulk) and [`Engine::describe_table`] +/// (single) so both paths emit byte-identical shapes. Callers check existence +/// first; a missing table surfaces as the underlying catalog error. fn describe_table_with_catalog(catalog: &Catalog<'_>, name: &str) -> Result { let def = catalog.get_table_definition(name).map_err(McpError::from)?; let row_count = catalog.get_row_count(name).unwrap_or(0); @@ -2207,7 +2196,6 @@ fn translate_table_missing(err: McpError, table_name: &str) -> McpError { /// `SHOW`, or `VALUES`. Anything else (`CREATE`, `INSERT`, `UPDATE`, `DELETE`, /// `DROP`, `ALTER`, `COPY`, ...) is considered mutating. /// -/// The check is a simple prefix match after trimming and upper-casing the first /// Checks whether the first SQL keyword indicates a read-only statement. /// /// Strips leading whitespace and SQL comments (line `--` and block `/* */`) @@ -2648,11 +2636,10 @@ mod statement_helper_tests { mod endpoint_description_tests { use super::*; - /// The TCP shape every `hyperdb-mcp` session uses today: both the daemon - /// (`daemon::run` sets `TransportMode::Tcp` explicitly) and the local - /// fallback (`HyperProcess` defaults to TCP on every platform) hand the - /// engine a `host:port` string. The descriptor must round-trip back to - /// the `tab.tcp://` form `hyperd` sent over its callback connection. + /// The TCP shape a private `hyperd` uses (`--no-daemon` or the daemon + /// fallback; `HyperProcess` defaults to TCP on every platform): the + /// engine receives a `host:port` string. The descriptor must round-trip + /// back to the `tab.tcp://` form `hyperd` sent over its callback connection. #[test] fn tcp_endpoint_decomposes_into_host_port_and_descriptor() { let described = describe_endpoint("127.0.0.1:64687"); diff --git a/hyperdb-mcp/src/error.rs b/hyperdb-mcp/src/error.rs index 380de995..b5d2edef 100644 --- a/hyperdb-mcp/src/error.rs +++ b/hyperdb-mcp/src/error.rs @@ -39,7 +39,7 @@ pub enum ErrorCode { /// The connection to `hyperd` was lost (crash, broken pipe, EOF) or /// the wire protocol fell out of sync (a bounded drain exhausted /// without reaching `ReadyForQuery`, surfacing as a - /// `"desynchronized"` error message from the `hyper-client` layer). + /// `"desynchronized"` error message from the `hyperdb_api_core::client` layer). /// Either way, the connection is unusable and the MCP server will /// automatically tear down the [`crate::engine::Engine`] and /// reconnect on the next call. @@ -289,7 +289,7 @@ impl From for McpError { /// `PostgreSQL` client produces when `hyperd` crashes or is killed /// mid-transaction. /// -/// 2. **Wire-protocol desync** — the `hyper-client` layer marks a +/// 2. **Wire-protocol desync** — the `hyperdb_api_core::client` layer marks a /// connection `desynchronized` when its bounded drain exhausts the /// `POST_ERROR_DRAIN_CAP` budget without reaching `ReadyForQuery` or /// hits an I/O error mid-drain. Subsequent operations on that diff --git a/hyperdb-mcp/src/export.rs b/hyperdb-mcp/src/export.rs index 01d6d596..5495f403 100644 --- a/hyperdb-mcp/src/export.rs +++ b/hyperdb-mcp/src/export.rs @@ -34,9 +34,9 @@ use serde_json::{Map, Value}; /// Specifies what to export and where. /// /// For row-oriented formats (`csv`, `parquet`, `arrow_ipc`, `iceberg`) -/// exactly one of `sql` or `table` must be provided; `sql` takes priority -/// if both are set. For `hyper` format both are ignored — every user -/// table in the workspace is copied into a new `.hyper` file. +/// provide `sql` or `table`; `sql` takes priority if both are set. For +/// `hyper` format both are ignored — every user table in the workspace +/// is copied into a new `.hyper` file. #[derive(Debug, Default)] pub struct ExportOptions { /// A SELECT query whose results will be exported. Ignored when @@ -166,12 +166,13 @@ pub fn export_to_file(engine: &Engine, opts: &ExportOptions) -> Result Result<(), McpError> { let bad = key.is_empty() || !key @@ -298,8 +299,8 @@ fn export_csv( /// `COPY ... WITH (format => 'parquet')`. Types (NUMERIC precision, /// DATE, TIMESTAMP, non-null flags, ...) are preserved exactly — /// hyperd writes its own Arrow schema from the query's `RowDescription`, -/// bypassing the JSON round-trip the previous Rust-side pipeline used. -/// Caller can override via `format_options` (e.g. `{"compression": +/// bypassing any JSON round-trip. +/// Caller can override via `format_options` (e.g. `{"codec": /// "zstd", "rows_per_row_group": 100000}`). fn export_parquet( engine: &Engine, @@ -355,8 +356,6 @@ fn export_arrow_ipc( /// or file already exists there and `overwrite` is true, we remove /// it first — hyperd's `COPY TO` refuses to write into an existing /// non-empty Iceberg location. -/// - The SELECT must return at least one row. An empty query succeeds -/// but produces an empty table metadata file. fn export_iceberg( engine: &Engine, sql: &str, @@ -432,25 +431,6 @@ fn walk_dir_size(dir: &std::path::Path) -> std::io::Result { Ok(total) } -/// Export the workspace tables as a new `.hyper` file. Issues -/// `CREATE DATABASE` + `ATTACH DATABASE` against the target path and -/// populates it with one constraint-preserving copy per user table. -/// -/// We can't just `std::fs::copy(workspace, target)` because on Windows -/// hyperd holds the workspace file open with an exclusive lock, and -/// Windows blocks any concurrent open of a locked file (Unix allows it -/// via shared handle semantics). Going through hyperd keeps this -/// cross-platform at the cost of only copying tables — views, -/// sequences, and other catalog objects in the source are not -/// reproduced. That's acceptable for the current callers (LLMs -/// exporting workspace data for Tableau Desktop), but documented here -/// so a future caller that needs full catalog fidelity knows why. -/// -/// Within a table the copy is faithful: `NOT NULL`, `DEFAULT`, -/// `ASSUMED PRIMARY KEY`, and `ASSUMED UNIQUE` all survive. Anything that -/// could not be reproduced comes back in -/// [`ExportResult::schema_fidelity`] so the caller can say so out loud -/// instead of handing back a quietly degraded backup. /// True when a filesystem error means "another process is holding this file /// open" rather than "you lack permission to touch it". /// @@ -471,6 +451,25 @@ fn is_file_in_use(err: &std::io::Error) -> bool { || (cfg!(windows) && matches!(err.raw_os_error(), Some(32 | 33))) } +/// Export the workspace tables as a new `.hyper` file. Issues +/// `CREATE DATABASE` + `ATTACH DATABASE` against the target path and +/// populates it with one constraint-preserving copy per user table. +/// +/// We can't just `std::fs::copy(workspace, target)` because on Windows +/// hyperd holds the workspace file open with an exclusive lock, and +/// Windows blocks any concurrent open of a locked file (Unix allows it +/// via shared handle semantics). Going through hyperd keeps this +/// cross-platform at the cost of only copying tables — views, +/// sequences, and other catalog objects in the source are not +/// reproduced. That's acceptable for the current callers (LLMs +/// exporting workspace data for Tableau Desktop), but documented here +/// so a future caller that needs full catalog fidelity knows why. +/// +/// Within a table the copy is faithful: `NOT NULL`, `DEFAULT`, +/// `ASSUMED PRIMARY KEY`, and `ASSUMED UNIQUE` all survive. Anything that +/// could not be reproduced comes back in +/// [`ExportResult::schema_fidelity`] so the caller can say so out loud +/// instead of handing back a quietly degraded backup. fn export_hyper( engine: &Engine, path: &str, diff --git a/hyperdb-mcp/src/ingest.rs b/hyperdb-mcp/src/ingest.rs index 285aa7b4..9a98b22c 100644 --- a/hyperdb-mcp/src/ingest.rs +++ b/hyperdb-mcp/src/ingest.rs @@ -578,7 +578,7 @@ where /// whatever the Rust inferrer emitted (`"INT"`, `"DOUBLE /// PRECISION"`, `"NUMERIC(15, 2)"`). Comparing through /// [`crate::schema::map_hyper_type`] collapses all known aliases -/// into a single [`SqlType`]. +/// into a single [`hyperdb_api::SqlType`]. /// /// If *either* side fails to parse (returns `None`), we err on the /// side of permissive: treat the pair as compatible and let Hyper diff --git a/hyperdb-mcp/src/lib.rs b/hyperdb-mcp/src/lib.rs index ceb25e04..cbc7a805 100644 --- a/hyperdb-mcp/src/lib.rs +++ b/hyperdb-mcp/src/lib.rs @@ -33,6 +33,8 @@ //! //! The crate is layered bottom-up: //! +//! - `daemon` — Single-instance daemon that shares one `hyperd` across MCP clients (discovery, spawn, health listener). +//! - `paths` — Platform-standard default path for the persistent database. //! - `error` — Structured error codes with recovery suggestions for LLM self-correction. //! - `attach` — Registry of additional `.hyper` databases attached to the primary //! workspace for cross-database JOINs and `copy_query`. Replays attachments @@ -44,17 +46,19 @@ //! query execution across the local database and optional persistent database. //! - `ingest` — Loads inline JSON (row-by-row INSERT) and CSV (`COPY FROM`) into Hyper. //! - `ingest_arrow` — Loads Parquet and Arrow IPC files via the Arrow crate. +//! - `lakehouse` — Loads Apache Iceberg tables via hyperd's native `external(..., format => 'iceberg')` scan. //! - `inspect` — Dry-run file inspection powering the `inspect_file` MCP tool. -//! - `export` — Writes query results to CSV, Parquet, Arrow IPC, or `.hyper` files. +//! - `export` — Writes query results to CSV, Parquet, Arrow IPC, Iceberg, or `.hyper` files. //! - `chart` — Renders SQL query results as PNG/SVG charts via the `plotters` crate. //! - `saved_queries` — Named read-only SQL queries exposed via tools and `hyper://queries/...` resources. //! - `subscriptions` — Per-URI registry of MCP clients that asked for resource-update notifications. //! - `table_catalog` — User-visible catalog of data tables (`_table_catalog`) tracking -//! source, purpose, and load history so workspaces are self-documenting. Disabled by `--bare`. +//! source, purpose, and load history so workspaces are self-documenting. //! - `version` — Compile-time-captured version strings for the MCP crate and the underlying `hyperdb-api`, with a git-hash suffix. //! - `readme` — Static LLM-facing README returned by the `get_readme` tool. //! - `watcher` — Monitors directories for incremental ingest via a `.ready` sentinel protocol. //! - `server` — MCP tool definitions and the `rmcp` server handler that ties everything together. +//! - `diagnostics` — Installation and launcher identity, and the `doctor` report. // Every module below is `#[doc(hidden)]`: reachable so the binary, the // integration tests, and the examples compile, but excluded from the published diff --git a/hyperdb-mcp/src/main.rs b/hyperdb-mcp/src/main.rs index 3e594cb5..96c3adec 100644 --- a/hyperdb-mcp/src/main.rs +++ b/hyperdb-mcp/src/main.rs @@ -17,8 +17,9 @@ //! 2. **`/hyperdb-mcp.log`** — append-only file, same log filter. //! The path is reported in the `status` tool's `logs.client_log` field. //! -//! Both `hyperd` and the client write to the same `log_dir` (see -//! [`hyperdb_mcp::engine::resolve_log_dir`]). Check the `status` tool for +//! With a private `hyperd`, both it and the client write to the same `log_dir` +//! (see [`hyperdb_mcp::engine::resolve_log_dir`]); the shared daemon's `hyperd` +//! logs under the daemon state directory instead. Check the `status` tool for //! the exact paths. use clap::{Parser, Subcommand}; @@ -33,7 +34,7 @@ use hyperdb_mcp::server::HyperMcpServer; use rmcp::ServiceExt; use tracing_subscriber::{EnvFilter, fmt, prelude::*}; -// Both MCP_VERSION and HYPERDB_GIT_HASH are env! string literals, so this +// Both CARGO_PKG_VERSION and HYPERDB_GIT_HASH are env! string literals, so this // concat! resolves at compile time into a single &'static str — exactly // what clap wants for the `version = ...` attribute. const VERSION: &str = concat!(env!("CARGO_PKG_VERSION"), ".r", env!("HYPERDB_GIT_HASH")); diff --git a/hyperdb-mcp/src/paths.rs b/hyperdb-mcp/src/paths.rs index 746cb82d..3d0540e1 100644 --- a/hyperdb-mcp/src/paths.rs +++ b/hyperdb-mcp/src/paths.rs @@ -191,7 +191,7 @@ mod tests { use super::*; /// Process-wide lock for env-var tests. `std::env::set_var` is - /// `unsafe` in newer toolchains because it's not thread-safe; we + /// `unsafe` in edition 2024 because it's not thread-safe; we /// serialize all env-touching tests to keep them sound. static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); @@ -203,7 +203,7 @@ mod tests { } /// Sets an env var. Marked `unsafe` because [`std::env::set_var`] is - /// `unsafe` in newer toolchains; callers hold `ENV_LOCK`. + /// `unsafe` in edition 2024; callers hold `ENV_LOCK`. unsafe fn set_env(key: &str, value: &str) { // SAFETY: serialized by ENV_LOCK; matches std::env contract. unsafe { std::env::set_var(key, value) } diff --git a/hyperdb-mcp/src/readme.rs b/hyperdb-mcp/src/readme.rs index bda0d090..32161737 100644 --- a/hyperdb-mcp/src/readme.rs +++ b/hyperdb-mcp/src/readme.rs @@ -63,8 +63,9 @@ When you need to remember something, pick the lighter tool: Both persist the same way: default is the local database (lost on restart); pass `database: \"persistent\"` to keep either one across -sessions. The KV and `load_*` tools also accept the `persist: true` -shorthand; `execute` takes `database` only. +sessions. The KV tools, `load_data`, `load_file`, `load_files`, and +`watch_directory` also accept the `persist: true` shorthand; `execute` +takes `database` only, and `load_iceberg` always loads into local. ### Routing data to a destination @@ -132,6 +133,7 @@ alias. `copy_query` also retains `target_database`. ### Query - `query` — run a read-only SELECT / WITH / EXPLAIN / SHOW / VALUES. + Max 10,000 rows; check `truncated` / `total_rows`. - `execute` — run one or more DDL/DML statements as an atomic batch. `sql` is an array; multi-element batches run inside a transaction (all commit or all roll back). Disabled in read-only mode. @@ -155,9 +157,9 @@ interop; `.hyper` export snapshots every table for Tableau. `append` / `merge`. Use `merge` to upsert by `merge_key` (column name or list); new columns in the incoming file are auto-added via `ALTER TABLE`. -- `load_files` — load many files in parallel. Files must share a - schema (or be unioned). `merge` mode is not supported here — call - `load_file` per-file if you need merge. +- `load_files` — load many files in parallel; each entry + `{path, table, mode?}` loads into its own table. `merge` mode is + not supported here — call `load_file` per-file if you need merge. - `load_data` — load inline JSON / CSV into a named database table. - `load_iceberg` — load an Apache Iceberg table by absolute path to its root directory; supports snapshot pinning via `metadata_filename` or @@ -173,7 +175,7 @@ interop; `.hyper` export snapshots every table for Tableau. - `sample` — return schema + first N rows of a table. Use this before writing a non-trivial query. - `inspect_file` — dry-run schema inference on a CSV / Parquet / Arrow - IPC file without loading it. + IPC / JSON file without loading it. - `status` — plugin and native/API identity; daemon/Hyper connection facts; local/persistent paths; table count; disk usage; watchers; attachments; read-only flag. @@ -182,7 +184,8 @@ interop; `.hyper` export snapshots every table for Tableau. `named_pipe`), `host` + `port` (TCP only, else null), `socket_path` (IPC only, else null), and `connection_descriptor` — the scheme- qualified string `hyperd` emits and the Hyper API accepts, e.g. - `tab.tcp://host:port`. TCP on every platform today. + `tab.tcp://host:port`. The shared daemon uses IPC; a private + `hyperd` (`--no-daemon` or daemon fallback) uses TCP. Both full and degraded responses report `default_database: \"local\"`. When `engine_busy: true`, the response is partial and non-definitive: @@ -308,8 +311,10 @@ scale 0 and truncates decimal places. Example: `41.54178215::numeric` - **`copy_query` modes:** `create` requires the target not exist; `append` requires it does; `replace` drops and recreates atomically. - **`load_file` merge mode:** `mode = \"merge\"` requires `merge_key` - (column name or list of column names). Rows whose key matches an - existing row UPDATE; non-matching rows INSERT. Columns present in + (column name or list of column names). Rows whose key matches are + deleted and replaced by the incoming row (target columns absent from + the file become NULL; duplicate incoming keys all insert); + non-matching rows INSERT. Columns present in the incoming file but not the target are auto-added via `ALTER TABLE ADD COLUMN` (nullable; existing rows fill with NULL). **Type changes on existing columns are rejected** — use `replace` @@ -392,8 +397,8 @@ differences from standard PostgreSQL: WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'theme')\" ]}) ``` - Single-element arrays auto-commit (same as the legacy single-statement - shape). Mixing DDL with DML in one batch is rejected — Hyper aborts + A single-element array runs one auto-commit statement. Mixing DDL + with DML in one batch is rejected — Hyper aborts such transactions with SQLSTATE 0A000. Issue DDL in its own `execute` call. Do NOT include `BEGIN` / `COMMIT` / `ROLLBACK` / `SAVEPOINT` in batch elements — the tool manages the transaction for you and these @@ -440,7 +445,7 @@ export({ // Refresh existing rows + auto-add new columns (upsert by job_id). // Use this when you re-parsed source data with extra fields and want -// to update the table in place without dropping it. +// to refresh the table in place without dropping it. load_file({ \"path\": \"/tmp/extract_failures-with-host.json\", \"table\": \"extract_timing_failures\", @@ -448,7 +453,7 @@ load_file({ \"merge_key\": \"job_id\" }) -// Single-statement execute (auto-commit, same as before) +// Single-statement execute (auto-commit) execute({ \"sql\": [\"DELETE FROM events WHERE created_at < CURRENT_DATE - INTERVAL '90' DAY\"] }) diff --git a/hyperdb-mcp/src/saved_queries.rs b/hyperdb-mcp/src/saved_queries.rs index 251eb5b8..f4c526e4 100644 --- a/hyperdb-mcp/src/saved_queries.rs +++ b/hyperdb-mcp/src/saved_queries.rs @@ -7,12 +7,12 @@ //! //! Two [`SavedQueryStore`] implementations: //! -//! * [`SessionStore`] — in-memory `HashMap` behind a `Mutex`. Used for -//! ephemeral servers (no `--workspace`) where persistence across restarts -//! is meaningless because the whole `.hyper` file is thrown away. +//! * [`SessionStore`] — in-memory `HashMap` behind a `Mutex`. Used when no +//! persistent database is configured (`--ephemeral-only`, or no platform +//! data directory), where persistence across restarts is meaningless. //! * [`WorkspaceStore`] — backs onto a dedicated meta-table -//! `_hyperdb_saved_queries` inside the Hyper workspace. Chosen when a -//! `--workspace` path is configured so saved queries survive restarts +//! `_hyperdb_saved_queries` inside the Hyper workspace. Chosen whenever a +//! persistent database is configured so saved queries survive restarts //! alongside the data they query. //! //! The server picks a store variant in `HyperMcpServer::new` and hands it @@ -77,18 +77,16 @@ impl SavedQuery { /// Hyper; stores that don't need it (e.g. [`SessionStore`]) simply ignore /// the argument. pub trait SavedQueryStore: Send + Sync { - /// Persist a new query. Returns an `AlreadyExists`-class error - /// (currently `SchemaMismatch`, with a clear message) if `name` is - /// already in use — callers should `delete` first if overwriting is - /// intended. + /// Persist a new query. Fails if `name` is already in use; callers + /// should `delete` first if overwriting is intended. /// /// # Errors /// /// - Returns [`ErrorCode::InvalidArgument`] if a query with the same /// name already exists. /// - Returns [`ErrorCode::InternalError`] for store-specific failures - /// (poisoned mutex in [`SessionStore`], Hyper catalog errors in - /// `CatalogStore`). + /// (poisoned mutex in [`SessionStore`]). [`WorkspaceStore`] propagates + /// engine errors with their own codes. fn save(&self, engine: Option<&Engine>, query: SavedQuery) -> Result<(), McpError>; /// Retrieve a single saved query by name, or `Ok(None)` if not found. @@ -191,9 +189,9 @@ impl SavedQueryStore for SessionStore { // --- WorkspaceStore --------------------------------------------------------- /// Persistent [`SavedQueryStore`] backed by the `_hyperdb_saved_queries` -/// meta-table inside the `.hyper` workspace. Rows round-trip through SQL -/// parameter binding so saved queries containing quotes or backslashes are -/// handled safely. +/// meta-table inside the `.hyper` workspace. Values are embedded as escaped +/// literals via `sql_literal`, so saved queries containing quotes or +/// backslashes are handled safely. /// /// Lazy init: the meta-table is created on the first `save`/`list`/`get`/ /// `delete` call, guarded by a `Mutex` so concurrent first-touches @@ -297,8 +295,8 @@ fn row_to_saved_query(row: &Value) -> Result { .map(String::from); let created_at_str = row.get("created_at").and_then(|v| v.as_str()).unwrap_or(""); // Accept both RFC 3339 and the space-separated `YYYY-MM-DD HH:MM:SS[.fff]` - // shape Hyper emits for TIMESTAMP columns; fall back to "now" on parse - // failure rather than losing the whole row. + // shape Hyper emits for TIMESTAMP columns; a parse failure is an + // `InternalError`. let created_at = DateTime::parse_from_rfc3339(created_at_str) .map(|d| d.with_timezone(&Utc)) .or_else(|_| { @@ -334,8 +332,7 @@ impl SavedQueryStore for WorkspaceStore { self.ensure_table(engine)?; let table = Self::qualified_table(); - // Up-front existence check — clearer error than Hyper's raw PK - // violation message, and matches SessionStore's behaviour. + // Up-front existence check — matches SessionStore's behaviour. let existing_sql = format!( "SELECT name FROM {table} WHERE name = {}", sql_literal(&query.name) @@ -431,7 +428,8 @@ impl SavedQueryStore for WorkspaceStore { /// Build the right store for a given workspace mode. /// /// `Some(path)` → [`WorkspaceStore`] (persisted in the `.hyper` file). -/// `None` → [`SessionStore`] (in-memory, dies with the process). +/// `None` → [`SessionStore`] (in-memory, dies with the process); used +/// when no persistent database is configured. #[must_use] pub fn build_store(workspace_path: Option<&str>) -> Arc { if workspace_path.is_some() { diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index 73637bfa..99c9f3a5 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -332,15 +332,13 @@ pub struct LoadFilesEntry { pub table: String, /// Absolute path to a CSV, Parquet, Arrow IPC, or JSON file. pub path: String, - /// `"replace"` (default), `"append"`, or `"merge"` — see - /// [`LoadFileParams::mode`] for semantics. + /// `"replace"` (default) or `"append"`; `"merge"` is rejected. pub mode: Option, /// Partial schema override keyed by column name. pub schema: Option, /// Optional JSON extract path — see `LoadFileParams::json_extract_path`. pub json_extract_path: Option, - /// When `mode = "merge"`, the column(s) to match on for upsert. See - /// [`LoadFileParams::merge_key`]. + /// Rejected if set: `load_files` does not support merge. pub merge_key: Option, } @@ -353,7 +351,7 @@ pub struct LoadFilesParams { pub files: Vec, /// Maximum number of concurrent ingest tasks. Each task checks out /// its own connection from a pool sized to match. Default: - /// `min(files.len(), 8)`. Large parquet ingests are I/O-bound on + /// `min(files.len(), 8)`; capped at 16. Large parquet ingests are I/O-bound on /// hyperd's side; more connections don't help past a certain point /// and can starve the primary connection. pub concurrency: Option, @@ -515,11 +513,11 @@ pub struct ChartParams { pub title: Option, /// Output format: "png" (default) or "svg" pub format: Option, - /// Width in pixels (default 800) + /// Width in pixels (default 800, clamped to 200-4096) pub width: Option, - /// Height in pixels (default 480) + /// Height in pixels (default 480, clamped to 150-4096) pub height: Option, - /// Number of bins for histograms (default 20) + /// Number of bins for histograms (default 20, max 500) pub bins: Option, /// Force the x column to evenly spaced categorical positions. By default, /// line/scatter DATE, TIMESTAMP, and TIMESTAMPTZ values use proportional @@ -1149,10 +1147,9 @@ impl HyperMcpServer { watchers: Arc::new(crate::watcher::WatcherRegistry::new()), saved_queries, subscriptions: Arc::new(SubscriptionRegistry::new()), - // The catalog policy is now uniform: seed `_table_catalog` - // whenever MCP creates a fresh `.hyper` file. The opt-out - // `--bare` path was removed; users wanting a pristine file - // can `DROP TABLE _table_catalog` after creation. + // `_table_catalog` is seeded whenever MCP creates a fresh + // `.hyper` file; users wanting a pristine file can + // `DROP TABLE _table_catalog` after creation. attachments: Arc::new(AttachRegistry::new()), workspace_path: persistent_path, read_only, @@ -1522,8 +1519,7 @@ impl HyperMcpServer { } /// Idempotently create and reconcile `_table_catalog` on first call - /// per engine. No-op in bare or read-only mode (read-only can't - /// mutate; bare callers never wanted the catalog in the first place). + /// per engine. No-op in read-only mode, which cannot mutate. /// /// Catalog failures during bootstrap are logged at WARN but do not /// fail the outer tool call — a broken catalog should never block a @@ -1644,7 +1640,7 @@ impl HyperMcpServer { // Bootstrap the catalog exactly once per engine. Intentionally // runs *inside* `with_engine` (not `ensure_engine`) so the // catalog SQL can see errors classified via the normal error - // path. No-op in bare or read-only mode. + // path. No-op in read-only mode. self.ensure_catalog_ready(engine); let result = f(engine); let connection_lost = result @@ -2282,7 +2278,7 @@ impl HyperMcpServer { return Self::err_content(McpError::new( ErrorCode::InvalidArgument, format!( - "load_files does not support mode=merge yet (entry {idx}, table \ + "load_files does not support mode=merge (entry {idx}, table \ '{}'). Call load_file once per file when you need merge semantics.", entry.table ), @@ -2803,9 +2799,8 @@ impl HyperMcpServer { let (per_statement, affected_total, operation): (Vec, u64, &'static str) = engine.with_search_path(target_db.as_deref(), |engine| { if params.sql.len() == 1 { - // Singletons skip BEGIN/COMMIT — same auto-commit behavior - // as the pre-batch `execute` tool, and DDL singletons stay - // legal (Hyper auto-commits DDL anyway). + // Singletons skip BEGIN/COMMIT and run in auto-commit mode, + // so DDL singletons stay legal (Hyper auto-commits DDL anyway). let stmt = ¶ms.sql[0]; let t = crate::stats::StatsTimer::start(); let affected = engine.execute_command(stmt)?; @@ -2996,8 +2991,8 @@ impl HyperMcpServer { x_measure_column, )?; - // Parse color_map: skip entries whose hex string is malformed, - // logging them via the description rather than hard-failing. + // Parse color_map: entries whose hex string is malformed are + // silently dropped and fall back to the default palette. let color_map = params .color_map .as_ref() @@ -3214,12 +3209,14 @@ impl HyperMcpServer { } } - /// Dry-run schema inference on a file (CSV, Parquet, Arrow IPC) without - /// ingesting it. Returns the inferred schema plus per-column diagnostics - /// (`null_count`, `min`, `max`, `sample_values`) so an LLM can construct - /// a safer `schema` override for `load_file` / `load_data`. + /// Dry-run schema inference on a CSV, JSON, JSONL, Parquet, or Arrow IPC file. + /// + /// Nothing is ingested. Returns the inferred schema plus per-column + /// diagnostics (`null_count`, `sample_values`, and CSV-only `min` / `max`) + /// so an LLM can construct a safer `schema` override for `load_file` / + /// `load_data`. #[tool( - description = "Dry-run schema inference on a CSV / Parquet / Arrow IPC file without ingesting. Returns the schema load_file would use (including the full-file numeric widening pass), plus per-column null_count, min, max, and sample_values. Use this BEFORE load_file if you are unsure about types or ran into a SchemaMismatch / numeric overflow — then pass an explicit `schema` override on the subsequent load_file call. Use `json_extract_path` to inspect a nested data array inside a JSON wrapper file (e.g., MCP tool responses saved to disk)." + description = "Dry-run schema inference on a CSV / JSON / JSONL / Parquet / Arrow IPC file without ingesting. Returns the schema load_file would use (including the full-file numeric widening pass), plus per-column null_count and sample_values (CSV/JSON) and min/max (CSV only). Use this BEFORE load_file if you are unsure about types or ran into a SchemaMismatch / numeric overflow — then pass an explicit `schema` override on the subsequent load_file call. Use `json_extract_path` to inspect a nested data array inside a JSON wrapper file (e.g., MCP tool responses saved to disk)." )] #[expect( clippy::unused_self, @@ -3253,7 +3250,7 @@ impl HyperMcpServer { /// Export query results or a table to CSV, Parquet, Arrow IPC, /// Apache Iceberg, or a new `.hyper` file. #[tool( - description = "Export a query result or table to a file via hyperd's native server-side writers; every format round-trips back through `load_file` / `load_iceberg`. `format`: parquet (recommended default) / csv / arrow_ipc / iceberg / hyper. Requires `sql` or `table` — except `iceberg` and `hyper`, and `hyper` ignores both and snapshots every user table into a `.hyper` file openable in Tableau Desktop (a faithful backup; check `schema_fidelity` in the response). `path` is a single file except `iceberg`, which is a directory hyperd creates. `format_options` passes through to hyperd's `COPY ... WITH (...)` (e.g. parquet `codec`, csv `delimiter`). `database` selects the source. See get_readme for per-format tradeoffs." + description = "Export a query result or table to a file via hyperd's native server-side writers; every format round-trips back through `load_file` / `load_iceberg`. `format`: parquet (recommended default) / csv / arrow_ipc / iceberg / hyper. Requires `sql` or `table` — except `hyper`, which ignores both and snapshots every user table into a `.hyper` file openable in Tableau Desktop (a faithful backup; check `schema_fidelity` in the response). `path` is a single file except `iceberg`, which is a directory hyperd creates. `format_options` passes through to hyperd's `COPY ... WITH (...)` (e.g. parquet `codec`, csv `delimiter`). `database` selects the source. See get_readme for per-format tradeoffs." )] fn export( &self, @@ -3375,10 +3372,9 @@ impl HyperMcpServer { if let Err(e) = self.check_writable("save_query") { return Self::err_content(e); } - // Enforce read-only SQL at save time. This is belt-and-braces: the - // result resource runs via `execute_query_to_json` which would - // reject DDL/DML anyway, but rejecting here produces a clearer - // error and prevents the row landing in the meta-table at all. + // Enforce read-only SQL at save time. This is the only guard: the + // result resource re-runs the stored SQL verbatim via + // `execute_query_to_json`, which does not classify statements. if !is_read_only_sql(¶ms.sql) { return Self::err_content(McpError::new( ErrorCode::SqlError, @@ -3920,11 +3916,11 @@ impl HyperMcpServer { )] #[expect( clippy::unused_self, - reason = "the #[tool] macro dispatches on &self; signature must match the rest of the tool surface even though this tool is stateless" + reason = "kept as a &self method for a uniform tool surface; this tool is stateless" )] #[expect( clippy::unnecessary_wraps, - reason = "uniform Result across all tools so the #[tool_router] dispatcher has one signature shape" + reason = "uniform Result return type across all tools" )] fn get_readme(&self) -> Result { Ok(CallToolResult::success(vec![ContentBlock::text( @@ -4027,8 +4023,8 @@ impl HyperMcpServer { // Reject if any active watcher targets this alias. Otherwise the // watcher's pool would keep ingesting into the now-detached // workspace path; or, if the user re-attached the same alias to - // a different file, into the wrong database. Fixed by stopping - // the watcher first via `unwatch_directory`. + // a different file, into the wrong database. The caller must stop + // the watcher with `unwatch_directory` first. if let Ok(watchers) = self.watchers.watchers.lock() { let conflict = watchers .values() @@ -4064,15 +4060,9 @@ impl HyperMcpServer { } } - /// List currently attached databases. + /// List currently attached databases with a best-effort visible-table count. /// - /// Named `list_attached_databases` (not `list_attached`) so it - /// sits alongside `attach_database` / `detach_database` as a - /// symmetric verb-database trio. The earlier `list_attached` - /// name broke the pattern and consistently misled LLM callers - /// into hallucinating `list_attached_databases` anyway, so the - /// tool now matches the name the models were already reaching - /// for. + /// Named to pair with `attach_database` / `detach_database`. #[tool( description = "List every database currently attached under an alias: kind, path/endpoint, writable flag, attach time, and (best-effort) a count of visible public-schema tables." )] @@ -4227,7 +4217,7 @@ impl HyperMcpServer { // stub and the data it describes can't diverge — a new // engine might not even have the catalog materialized yet. // Skipped when the destination is an attached database - // (their catalog isn't ours) or when the server is bare / + // (their catalog isn't ours) or when the server is // read-only. `after_ingest_catalog_update` logs WARN on // failure, matching how `load_file` / `load_data` / // `execute` register their provenance. diff --git a/hyperdb-mcp/src/stats.rs b/hyperdb-mcp/src/stats.rs index 81b93155..39b2a1dd 100644 --- a/hyperdb-mcp/src/stats.rs +++ b/hyperdb-mcp/src/stats.rs @@ -34,7 +34,7 @@ impl StatsTimer { pub fn elapsed_ms(&self) -> u64 { // `Duration::as_millis` returns `u128`; `as u64` would // silently wrap at ~584 million years, which is absurd in - // practice but banned by AGENTS.md §9 (no narrowing integer + // practice but banned by AGENTS.md reminder 7 (no narrowing integer // casts) because the cast is a latent data-corruption // vector if reached. `u64::try_from` with a saturating // fallback makes the overflow handling explicit. diff --git a/hyperdb-mcp/src/subscriptions.rs b/hyperdb-mcp/src/subscriptions.rs index 7a7d9315..e127172a 100644 --- a/hyperdb-mcp/src/subscriptions.rs +++ b/hyperdb-mcp/src/subscriptions.rs @@ -23,14 +23,13 @@ //! defines no "already subscribed" error, so this is spec-compliant; //! well-behaved clients subscribe once per URI per session anyway. //! 2. Unsubscribing clears *all* entries for the given URI. In multi-client -//! setups this would be surprising, but for the stdio + SSE transports +//! setups this would be surprising, but for the stdio transport //! typically used with `HyperDB` each process serves at most a handful of //! clients and cross-client URI reuse is rare. //! 3. Notify failures (client disconnected) are logged but not pruned. //! Dead peers accumulate until the MCP session tears the registry -//! down; a future improvement could prune them from the failing -//! detached task, but since notify is a bounded per-tool-call cost -//! and sessions are short-lived this hasn't mattered in practice. +//! down. Notify is a bounded per-tool-call cost and sessions are +//! short-lived, so the accumulation is harmless. use rmcp::RoleServer; use rmcp::model::ResourceUpdatedNotificationParam; @@ -40,9 +39,9 @@ use std::sync::Mutex; /// Per-URI registry of subscribed peers with broadcast helpers. /// -/// Cheap to clone (all state is behind an `Arc>` externally — -/// typically `Arc` on the server). Methods take -/// `&self` because the internal [`Mutex`] provides interior mutability. +/// Share it via `Arc` (typically `Arc` on the server). +/// Methods take `&self` because the internal [`Mutex`] provides interior +/// mutability. #[derive(Debug, Default)] pub struct SubscriptionRegistry { inner: Mutex>>>, @@ -88,8 +87,7 @@ impl SubscriptionRegistry { guard.keys().cloned().collect() } - /// Drop all subscriptions for every URI. Invoked when the server is - /// shutting down or a full workspace reset has happened. + /// Drop all subscriptions for every URI. pub fn clear(&self) { let mut guard = self.lock(); guard.clear(); @@ -133,15 +131,13 @@ impl SubscriptionRegistry { /// subscribed peer (across all URIs). Called when tables are added or /// dropped, or saved queries created / deleted. /// - /// Deduplicates peers across URIs so a client subscribed to three - /// URIs only receives one list-changed notification. + /// A peer subscribed to N URIs receives N notifications; clients treat + /// `list_changed` idempotently. pub fn notify_list_changed(&self) { - // Collect one peer per Vec entry, then dedupe by identity: two - // entries that share the underlying mpsc channel are equivalent - // from the client's perspective. Since we can't compare peers, - // we simply tolerate the rare duplicate send in multi-URI - // subscribers — list_changed handlers on the client side are - // idempotent (they trigger a resources/list refresh). + // Collect one peer per Vec entry. Peers cannot be compared, so a + // multi-URI subscriber receives one notification per entry; + // list_changed handlers on the client side are idempotent (they + // trigger a resources/list refresh). let peers: Vec> = { let guard = self.lock(); guard.values().flat_map(|v| v.iter().cloned()).collect() diff --git a/hyperdb-mcp/src/watcher.rs b/hyperdb-mcp/src/watcher.rs index 18b7bfe5..8ce6e03d 100644 --- a/hyperdb-mcp/src/watcher.rs +++ b/hyperdb-mcp/src/watcher.rs @@ -22,17 +22,19 @@ //! //! - Write data to a temporary file (e.g. `batch.csv.tmp`), then **rename** //! it to the final name (`batch.csv`). Do not write directly to the target. -//! - Never replace a data file with a symlink between writing and creating -//! the `.ready` sentinel — the watcher resolves symlinks via -//! `canonicalize()`, but the window between existence check and open -//! cannot be fully eliminated without kernel-level file descriptors. +//! - Never use a symlink for the data file or the sentinel. The watcher +//! checks both with `symlink_metadata()` and skips the pair when either +//! is a symlink, but a swap between that check and the open cannot be +//! ruled out without kernel-level file descriptors. //! - On shared filesystems, ensure the rename is atomic (same mount point). //! //! Only one table per watched directory is supported; ingest is always in -//! append mode. File extensions decide the ingest path: `.csv`/`.json` go -//! through the CSV ingest (JSON-lines not supported today), `.parquet`/`.pq` -//! through the Parquet ingest, and `.arrow`/`.ipc`/`.feather` through the -//! Arrow IPC ingest. +//! append mode. The data file's extension selects the ingest path: +//! `.parquet`/`.pq` use the Parquet ingest, `.arrow`/`.ipc`/`.feather` the +//! Arrow IPC ingest, and `.json`/`.jsonl`/`.ndjson` the JSON ingest (arrays +//! and JSON Lines). Every other extension, `.csv` included, is +//! content-sniffed: a first non-whitespace byte of `[` or `{` selects JSON, +//! anything else CSV. //! //! # Concurrency model //! @@ -320,7 +322,7 @@ impl WatcherRegistry { .and_then(|t| now.duration_since(t).ok()) // `Duration::as_millis` is `u128`; saturate to // `u64::MAX` on the absurd-long-duration edge - // instead of silently wrapping (AGENTS.md §9). + // instead of silently wrapping (AGENTS.md reminder 7). .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)); json!({ "directory": h.directory.to_string_lossy(), @@ -367,7 +369,7 @@ impl Drop for InFlightGuard { #[expect( clippy::needless_pass_by_value, - reason = "call-site ergonomics: function consumes logically-owned parameters, refactoring signatures is not worth per-site churn" + reason = "engine, attachments, registry, subscriptions, dir and options are only borrowed or Arc-cloned; by-value keeps the single server call site simple" )] /// Begin watching `dir`. Builds a dedicated connection pool, runs the /// initial sweep (sequentially — there's no benefit to parallelism for @@ -380,17 +382,28 @@ impl Drop for InFlightGuard { /// /// # Errors /// -/// - Returns [`ErrorCode::FileNotFound`] if `dir` does not exist, is -/// not a directory, or cannot be canonicalized. -/// - Returns [`ErrorCode::InternalError`] if the watcher registry -/// mutex or engine mutex is poisoned, if the engine has not been -/// initialized, if `start_watching` is not called from a Tokio -/// runtime, or if the watcher pool / OS file-system watcher cannot -/// be constructed. -/// - Returns [`ErrorCode::InternalError`] wrapping the error string -/// when [`notify::RecommendedWatcher`] setup fails. -/// - Propagates any error from the initial sweep's per-file ingest -/// (file read, schema inference, or Hyper `COPY` / `INSERT` errors). +/// - Returns [`ErrorCode::FileNotFound`] if `dir` does not exist, is not a +/// directory, or cannot be canonicalized. +/// - Returns [`ErrorCode::InvalidArgument`] if `target_db` is `persistent` +/// while the server runs in `--ephemeral-only` mode, or names an alias that +/// is not attached or is attached read-only. +/// - Returns [`ErrorCode::InternalError`] if `dir` is already watched, the +/// watcher registry mutex is poisoned, the engine is not initialized +/// (including after a poisoned engine mutex is recovered), no `hyperd` +/// endpoint is available, the call is not made from inside a Tokio runtime, +/// or the pool, the OS file watcher, or the forwarder thread cannot be +/// created. +/// +/// Per-file failures during the initial sweep are not errors: those files +/// move to `failed/` and are counted in the returned +/// [`WatcherStats::files_failed`]. +/// +/// # Panics +/// +/// Panics if called from a `current_thread` Tokio runtime, because the +/// initial sweep runs under [`tokio::task::block_in_place`]. The +/// `max_concurrent` conversion cannot panic: `resolved_concurrency` clamps +/// to [`MAX_CONCURRENT_LIMIT`], which fits in a `u32`. pub fn start_watching( engine: Arc>>, attachments: Arc, @@ -452,9 +465,8 @@ pub fn start_watching( )?)); let stats = Arc::new(Mutex::new(WatcherStats { - // Concurrency is configured by the user via a `u32`-sized field - // upstream; saturating is a safe diagnostic. - max_concurrent: u32::try_from(concurrency).unwrap_or(u32::MAX), + max_concurrent: u32::try_from(concurrency) + .expect("resolved_concurrency clamps to MAX_CONCURRENT_LIMIT"), ..Default::default() })); let in_flight = Arc::new(AtomicU32::new(0)); diff --git a/hyperdb-mcp/tests/daemon_tests.rs b/hyperdb-mcp/tests/daemon_tests.rs index 31777178..91c2c9d3 100644 --- a/hyperdb-mcp/tests/daemon_tests.rs +++ b/hyperdb-mcp/tests/daemon_tests.rs @@ -341,9 +341,11 @@ fn health_listener_waits_for_command_after_accept() { } } - // Keep the already-accepted first socket idle for well over the - // listener's 5ms accept-loop poll interval before sending its - // first command. + // An accepted socket must not inherit the listener's + // non-blocking mode: a non-blocking read would fail with + // WouldBlock at once. Keep the first socket idle for 350 ms, + // well inside the handler's 5 s read timeout, before sending + // its first command. std::thread::sleep(Duration::from_millis(350)); idle_client.ping("delayed first-client") })(); @@ -435,9 +437,8 @@ fn health_protocol_report_hyperd_error_sets_flag() { let response = health::send_command(port, "REPORT_HYPERD_ERROR").unwrap(); assert_eq!(response.trim(), "OK"); - // The handler ran on a different thread; give it a moment to land the - // store. AcqRel ordering means the store is visible here as soon as the - // handler returns, but the response write is what unblocks send_command. + // The handler ran on a different thread. The flag is set before the OK + // reply is written, so it is observable as soon as send_command returns. assert!( state.consume_restart_request(), "REPORT_HYPERD_ERROR must set the restart-requested flag" @@ -2240,7 +2241,10 @@ struct EnvGuard { impl EnvGuard { fn set(key: &str, value: &str) -> Self { let previous = std::env::var(key).ok(); - // SAFETY: Callers hold ENV_LOCK, ensuring no concurrent env var access. + // SAFETY: every env mutation in this binary goes through EnvGuard under + // ENV_LOCK, so writes never race each other or the locked tests that + // read these keys. Tests that skip the lock read env only through std, + // which serialises its own accesses against set_var. unsafe { std::env::set_var(key, value) }; Self { key: key.to_string(), @@ -2250,7 +2254,7 @@ impl EnvGuard { fn remove(key: &str) -> Self { let previous = std::env::var(key).ok(); - // SAFETY: Callers hold ENV_LOCK, ensuring no concurrent env var access. + // SAFETY: see EnvGuard::set; all env mutation is serialised by ENV_LOCK. unsafe { std::env::remove_var(key) }; Self { key: key.to_string(), @@ -2262,9 +2266,9 @@ impl EnvGuard { impl Drop for EnvGuard { fn drop(&mut self) { match &self.previous { - // SAFETY: Callers hold ENV_LOCK for the lifetime of this guard. + // SAFETY: see EnvGuard::set; the guard's owner holds ENV_LOCK. Some(val) => unsafe { std::env::set_var(&self.key, val) }, - // SAFETY: Callers hold ENV_LOCK for the lifetime of this guard. + // SAFETY: see EnvGuard::set; the guard's owner holds ENV_LOCK. None => unsafe { std::env::remove_var(&self.key) }, } } diff --git a/hyperdb-mcp/tests/doctor_tests.rs b/hyperdb-mcp/tests/doctor_tests.rs index 18b20274..8ecc691f 100644 --- a/hyperdb-mcp/tests/doctor_tests.rs +++ b/hyperdb-mcp/tests/doctor_tests.rs @@ -262,16 +262,16 @@ impl RunningHealthListener { } } - fn prime_accept_sleep(&self) { + fn prime_listener(&self) { let response = health::send_command(self.port, "PING") .expect("real health listener must answer the priming PING"); assert!( response.starts_with("PONG hyperdb-mcp "), "unexpected health-listener PING response: {response:?}" ); - // The response comes from a per-connection worker. Give the accept - // thread a small scheduling window to re-enter its real 100 ms - // WouldBlock sleep before launching the already-warm doctor child. + // The PONG comes from a per-connection worker; yield briefly so the + // accept thread is back in its readiness wait before the doctor + // child connects. std::thread::sleep(std::time::Duration::from_millis(5)); } } @@ -1961,10 +1961,9 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { let launcher_metadata = sandbox.launcher_metadata("hyperdb-mcp-test-wrapper"); let missing_hyperd = sandbox.root.join("missing-hyperd"); - // Warm the already-built child before starting the real listener. The - // accept loop no longer runs on a sleep cadence for the child to land - // inside, but keeping process-loader latency out of the measured - // attempt still makes the retry loop below cheaper. + // Warm the already-built child before starting the real listener. + // Keeping process-loader latency out of the measured attempt makes + // the repeated attempts below (every attempt must pass) cheaper. let warm_before = snapshot_tree(&sandbox.root); let warm = sandbox.run_with_options( &["doctor", "--json"], @@ -1993,7 +1992,7 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { std::fs::write(&discovery_path, &discovery_bytes).expect("write discovery fixture"); let before = snapshot_tree(&sandbox.root); - listener.prime_accept_sleep(); + listener.prime_listener(); let output = sandbox.run_with_options( &["doctor", "--json"], Some(sandbox.persistent_path.as_os_str()), @@ -2012,7 +2011,7 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { failures.push(format!("attempt {attempt}: {failure}")); } if attempt == 1 { - listener.prime_accept_sleep(); + listener.prime_listener(); let human_output = sandbox.run_with_options( &["doctor"], Some(sandbox.persistent_path.as_os_str()), @@ -2089,7 +2088,7 @@ fn doctor_cli_reports_live_from_scan_via_real_health_listener() { let listener = RunningHealthListener::start(); let before = snapshot_tree(&sandbox.root); - listener.prime_accept_sleep(); + listener.prime_listener(); let output = sandbox.run_with_options( &["doctor", "--json"], Some(sandbox.persistent_path.as_os_str()), diff --git a/hyperdb-mcp/tests/error_tests.rs b/hyperdb-mcp/tests/error_tests.rs index 999389c1..7b52434c 100644 --- a/hyperdb-mcp/tests/error_tests.rs +++ b/hyperdb-mcp/tests/error_tests.rs @@ -30,13 +30,13 @@ fn error_to_json() { assert_eq!(json["message"], "syntax error at position 42"); } -/// Every transport-disappearance variant the `hyper-client` layer can +/// Every transport-disappearance variant the `hyperdb_api_core::client` layer can /// emit must be recognized as connection-lost so `with_engine` tears /// down the engine and reconnects on the next call. #[test] fn classifies_transport_disappearance_as_connection_lost() { // Exact strings drawn from the std / tokio / PG-wire error messages - // `hyper-client` forwards when `hyperd` crashes or is killed. + // `hyperdb_api_core::client` forwards when `hyperd` crashes or is killed. for msg in [ "Broken pipe (os error 32)", "Connection reset by peer", @@ -55,21 +55,19 @@ fn classifies_transport_disappearance_as_connection_lost() { } /// Wire-protocol desync is a distinct failure mode from transport -/// disappearance: the socket is still open but `hyper-client` has +/// disappearance: the socket is still open but `hyperdb_api_core::client` has /// marked the connection desynchronized because a bounded drain /// exhausted its budget. Subsequent operations fast-fail with a /// "desynchronized" message, and the mcp server must treat that the /// same way as a transport-lost error — recycle the engine, reconnect /// on the next call. /// -/// Regression guard: this was the first concrete mcp-side gap discovered -/// after the `desynchronized` flag landed in `hyper-client`; without -/// this match, drain-poisoned connections would stay poisoned until the -/// process was restarted. +/// Without this match, drain-poisoned connections would stay poisoned +/// until the process was restarted. #[test] fn classifies_desynchronized_wire_as_connection_lost() { // The exact message produced by `RawConnection::ensure_healthy` / - // `AsyncRawConnection::ensure_healthy` in the `hyper-client` crate. + // `AsyncRawConnection::ensure_healthy` in `hyperdb_api_core::client`. let desync_msg = "connection is desynchronized from the server and \ cannot be reused; discard it and open a new one"; assert!( diff --git a/hyperdb-mcp/tests/export_tests.rs b/hyperdb-mcp/tests/export_tests.rs index bc9064a5..2a58fbb1 100644 --- a/hyperdb-mcp/tests/export_tests.rs +++ b/hyperdb-mcp/tests/export_tests.rs @@ -1,9 +1,10 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Tests for the export module: verifying each output format (CSV, Parquet, -//! Hyper) produces valid files with correct row counts, and that both -//! table-based and query-based exports work. +//! Integration tests for export to CSV, Parquet, Arrow IPC, Iceberg, and Hyper. +//! +//! Covers table-based and query-based exports, `format_options`, overwrite +//! handling, `source_db`, and type and constraint preservation. mod common; use common::TestEngine; @@ -75,7 +76,7 @@ fn export_csv_from_query() { } /// Export an entire table to Parquet. Verify the row count and that the -/// output file is non-empty (valid Parquet files have a magic footer). +/// output file ends with the Parquet `PAR1` magic footer. #[test] fn export_parquet_from_table() { let te = TestEngine::new_ephemeral(); @@ -94,11 +95,11 @@ fn export_parquet_from_table() { }; let result = export_to_file(&te.engine, &opts).unwrap(); assert_eq!(result.rows, 2); - assert!(std::fs::metadata(path_str).unwrap().len() > 0); + assert!(std::fs::read(path_str).unwrap().ends_with(b"PAR1")); } -/// Export as .hyper by copying the workspace file. Verify the output is -/// a non-empty file (valid Hyper database file openable in Tableau Desktop). +/// A `table` value is ignored for `format = "hyper"`; the export still +/// writes a non-empty `.hyper` file. #[test] fn export_hyper_copies_workspace() { let te = TestEngine::new_ephemeral(); @@ -121,7 +122,7 @@ fn export_hyper_copies_workspace() { /// `format = "hyper"` with `source_db = Some("persistent")` snapshots /// the persistent attachment instead of the primary. Verifies the -/// rejection lifted in iter 2 produces the right output. +/// snapshot contains only the source database's tables. #[test] fn export_hyper_with_source_db_snapshots_persistent() { let te = TestEngine::new_ephemeral(); @@ -369,54 +370,21 @@ fn export_overwrite_true_replaces_existing_file() { assert!(contents.contains("Alice") && contents.contains("Bob")); } -/// Regression test for #277: `export.rs`'s `CREATE DATABASE` / -/// `ATTACH DATABASE` pair for `format: "hyper"` now routes through the -/// same attach-context error mapper (`Engine::execute_attach_command`) -/// that `attach.rs` uses, so a lock conflict on the export target -/// surfaces as `RESOURCE_BUSY` instead of a generic `SqlError`. +/// Regression test for #277: exporting `format: "hyper"` routes its +/// `CREATE DATABASE` / `ATTACH DATABASE` pair through +/// `Engine::execute_attach_command`, so a lock conflict on the target +/// surfaces as `RESOURCE_BUSY` rather than a generic `SqlError`. /// -/// **What this test does and doesn't prove**, from live investigation -/// against the real pinned `hyperd`: +/// `export_hyper` deletes any pre-existing target before `CREATE DATABASE`, +/// so no external lock can be forced through the public API on Unix. The +/// deterministic case is a target already attached under another alias in +/// the same session, and the result differs per platform: /// -/// - I confirmed live that a raw `ATTACH DATABASE` against a `.hyper` -/// file another `hyperd` process holds open returns SQLSTATE `55006` -/// ("the database file is locked by another process") — the -/// `is_attach_lock_conflict` model `execute_attach_command` relies on -/// is real, not hypothetical (see `engine_tests.rs`'s -/// `execute_attach_command_maps_real_lock_conflict_to_resource_busy`, -/// which reproduces it end to end through this exact helper). -/// - However, `export_hyper` unconditionally deletes any pre-existing -/// target before issuing `CREATE DATABASE` (deliberately avoiding -/// `IF NOT EXISTS`, so a stale target is never silently reused — see -/// the comment above the delete in `export.rs`). On Unix, `unlink` -/// on an open file always succeeds regardless of who else has it -/// open, so by the time `export_hyper`'s own `CREATE`/`ATTACH` run, -/// the target is always a fresh, unlocked inode. I verified directly -/// that exporting `format: "hyper"` over a path a *separate* `hyperd` -/// process holds open currently succeeds silently on this platform — -/// there is no live, non-flaky way to force `export_to_file`'s two -/// statements to observe a genuine external lock through the public -/// API. A tight racer thread attempting to attach the freshly -/// created file in the gap between our own `CREATE` and `ATTACH` -/// lost 5/5 attempts — the window is sub-millisecond. -/// - What *is* live and deterministic is the case exercised below: -/// exporting over a path that is already attached under a different -/// alias **in the same session**. `CREATE DATABASE` (no `IF NOT -/// EXISTS`) conflicts with hyperd's own per-connection registry and -/// fails with SQLSTATE `42P04` ("database already exists") — a -/// real, reachable error, but not one `is_resource_busy`'s phrase -/// list (or SQLSTATE `55006`) matches, so it correctly stays -/// `SqlError` both before and after this fix. This test pins that -/// boundary: the routing change must not misclassify it. -/// - **On Windows the same setup never reaches `CREATE DATABASE`**, and -/// that is the interesting half. `hyperd` holds the attached file open, -/// Windows refuses to unlink a file another process holds, and the -/// pre-delete fails with `ERROR_SHARING_VIOLATION`. So the pre-delete — -/// not the SQL — is where a contended export target actually surfaces, -/// which is why it now classifies as `RESOURCE_BUSY` instead of -/// `PERMISSION_DENIED`. This assertion is split per platform rather -/// than pinned to the Unix answer: pinning only the Unix answer is what -/// made this test fail on Windows CI when it was written. +/// - **Unix:** the pre-delete unlinks the open file, and `CREATE DATABASE` +/// fails with SQLSTATE `42P04` ("database already exists"). That does not +/// match `is_resource_busy`, so it stays `SqlError`. +/// - **Windows:** `hyperd` holds the file open, the pre-delete fails with +/// `ERROR_SHARING_VIOLATION`, and the error classifies as `RESOURCE_BUSY`. #[test] fn export_hyper_over_same_session_attached_target_is_classified_per_platform() { let te = TestEngine::new_ephemeral(); @@ -867,7 +835,7 @@ fn arrow_ipc_export_round_trips_through_load_file() { } /// `format_options` must actually reach hyperd. Export the same table -/// three times with different parquet `compression` values and confirm +/// twice with different parquet `codec` values (`uncompressed`, `zstd`) and confirm /// the written file size differs — that's the cleanest proof the /// option ended up in the `WITH (...)` clause rather than getting /// silently dropped. diff --git a/hyperdb-mcp/tests/ingest_arrow_tests.rs b/hyperdb-mcp/tests/ingest_arrow_tests.rs index ac7a58c0..054571dc 100644 --- a/hyperdb-mcp/tests/ingest_arrow_tests.rs +++ b/hyperdb-mcp/tests/ingest_arrow_tests.rs @@ -1,8 +1,10 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Tests for Parquet and Arrow IPC file ingest: verifying that data is loaded -//! with exact schema from file metadata, including nullability handling. +//! Integration tests for Parquet and Arrow IPC file ingest. +//! +//! Covers append, schema overrides, merge, pooled async ingest, decimal +//! round-trips, `NullType` rejection, and large row counts. #![expect( clippy::cast_possible_wrap, @@ -350,7 +352,7 @@ fn ingest_parquet_applies_schema_override() { } /// Ingest an Arrow IPC file with 2 rows of INT32 + FLOAT64 data, then query -/// back to verify the row count and that the exact schema was preserved. +/// back to verify the row count. #[test] fn ingest_arrow_ipc() { let mut te = TestEngine::new_ephemeral(); @@ -380,7 +382,7 @@ fn ingest_arrow_ipc() { /// must round-trip with the correct values over the binary COPY path. /// Mirrors the parquet decimal regression test — when the IPC loader still /// rendered values as SQL text, a missing Decimal128 branch turned every -/// cell into NULL. Under the new `ArrowInserter` path, the batch bytes go +/// cell into NULL. With the `ArrowInserter` path, the batch bytes go /// straight through; this test pins that behavior and the schema mapping /// that preserves NUMERIC(p, s). #[test] diff --git a/hyperdb-mcp/tests/ingest_tests.rs b/hyperdb-mcp/tests/ingest_tests.rs index 53fff9a4..22b09f95 100644 --- a/hyperdb-mcp/tests/ingest_tests.rs +++ b/hyperdb-mcp/tests/ingest_tests.rs @@ -1,8 +1,8 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Tests for inline data ingest (JSON and CSV): basic round-trips, append -//! mode, schema overrides, and error handling for empty input. +//! Integration tests for JSON, JSONL, and CSV ingest (inline and file), +//! format detection, `extract_json_path`, and merge mode. mod common; use common::TestEngine; @@ -382,9 +382,8 @@ fn detect_file_format_defaults_to_csv_when_unreadable() { #[test] fn ingest_json_file_handles_log_extension_via_content_sniff() { let mut te = TestEngine::new_ephemeral(); - // Construct the payload through `detect_file_format` so the test - // mirrors the production dispatch path rather than hard-coding a - // specific ingest function. + // Assert `detect_file_format` first to pin the content sniff that + // production dispatch relies on. let (path, _keep) = tmp_with_ext( "log", b"{\"id\":1,\"msg\":\"hello\"}\n\ diff --git a/hyperdb-mcp/tests/per_tool_database_tests.rs b/hyperdb-mcp/tests/per_tool_database_tests.rs index dd13bb4b..018a002d 100644 --- a/hyperdb-mcp/tests/per_tool_database_tests.rs +++ b/hyperdb-mcp/tests/per_tool_database_tests.rs @@ -70,10 +70,10 @@ fn qualified_table_escapes_embedded_quotes() { // --- Engine::resolve_target_db --------------------------------------------- -/// Empty / None / "local" all resolve to the primary db name. (LOCAL_ALIAS -/// filtering happens in the server-level `resolve_db` helper, which the -/// engine method doesn't see — so engine-level resolution returns "local" -/// verbatim. The server filter is exercised via end-to-end ingest below.) +/// `None`, `""`, and whitespace resolve to the primary db name at the engine +/// level. The `"local"` alias is filtered by `HyperMcpServer::resolve_db`, +/// covered in `end_to_end_mcp_tests.rs` by +/// `persist_true_plus_database_local_lets_database_win`. #[test] fn resolve_target_db_none_means_primary() { let te = TestEngine::new_ephemeral(); @@ -228,7 +228,7 @@ fn ingest_with_no_target_db_lands_in_primary() { // --- describe_tables_in / sample_table_in ---------------------------------- /// `describe_tables_in(Some("persistent"))` lists tables in the -/// persistent attachment, hiding internal `_hyperdb_*` tables. +/// persistent attachment. #[test] fn describe_tables_in_persistent_lists_only_persistent_tables() { let te = TestEngine::new_ephemeral(); @@ -665,8 +665,7 @@ fn ensure_exists_in_seeds_catalog_inside_user_attached_db() { // Catalog doesn't exist yet in user_db (attach-only doesn't seed // for an existing file; this attached file was just created so - // attach-with-on_missing=create *does* seed — verify and then - // make a separate user-attach test below for the no-seed case). + // attach-with-on_missing=create *does* seed). // Here we just confirm explicit ensure_exists_in is idempotent. hyperdb_mcp::table_catalog::ensure_exists_in(&engine, Some("user_db")).unwrap(); hyperdb_mcp::table_catalog::ensure_exists_in(&engine, Some("user_db")).unwrap(); @@ -820,7 +819,7 @@ fn reconcile_in_per_db_does_not_touch_persistent_catalog() { ) .unwrap(); - // Two tables in persistent (existing behavior). + // One table in persistent. engine .execute_command("CREATE TABLE \"persistent\".\"public\".\"persist_t\" (n INT)") .unwrap(); diff --git a/hyperdb-mcp/tests/readme_tests.rs b/hyperdb-mcp/tests/readme_tests.rs index 26782286..4ffcf5ff 100644 --- a/hyperdb-mcp/tests/readme_tests.rs +++ b/hyperdb-mcp/tests/readme_tests.rs @@ -788,7 +788,7 @@ fn public_docs_database_and_read_only_contract() { } /// The executable smoke guide, demo commentary, and unreleased changelog must -/// describe the surfaces added or corrected in this release candidate. +/// describe the current KV and smoke-guide surfaces. /// This catches mutations to the smoke sequence/result examples and release /// note claims that omit mandatory KV response fields or no longer match KV /// routing or Hyper-export side effects. diff --git a/hyperdb-mcp/tests/recovery_tests.rs b/hyperdb-mcp/tests/recovery_tests.rs index 22a198c3..0247498d 100644 --- a/hyperdb-mcp/tests/recovery_tests.rs +++ b/hyperdb-mcp/tests/recovery_tests.rs @@ -1,9 +1,15 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Regression tests for the Claude-session bug: `sample` returning a -//! spurious `TABLE_NOT_FOUND` due to a racy `has_table` probe, and the -//! connection-lost detection heuristics that drive auto-reconnect. +//! Regression tests for engine recovery and connection-loss handling. +//! +//! - `sample` must not return a spurious `TABLE_NOT_FOUND` from a racy +//! `has_table` probe. +//! - The connection-lost classifier must recognize transport errors and +//! ignore SQL errors, which drives auto-reconnect. +//! - A slow daemon error report must not hold the engine mutex. +//! - The watchdog meta-tests and the `hyperd` identity guard keep the +//! self-child harness itself reliable. mod common; @@ -132,6 +138,7 @@ fn slow_health_report_does_not_hold_engine_mutex() { } } +/// Meta-test: the watchdog reaps `hyperd` and preserves the exit code of a child that fails. #[test] fn slow_health_watchdog_reaps_hyperd_after_child_failure() { match child_mode().as_deref() { @@ -159,6 +166,7 @@ fn slow_health_watchdog_reaps_hyperd_after_child_failure() { ); } +/// Meta-test: the watchdog reaps `hyperd` after a child that hangs times out. #[test] fn slow_health_watchdog_reaps_hyperd_after_child_timeout() { match child_mode().as_deref() { @@ -472,8 +480,8 @@ fn run_slow_health_mutex_child() { started_at: "2026-08-14T00:00:00Z".to_string(), version: hyperdb_mcp::version::MCP_VERSION.to_string(), }; - // Discovery is the only routing input: Task 5 must carry this effective - // health port through the engine and into the loss-report path. The child + // Discovery is the only routing input: the engine must route + // `REPORT_HYPERD_ERROR` to this effective health port. The child // intentionally does not mutate the process-global daemon-port setting. discovery::write_discovery_file(&daemon_info).expect("write isolated daemon discovery"); diff --git a/hyperdb-mcp/tests/table_catalog_tests.rs b/hyperdb-mcp/tests/table_catalog_tests.rs index 07e5cff6..566c1d5a 100644 --- a/hyperdb-mcp/tests/table_catalog_tests.rs +++ b/hyperdb-mcp/tests/table_catalog_tests.rs @@ -6,7 +6,7 @@ //! //! The module-level tests drive the catalog API directly against a fresh //! [`Engine`]; the server-level tests verify the lazy bootstrap path and -//! the `--bare` opt-out by opening the workspace file twice (once through +//! the read-only no-op by opening the persistent database file twice (once through //! a server, then again with a plain `Engine` to inspect the on-disk //! state). @@ -16,7 +16,7 @@ use hyperdb_mcp::server::HyperMcpServer; use hyperdb_mcp::table_catalog::{self, MetadataFields, TABLE_CATALOG_TABLE}; use tempfile::TempDir; -/// Build a fresh engine against a temp `.hyper` workspace file. Matches +/// Build a fresh engine against a temp `.hyper` persistent database file. Matches /// the pattern used by `saved_queries_tests::workspace_engine` so the /// module interop surface stays consistent. /// Uses `new_no_daemon` to avoid interference from any daemon running @@ -343,8 +343,8 @@ fn delete_for_is_idempotent() { // run). Without this, the server or engine may connect to a daemon left over // from another test file, causing "database still in use" errors on Windows. -/// Default (non-bare) server: the catalog is created on first engine -/// use and survives across reopens of the workspace file. +/// Default server: the catalog is created on first engine +/// use and survives across reopens of the persistent database file. #[test] fn default_server_auto_creates_catalog_on_first_engine_use() { let dir = TempDir::new().unwrap(); @@ -360,12 +360,12 @@ fn default_server_auto_creates_catalog_on_first_engine_use() { let engine = Engine::new_no_daemon(Some(path_str)).unwrap(); assert!( table_exists(&engine, TABLE_CATALOG_TABLE), - "_table_catalog must be present in the workspace after a default server has touched it" + "_table_catalog must be present in the persistent database after a default server has touched it" ); } /// Read-only mode must not attempt to create the catalog either — the -/// first tool call on a pristine workspace shouldn't turn around and +/// first tool call on a pristine persistent database shouldn't turn around and /// issue a `CREATE TABLE`. #[test] fn read_only_server_does_not_create_catalog() { @@ -609,8 +609,8 @@ fn set_metadata_data_url_roundtrip() { /// #195 (as filed): a table created via `execute` on the ephemeral primary /// must be stubbed by the post-execute reconcile so `set_metadata` succeeds. -/// RED before the fix: `set_metadata` returns `TABLE_NOT_FOUND` because -/// reconcile only ever enumerated the persistent attachment. +/// Reconcile must enumerate the ephemeral primary as well as the persistent +/// attachment; otherwise `set_metadata` returns `TABLE_NOT_FOUND`. #[test] fn set_metadata_finds_execute_created_ephemeral_primary_table() { let (engine, _dir) = workspace_engine(); @@ -634,8 +634,8 @@ fn set_metadata_finds_execute_created_ephemeral_primary_table() { /// The delete-trap: a correctly-stubbed ephemeral-primary table (registered /// via the working ingest path, with user prose) must survive an unrelated -/// later structural `execute`. RED before the fix: the row is silently reaped -/// because it is absent from the persistent-only live set. +/// later structural `execute`. The row must not be reaped for being absent +/// from a persistent-only live set. #[test] fn reconcile_preserves_ephemeral_primary_metadata_across_unrelated_ddl() { let (engine, _dir) = workspace_engine(); @@ -675,9 +675,8 @@ fn reconcile_preserves_ephemeral_primary_metadata_across_unrelated_ddl() { } /// Delete-trap guard: reconciling the shared catalog must not delete -/// persistent-table rows. Passes on current code (persistent tables are -/// already enumerated) — it guards against a naive fix that swaps -/// enumeration to ephemeral-only. +/// persistent-table rows; enumeration must cover persistent tables, not +/// only ephemeral ones. #[test] fn reconcile_does_not_delete_persistent_rows_when_reconciling_shared_catalog() { let (engine, _dir) = workspace_engine(); @@ -727,12 +726,10 @@ fn reconcile_reaps_dropped_ephemeral_primary_table() { ); } -/// C1 (deep-review Critical): the rename heuristic must NOT match a -/// disappeared PERSISTENT row against a new EPHEMERAL table on a coincident -/// row count — that would migrate persistent prose onto an unrelated scratch -/// table. RED today (scratch isn't stubbed at all → the `.expect` fails); -/// RED under a naive union (the false-rename fires → `purpose.is_none()` -/// fails); GREEN only when rename targets are restricted to persistent origin. +/// The rename heuristic must NOT match a disappeared PERSISTENT row against a +/// new EPHEMERAL table on a coincident row count — that would migrate +/// persistent prose onto an unrelated scratch table. The scratch table must be +/// stubbed, and rename targets are restricted to persistent origin. #[test] fn reconcile_does_not_false_rename_persistent_row_onto_new_ephemeral_table() { let (engine, _dir) = workspace_engine(); diff --git a/hyperdb-mcp/tests/watcher_tests.rs b/hyperdb-mcp/tests/watcher_tests.rs index 1b0c9e22..8c4d0887 100644 --- a/hyperdb-mcp/tests/watcher_tests.rs +++ b/hyperdb-mcp/tests/watcher_tests.rs @@ -348,8 +348,8 @@ async fn watcher_ingests_many_files_concurrently() { assert_eq!(total, (FILES * ROWS_PER_FILE) as i64); } -/// Iter 3: a watcher with `target_db = Some("persistent")` opens the -/// persistent file as its pool workspace and ingests rows there +/// A watcher with `target_db = Some("persistent")` opens the +/// persistent file in its connection pool and ingests rows there /// instead of into primary. Verifies the pool resolution path picks /// the right .hyper file. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -358,8 +358,7 @@ async fn watcher_ingests_into_persistent_target() { // Pre-create the target table in persistent so the watcher's // append mode has somewhere to land. The current watcher contract - // is "append into an existing table" — auto-create is not part of - // this iteration's scope. + // is "append into an existing table"; the watcher does not auto-create. { let guard = engine.lock().unwrap(); guard @@ -414,7 +413,7 @@ async fn watcher_ingests_into_persistent_target() { // The watcher must also stamp persistent's _table_catalog after the // ingest. Without this row, set_table_metadata against the table // would error confusingly even though the table exists. This is - // the C1 fix from the post-merge architectural review. + // a requirement of the persistent-target ingest path. let catalog_rows: i64 = engine .lock() .unwrap() diff --git a/sea-query-hyperdb/CHANGELOG.md b/sea-query-hyperdb/CHANGELOG.md index 20abf156..cd526758 100644 --- a/sea-query-hyperdb/CHANGELOG.md +++ b/sea-query-hyperdb/CHANGELOG.md @@ -9,9 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [1.0.0-rc.3] - 2026-09-07 +These changes first shipped in 1.0.0-rc.1. + ### Changed -- **BREAKING:** the minimum supported Rust version is now **1.88**, up from +- The minimum supported Rust version is now **1.88**, up from 1.81, and the crate is compiled with **edition 2024**. 1.88 is the version Red Hat Enterprise Linux 9.7 ships as `rust-toolset`. The previous 1.81 was not achievable in practice — the lockfile already required 1.88 for several diff --git a/sea-query-hyperdb/DEVELOPMENT.md b/sea-query-hyperdb/DEVELOPMENT.md index b3a37a16..32af782d 100644 --- a/sea-query-hyperdb/DEVELOPMENT.md +++ b/sea-query-hyperdb/DEVELOPMENT.md @@ -40,7 +40,7 @@ DELETE, CREATE TABLE). Run tests: ```bash -cargo test -p sea-query-hyper +cargo test -p sea-query-hyperdb ``` Run the example: diff --git a/sea-query-hyperdb/README.md b/sea-query-hyperdb/README.md index 54c27eea..ef128b01 100644 --- a/sea-query-hyperdb/README.md +++ b/sea-query-hyperdb/README.md @@ -1,4 +1,4 @@ -# sea-query-hyper +# sea-query-hyperdb HyperDB SQL dialect backend for [sea-query](https://crates.io/crates/sea-query). @@ -15,12 +15,12 @@ syntax where needed. ```toml [dependencies] sea-query-hyperdb = "1.0" -sea-query = "0.30" +sea-query = "1.0" ``` ```rust -use sea_query::{Query, Expr, Iden}; -use sea_query_hyper::HyperQueryBuilder; +use sea_query::{Expr, ExprTrait, Iden, Query}; +use sea_query_hyperdb::HyperQueryBuilder; #[derive(Iden)] enum Users { @@ -52,8 +52,8 @@ Build queries with sea-query, convert to SQL strings, and pass them to ```rust,ignore use hyperdb_api::Connection; -use sea_query::{Query, Expr, Iden, Order}; -use sea_query_hyper::HyperQueryBuilder; +use sea_query::{Expr, ExprTrait, Iden, Order, Query}; +use sea_query_hyperdb::HyperQueryBuilder; #[derive(Iden)] enum Products { @@ -99,14 +99,13 @@ Sea-query supports advanced SQL features that work with `HyperQueryBuilder`. **Window functions:** ```rust,ignore -use sea_query::{Query, Expr, Alias, WindowDef}; +use sea_query::{Alias, Expr, ExprTrait, Query, WindowStatement}; let sql = Query::select() .columns([Users::Name, Users::Salary]) - .expr_as( - Expr::col(Users::Salary).sum().over( - WindowDef::new().partition_by(Users::Department) - ), + .expr_window_as( + Expr::col(Users::Salary).sum(), + WindowStatement::partition_by(Users::Department), Alias::new("dept_total"), ) .from(Users::Table) @@ -116,10 +115,11 @@ let sql = Query::select() **Complex JOINs:** ```rust,ignore -use sea_query::{Query, Expr}; +use sea_query::{Expr, ExprTrait, Query}; let sql = Query::select() - .columns([(Users::Table, Users::Name), (Departments::Table, Departments::Name)]) + .column((Users::Table, Users::Name)) + .column((Departments::Table, Departments::Name)) .from(Users::Table) .inner_join( Departments::Table, @@ -132,6 +132,8 @@ let sql = Query::select() **Aggregates with GROUP BY / HAVING:** ```rust,ignore +use sea_query::{Alias, Expr, ExprTrait, Query}; + let sql = Query::select() .column(Users::Department) .expr_as(Expr::cust("AVG(salary)"), Alias::new("avg_salary")) @@ -147,8 +149,15 @@ let sql = Query::select() When using sea-query with HyperDB, be aware of these architectural constraints: -- **No indexes** -- `CREATE INDEX` is accepted but has no performance effect -- **No primary/foreign/unique keys** -- constraint definitions are accepted but not enforced +- **No key or check constraints** -- Hyper rejects `PRIMARY KEY`, `UNIQUE`, + `FOREIGN KEY`/`REFERENCES`, and `CHECK` in `CREATE TABLE`, so don't call + `.primary_key()`, `.unique_key()`, or `.check()` on a `ColumnDef` or add + foreign keys +- **No indexes** -- `CREATE INDEX` fails with `0A000: Index support is + disabled`, so don't emit `Index::create()` statements +- **No auto-increment** -- `.auto_increment()` emits a + `GENERATED BY DEFAULT AS IDENTITY` column, which Hyper does not accept +- **No `JSON`, `JSONB`, or `UUID` column types** -- use `TEXT` - **Limited ALTER TABLE** -- some column modifications may not be supported --- diff --git a/sea-query-hyperdb/examples/basic_usage.rs b/sea-query-hyperdb/examples/basic_usage.rs index e4a767a9..53ae9571 100644 --- a/sea-query-hyperdb/examples/basic_usage.rs +++ b/sea-query-hyperdb/examples/basic_usage.rs @@ -3,7 +3,7 @@ //! Demonstrates building SQL queries with `HyperQueryBuilder`. //! -//! Run with: `cargo run -p sea-query-hyper --example basic_usage` +//! Run with: `cargo run -p sea-query-hyperdb --example basic_usage` use sea_query::{ColumnDef, Expr, ExprTrait, Iden, Query, Table}; use sea_query_hyperdb::HyperQueryBuilder; @@ -72,13 +72,7 @@ fn main() { let create = Table::create() .table(Products::Table) .if_not_exists() - .col( - ColumnDef::new(Products::Id) - .integer() - .not_null() - .auto_increment() - .primary_key(), - ) + .col(ColumnDef::new(Products::Id).integer().not_null()) .col(ColumnDef::new(Products::Name).string().not_null()) .col(ColumnDef::new(Products::Price).double().not_null()) .col(ColumnDef::new(Products::Category).string()) From 0a3c4d7433ba20103c4cb3e9eff439dc4d31db45 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 09:38:56 -0700 Subject: [PATCH 06/61] fix: escape scratch-table DROP and label queries; fix has_table and Interval sign - mcp: query_data/query_file built `DROP TABLE IF EXISTS ""` with the caller's table_name unescaped; embedded quotes now double. - core: get_table_labels/get_column_labels interpolated schema/table into string literals unescaped; use escape_literal. - core: AuthenticatedGrpcClient::has_table judged existence by Arrow stream byte length, so it was true for every table; count rows instead. - core: Interval Display dropped the sign of negative intervals under 1h. Each has a test that failed before the fix. --- hyperdb-api-core/CHANGELOG.md | 12 ++ .../src/client/grpc/authenticated_client.rs | 105 ++++++++++++++---- hyperdb-api-core/src/types/special.rs | 48 +++++--- hyperdb-mcp/CHANGELOG.md | 4 + hyperdb-mcp/src/server.rs | 32 +++++- 5 files changed, 165 insertions(+), 36 deletions(-) diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index b68f5599..092ccc8e 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -13,6 +13,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed + +- **`Interval` `Display` kept the sign only when the time component was an hour + or more.** A negative interval under one hour (for example −30 minutes) + rendered as a positive `00:30:00`; it now renders `-00:30:00`. +- **`AuthenticatedGrpcClient::has_table` returned `true` for every table.** It + judged existence by the byte length of the Arrow stream, which is non-empty + even for a zero-row result; it now counts rows. +- **`AuthenticatedGrpcClient::get_table_labels` / `get_column_labels` escape + their `schema` and `table` arguments** instead of interpolating them into + the catalog query unescaped. + ## [1.0.0-rc.3] - 2026-09-07 ### Added diff --git a/hyperdb-api-core/src/client/grpc/authenticated_client.rs b/hyperdb-api-core/src/client/grpc/authenticated_client.rs index 5e83fcae..034a7e0b 100644 --- a/hyperdb-api-core/src/client/grpc/authenticated_client.rs +++ b/hyperdb-api-core/src/client/grpc/authenticated_client.rs @@ -45,6 +45,7 @@ use super::proto::{ AttachedDatabase, CancelQueryParam, HyperServiceClient, OutputFormat, QueryParam, }; use super::result::GrpcQueryResult; +use crate::protocol::escape::escape_literal; /// Information about a database table. #[derive(Debug, Clone, PartialEq, Eq)] @@ -825,7 +826,7 @@ impl AuthenticatedGrpcClient { ); let result = self.execute_query(&query).await?; - Ok(!result.arrow_data().is_empty() && result.arrow_data().len() > 8) + arrow_has_rows(&result.arrow_data()) } /// Extracts a string column from Arrow IPC data. @@ -932,15 +933,7 @@ impl AuthenticatedGrpcClient { &mut self, schema: &str, ) -> Result> { - let query = format!( - r"SELECT c.relname as table_name, - COALESCE(d.description, c.relname) as label - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0 - JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid - WHERE n.nspname = '{schema}' AND c.relkind IN ('r', 'v', 'm') - ORDER BY c.relname" - ); + let query = table_labels_query(schema); let result = self.execute_query(&query).await?; parse_label_pairs(&result.arrow_data()) @@ -968,16 +961,7 @@ impl AuthenticatedGrpcClient { schema: &str, table: &str, ) -> Result> { - let query = format!( - r"SELECT a.attname as column_name, - COALESCE(d.description, a.attname) as label - FROM pg_catalog.pg_attribute a - LEFT JOIN pg_catalog.pg_description d ON d.objoid = a.attrelid AND d.objsubid = a.attnum - JOIN pg_catalog.pg_class c ON a.attrelid = c.oid - JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid - WHERE n.nspname = '{schema}' AND c.relname = '{table}' AND a.attnum > 0 AND NOT a.attisdropped - ORDER BY a.attnum" - ); + let query = column_labels_query(schema, table); let result = self.execute_query(&query).await?; parse_label_pairs(&result.arrow_data()) @@ -1491,6 +1475,62 @@ impl AuthenticatedGrpcClientSync { } } +/// Builds the catalog query behind [`AuthenticatedGrpcClient::get_table_labels`]. +fn table_labels_query(schema: &str) -> String { + let schema = escape_literal(schema); + format!( + r"SELECT c.relname as table_name, + COALESCE(d.description, c.relname) as label + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0 + JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname = {schema} AND c.relkind IN ('r', 'v', 'm') + ORDER BY c.relname" + ) +} + +/// Builds the catalog query behind [`AuthenticatedGrpcClient::get_column_labels`]. +fn column_labels_query(schema: &str, table: &str) -> String { + let schema = escape_literal(schema); + let table = escape_literal(table); + format!( + r"SELECT a.attname as column_name, + COALESCE(d.description, a.attname) as label + FROM pg_catalog.pg_attribute a + LEFT JOIN pg_catalog.pg_description d ON d.objoid = a.attrelid AND d.objsubid = a.attnum + JOIN pg_catalog.pg_class c ON a.attrelid = c.oid + JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE n.nspname = {schema} AND c.relname = {table} AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum" + ) +} + +/// Returns whether an Arrow IPC stream contains at least one row. +/// +/// A zero-row result is still a non-empty IPC stream (schema message plus +/// end-of-stream marker), so the byte length says nothing about the row count. +/// +/// # Errors +/// +/// Returns [`crate::client::Error`] if the stream cannot be opened or a record +/// batch fails to decode. +fn arrow_has_rows(arrow_data: &[u8]) -> Result { + if arrow_data.is_empty() { + return Ok(false); + } + let reader = arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(arrow_data), None) + .map_err(|e| crate::client::Error::other(format!("Failed to parse Arrow data: {e}")))?; + for batch in reader { + let batch = batch.map_err(|e| { + crate::client::Error::other(format!("Failed to decode Arrow record batch: {e}")) + })?; + if batch.num_rows() > 0 { + return Ok(true); + } + } + Ok(false) +} + /// Parses an Arrow IPC stream of `(name, label)` text pairs into a map. /// /// Shared by [`AuthenticatedGrpcClient::get_table_labels`] and @@ -1587,7 +1627,7 @@ mod tests { use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; - use super::parse_label_pairs; + use super::{arrow_has_rows, column_labels_query, parse_label_pairs, table_labels_query}; /// Encodes one record batch as an Arrow IPC stream, as the server would. fn ipc_stream(batch: &RecordBatch) -> Vec { @@ -1720,4 +1760,27 @@ mod tests { assert_eq!(labels.len(), 1, "the NULL-name row should be skipped"); assert_eq!(labels.get("accounts").map(String::as_str), Some("Accounts")); } + + #[test] + fn label_queries_escape_single_quotes_in_names() { + let evil = "x' OR '1'='1"; + let tq = table_labels_query(evil); + assert!(tq.contains("n.nspname = 'x'' OR ''1''=''1'"), "{tq}"); + let cq = column_labels_query("public", evil); + assert!(cq.contains("c.relname = 'x'' OR ''1''=''1'"), "{cq}"); + let cq = column_labels_query(evil, "t"); + assert!(cq.contains("n.nspname = 'x'' OR ''1''=''1'"), "{cq}"); + } + + #[test] + fn arrow_has_rows_distinguishes_empty_result_from_non_empty() { + let with_rows = ipc_stream(&two_text_batch(vec!["a"], vec!["A"])); + let no_rows = ipc_stream(&two_text_batch(vec![], vec![])); + // The zero-row stream still carries a schema message. + assert!(no_rows.len() > 8); + + assert!(arrow_has_rows(&with_rows).expect("decode")); + assert!(!arrow_has_rows(&no_rows).expect("decode")); + assert!(!arrow_has_rows(&[]).expect("decode")); + } } diff --git a/hyperdb-api-core/src/types/special.rs b/hyperdb-api-core/src/types/special.rs index 7a35ce64..99b5c9a4 100644 --- a/hyperdb-api-core/src/types/special.rs +++ b/hyperdb-api-core/src/types/special.rs @@ -899,25 +899,20 @@ impl fmt::Display for Interval { )); } if self.microseconds != 0 || parts.is_empty() { - let total_seconds = self.microseconds / 1_000_000; - let micros = (self.microseconds % 1_000_000).abs(); + // Take the sign once and format magnitudes: deriving it from the + // hours field alone drops it for any negative interval under 1h. + let sign = if self.microseconds < 0 { "-" } else { "" }; + let abs = self.microseconds.unsigned_abs(); + let total_seconds = abs / 1_000_000; + let micros = abs % 1_000_000; let hours = total_seconds / 3600; let minutes = (total_seconds % 3600) / 60; let seconds = total_seconds % 60; if micros == 0 { - parts.push(format!( - "{:02}:{:02}:{:02}", - hours, - minutes.abs(), - seconds.abs() - )); + parts.push(format!("{sign}{hours:02}:{minutes:02}:{seconds:02}")); } else { parts.push(format!( - "{:02}:{:02}:{:02}.{:06}", - hours, - minutes.abs(), - seconds.abs(), - micros + "{sign}{hours:02}:{minutes:02}:{seconds:02}.{micros:06}" )); } } @@ -2765,3 +2760,30 @@ mod tests { assert!(result.is_err(), "from_f64 should panic for out-of-range"); } } + +#[cfg(test)] +mod interval_display_tests { + use super::Interval; + + #[test] + fn negative_time_component_keeps_its_sign() { + let minus_30_min = Interval::new(0, 0, -30 * 60 * 1_000_000); + assert_eq!(minus_30_min.to_string(), "-00:30:00"); + let minus_90_min = Interval::new(0, 0, -90 * 60 * 1_000_000); + assert_eq!(minus_90_min.to_string(), "-01:30:00"); + let minus_half_sec = Interval::new(0, 0, -500_000); + assert_eq!(minus_half_sec.to_string(), "-00:00:00.500000"); + let mixed = Interval::new(14, -3, -45 * 1_000_000); + assert_eq!(mixed.to_string(), "1 year 2 mons -3 days -00:00:45"); + } + + #[test] + fn positive_and_zero_intervals_are_unchanged() { + assert_eq!(Interval::new(0, 0, 0).to_string(), "00:00:00"); + assert_eq!(Interval::new(0, 0, 3_661_000_000).to_string(), "01:01:01"); + assert_eq!( + Interval::new(0, 1, 1_500_000).to_string(), + "1 day 00:00:01.500000" + ); + } +} diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index a95a8511..2631912a 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- **`query_data` / `query_file` no longer interpolate `table_name` unescaped + into the scratch-table `DROP`.** A `table_name` containing a double quote + could end the quoted identifier and append further SQL to the cleanup + statement. - **`inspect_file` and JSON schema inference no longer panic on non-ASCII text.** Sample-value previews were truncated at a byte offset and ISO 8601 date/timestamp detection sliced strings by byte index, so a multi-byte diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index 99c9f3a5..ae3ae394 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -1939,7 +1939,7 @@ impl HyperMcpServer { // propagating the query error with `?` before the drop would // orphan the temp table (it then surfaces in `describe`). let query_result = engine.execute_query_to_json(&query_sql); - let _ = engine.execute_command(&format!("DROP TABLE IF EXISTS \"{temp_table}\"")); + let _ = engine.execute_command(&drop_scratch_table_sql(&temp_table)); let rows = query_result?; Ok(json!({ @@ -2002,7 +2002,7 @@ impl HyperMcpServer { // propagating the query error with `?` before the drop would // orphan the temp table (it then surfaces in `describe`). let query_result = engine.execute_query_to_json(&query_sql); - let _ = engine.execute_command(&format!("DROP TABLE IF EXISTS \"{temp_table}\"")); + let _ = engine.execute_command(&drop_scratch_table_sql(&temp_table)); let rows = query_result?; Ok(json!({ @@ -5377,6 +5377,16 @@ fn rand_suffix() -> String { format!("{}", t.as_nanos() % 1_000_000_000) } +/// `DROP` statement for a `query_data` / `query_file` scratch table. The name +/// embeds the caller-supplied `table_name`, so it must be quoted as an +/// identifier rather than wrapped in bare quotes. +fn drop_scratch_table_sql(temp_table: &str) -> String { + format!( + "DROP TABLE IF EXISTS \"{}\"", + temp_table.replace('"', "\"\"") + ) +} + /// Replace whole-word occurrences of the identifier `needle` with /// `replacement` in `sql`. A match qualifies only when the characters on /// both sides are not identifier characters (`[A-Za-z0-9_]`), so rewriting @@ -5626,6 +5636,24 @@ fn perform_copy( })) } +#[cfg(test)] +mod drop_scratch_table_tests { + use super::drop_scratch_table_sql; + + #[test] + fn drop_sql_escapes_embedded_quotes() { + assert_eq!( + drop_scratch_table_sql("_tmp_data_1"), + "DROP TABLE IF EXISTS \"_tmp_data_1\"" + ); + // A `table_name` of `x"; DROP TABLE victim; --` must stay one identifier. + assert_eq!( + drop_scratch_table_sql("_tmp_x\"; DROP TABLE victim; --_1"), + "DROP TABLE IF EXISTS \"_tmp_x\"\"; DROP TABLE victim; --_1\"" + ); + } +} + #[cfg(test)] mod replace_identifier_tests { use super::replace_identifier; From dc8e1ac69ebd742d9050cf6982d36b05b37f4e1f Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 10:10:24 -0700 Subject: [PATCH 07/61] fix(salesforce): stop leaking client secrets and tokens through Debug and logs Debug on the OAuth config and token types printed client_secret and raw tokens; the debug-level response logging wrote full OAuth response bodies. Manual Debug impls now redact them and the logs record only byte counts. --- hyperdb-api-salesforce/CHANGELOG.md | 9 +++ hyperdb-api-salesforce/src/config.rs | 37 ++++++++++- hyperdb-api-salesforce/src/provider.rs | 9 ++- hyperdb-api-salesforce/src/token.rs | 88 ++++++++++++++++++++++++-- 4 files changed, 136 insertions(+), 7 deletions(-) diff --git a/hyperdb-api-salesforce/CHANGELOG.md b/hyperdb-api-salesforce/CHANGELOG.md index fa9ab24c..3c09d883 100644 --- a/hyperdb-api-salesforce/CHANGELOG.md +++ b/hyperdb-api-salesforce/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Security + +- **Secrets no longer appear in `Debug` output or debug logs.** + `SalesforceAuthConfig` printed `client_secret`, and `OAuthTokenResponse`, + `OAuthToken`, `DataCloudTokenResponse` and `DataCloudToken` printed their + access-token values; all five now redact them. The token provider also no + longer logs the raw OAuth and DC JWT response bodies at `debug` level (it + logs their size instead). + ## [1.0.0-rc.3] - 2026-09-07 These changes first shipped in 1.0.0-rc.1. diff --git a/hyperdb-api-salesforce/src/config.rs b/hyperdb-api-salesforce/src/config.rs index f3511549..74e9ba2d 100644 --- a/hyperdb-api-salesforce/src/config.rs +++ b/hyperdb-api-salesforce/src/config.rs @@ -162,7 +162,7 @@ impl std::fmt::Debug for AuthMode { /// # Ok(()) /// # } /// ``` -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct SalesforceAuthConfig { /// Salesforce login URL (e.g., "" or custom domain) pub(crate) login_url: Url, @@ -187,6 +187,23 @@ pub struct SalesforceAuthConfig { pub(crate) max_retries: u32, } +impl std::fmt::Debug for SalesforceAuthConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SalesforceAuthConfig") + .field("login_url", &self.login_url) + .field("client_id", &self.client_id) + .field( + "client_secret", + &self.client_secret.as_ref().map(|_| "[REDACTED]"), + ) + .field("auth_mode", &self.auth_mode) + .field("dataspace", &self.dataspace) + .field("timeout_secs", &self.timeout_secs) + .field("max_retries", &self.max_retries) + .finish() + } +} + impl SalesforceAuthConfig { /// Creates a new configuration with the given login URL and client ID. /// @@ -350,3 +367,21 @@ pub(crate) fn is_known_salesforce_host(host: &str) -> bool { false } + +#[cfg(test)] +mod redaction_tests { + use super::*; + + #[test] + fn config_debug_redacts_client_secret() { + let config = SalesforceAuthConfig::new("https://login.salesforce.com", "client-id-123") + .expect("config") + .client_secret("hunter2-client-secret"); + let dbg = format!("{config:?}"); + assert!(!dbg.contains("hunter2-client-secret"), "{dbg}"); + assert!( + dbg.contains("client-id-123"), + "non-secret fields stay: {dbg}" + ); + } +} diff --git a/hyperdb-api-salesforce/src/provider.rs b/hyperdb-api-salesforce/src/provider.rs index d2294b53..eee2e302 100644 --- a/hyperdb-api-salesforce/src/provider.rs +++ b/hyperdb-api-salesforce/src/provider.rs @@ -373,7 +373,11 @@ impl DataCloudTokenProvider { let response = self.post_with_retry(&token_url, &form_data).await?; let response_text = response.text().await?; - debug!(response = %response_text, "OAuth Access Token response received"); + // The body carries the access token; log only its size. + debug!( + bytes = response_text.len(), + "OAuth Access Token response received" + ); let oauth_response: OAuthTokenResponse = serde_json::from_str(&response_text).map_err(|e| { @@ -433,7 +437,8 @@ impl DataCloudTokenProvider { let response = self.post_with_retry(&exchange_url, &form_data).await?; let response_text = response.text().await?; - debug!(response = %response_text, "DC JWT response received"); + // The body carries the DC JWT; log only its size. + debug!(bytes = response_text.len(), "DC JWT response received"); let dc_response: DataCloudTokenResponse = serde_json::from_str(&response_text).map_err(|e| { diff --git a/hyperdb-api-salesforce/src/token.rs b/hyperdb-api-salesforce/src/token.rs index a5ff46b6..c3a1129b 100644 --- a/hyperdb-api-salesforce/src/token.rs +++ b/hyperdb-api-salesforce/src/token.rs @@ -28,7 +28,7 @@ const DC_JWT_VALIDITY_BUFFER_SECS: i64 = 300; /// OAuth Access Token response from Salesforce `/services/oauth2/token`. /// /// See: -#[derive(Debug, Deserialize)] +#[derive(Deserialize)] pub struct OAuthTokenResponse { /// OAuth Access Token pub access_token: String, @@ -57,6 +57,20 @@ pub struct OAuthTokenResponse { pub error_description: Option, } +impl std::fmt::Debug for OAuthTokenResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthTokenResponse") + .field("access_token", &"[REDACTED]") + .field("instance_url", &self.instance_url) + .field("token_type", &self.token_type) + .field("scope", &self.scope) + .field("issued_at", &self.issued_at) + .field("error", &self.error) + .field("error_description", &self.error_description) + .finish() + } +} + impl OAuthTokenResponse { /// Checks if the response contains an error. pub fn check_error(&self) -> SalesforceAuthResult<()> { @@ -79,7 +93,7 @@ impl OAuthTokenResponse { /// /// Obtained from `/services/oauth2/token`. This token is exchanged for a /// DC JWT via `/services/a360/token`. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct OAuthToken { /// OAuth Access Token value pub token: String, @@ -92,6 +106,17 @@ pub struct OAuthToken { pub expires_at: DateTime, } +impl std::fmt::Debug for OAuthToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthToken") + .field("token", &"[REDACTED]") + .field("instance_url", &self.instance_url) + .field("obtained_at", &self.obtained_at) + .field("expires_at", &self.expires_at) + .finish() + } +} + /// Default OAuth Access Token lifetime in seconds. /// /// Salesforce reports `access-token-expires-in: 7199` (~2 hours), but the @@ -143,7 +168,7 @@ impl OAuthToken { /// DC JWT response from `/services/a360/token`. /// /// See: -#[derive(Debug, Deserialize)] +#[derive(Deserialize)] pub struct DataCloudTokenResponse { /// DC JWT value pub access_token: String, @@ -168,6 +193,19 @@ pub struct DataCloudTokenResponse { pub error_description: Option, } +impl std::fmt::Debug for DataCloudTokenResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DataCloudTokenResponse") + .field("access_token", &"[REDACTED]") + .field("instance_url", &self.instance_url) + .field("token_type", &self.token_type) + .field("expires_in", &self.expires_in) + .field("error", &self.error) + .field("error_description", &self.error_description) + .finish() + } +} + impl DataCloudTokenResponse { /// Checks if the response contains an error. pub fn check_error(&self) -> SalesforceAuthResult<()> { @@ -197,7 +235,7 @@ impl DataCloudTokenResponse { /// refreshes it within 5 minutes of expiry; callers that also want age-based /// refresh, as `AuthenticatedGrpcClient` does every ~15 minutes, use /// [`Self::needs_refresh`]. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct DataCloudToken { /// Token type (e.g., "Bearer") token_type: String, @@ -211,6 +249,18 @@ pub struct DataCloudToken { expires_at: DateTime, } +impl std::fmt::Debug for DataCloudToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DataCloudToken") + .field("token_type", &self.token_type) + .field("token", &"[REDACTED]") + .field("tenant_url", &self.tenant_url) + .field("created_at", &self.created_at) + .field("expires_at", &self.expires_at) + .finish() + } +} + impl DataCloudToken { /// Creates a DC JWT from a `/services/a360/token` response. /// @@ -416,6 +466,36 @@ fn base64_url_decode(input: &str) -> SalesforceAuthResult> { mod tests { use super::*; + const SECRET: &str = "00Dxx!SUPER-SECRET-TOKEN"; + + #[test] + fn debug_output_never_contains_token_values() { + let oauth_response: OAuthTokenResponse = serde_json::from_str(&format!( + r#"{{"access_token":"{SECRET}","instance_url":"https://na1.salesforce.com"}}"# + )) + .expect("parse"); + let dbg = format!("{oauth_response:?}"); + assert!(!dbg.contains(SECRET), "{dbg}"); + assert!( + dbg.contains("na1.salesforce.com"), + "non-secret fields stay: {dbg}" + ); + + let oauth = OAuthToken::from_response(oauth_response).expect("token"); + assert!(!format!("{oauth:?}").contains(SECRET)); + + let dc_response: DataCloudTokenResponse = serde_json::from_str(&format!( + r#"{{"access_token":"{SECRET}","instance_url":"tenant.example.com"}}"# + )) + .expect("parse"); + assert!(!format!("{dc_response:?}").contains(SECRET)); + + let dc = DataCloudToken::from_response(dc_response).expect("token"); + let dbg = format!("{dc:?}"); + assert!(!dbg.contains(SECRET), "{dbg}"); + assert!(dbg.contains("tenant.example.com"), "{dbg}"); + } + #[test] fn test_oauth_access_token_response_error() { let response = OAuthTokenResponse { From e6cf3a30081c7d43d701bf03ff71bf4fd2cd4acd Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 10:10:24 -0700 Subject: [PATCH 08/61] fix(bootstrap): reject path-like release versions before install A version such as ../victim made the installer remove_dir_all a path outside the install root. Versions are now validated and rejected with the new Error::InvalidVersion. --- hyperdb-bootstrap/CHANGELOG.md | 8 ++++ hyperdb-bootstrap/src/error.rs | 8 ++++ hyperdb-bootstrap/src/install.rs | 82 ++++++++++++++++++++++++++++++-- 3 files changed, 94 insertions(+), 4 deletions(-) diff --git a/hyperdb-bootstrap/CHANGELOG.md b/hyperdb-bootstrap/CHANGELOG.md index 94f2e1a9..fbe3cb15 100644 --- a/hyperdb-bootstrap/CHANGELOG.md +++ b/hyperdb-bootstrap/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed + +- **The installer now validates the release `version` before using it as a + directory name.** A version such as `../victim` made the install step + `remove_dir_all` a path outside the install root. A version containing + anything other than ASCII alphanumerics, `.`, `-`, `_` or `+` (or equal to + `.` / `..`) is rejected with the new `Error::InvalidVersion` variant. + ## [1.0.0-rc.3] - 2026-09-07 Entries accumulated under `[Unreleased]` from 0.1.1 until this section was diff --git a/hyperdb-bootstrap/src/error.rs b/hyperdb-bootstrap/src/error.rs index e17c18af..78d4abc8 100644 --- a/hyperdb-bootstrap/src/error.rs +++ b/hyperdb-bootstrap/src/error.rs @@ -68,6 +68,14 @@ pub enum Error { #[error("failed to parse version TOML: {0}")] TomlParse(#[source] toml::de::Error), + /// The release `version` is not a plain version string. It is used as a + /// directory name that [`install`](crate::install()) may delete and + /// recreate, so path separators, `..` and the like are rejected. + #[error( + "invalid hyperd version {0:?}: expected a plain version string such as \"0.0.26479\" (ASCII letters, digits, '.', '-', '_', '+'; no path separators)" + )] + InvalidVersion(String), + /// The downloaded ZIP archive was malformed or could not be extracted. #[error("zip error: {0}")] Zip(#[source] zip::result::ZipError), diff --git a/hyperdb-bootstrap/src/install.rs b/hyperdb-bootstrap/src/install.rs index 92045486..03bd6327 100644 --- a/hyperdb-bootstrap/src/install.rs +++ b/hyperdb-bootstrap/src/install.rs @@ -125,10 +125,30 @@ pub fn install(opts: InstallOptions) -> Result { } fn resolve_release(source: &VersionSource) -> Result { - match source { - VersionSource::Builtin => Ok(PinnedRelease::builtin()), - VersionSource::TomlFile(path) => PinnedRelease::from_toml_file(path), - VersionSource::Explicit(r) => Ok(r.clone()), + let release = match source { + VersionSource::Builtin => PinnedRelease::builtin(), + VersionSource::TomlFile(path) => PinnedRelease::from_toml_file(path)?, + VersionSource::Explicit(r) => r.clone(), + }; + validate_version(&release.version)?; + Ok(release) +} + +/// Rejects a `version` that is not safe to use as a directory name under +/// `dest_root`. [`download_and_extract`] runs `remove_dir_all` on +/// `dest_root/`, so a TOML or CLI `version` such as `".."` or +/// `"../../home"` would otherwise delete a directory outside the install root. +fn validate_version(version: &str) -> Result<(), Error> { + let plain = !version.is_empty() + && version != "." + && version != ".." + && version + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+')); + if plain { + Ok(()) + } else { + Err(Error::InvalidVersion(version.to_string())) } } @@ -201,3 +221,57 @@ fn copy_dir_contents(from: &Path, to: &Path) -> Result<(), Error> { } Ok(()) } + +#[cfg(test)] +mod version_validation_tests { + use super::*; + + fn release(version: &str) -> PinnedRelease { + PinnedRelease { + version: version.to_string(), + wheel_tag: std::collections::HashMap::new(), + sha256: std::collections::HashMap::new(), + } + } + + #[test] + fn plain_versions_are_accepted() { + for v in ["0.0.26479", "1.2.3-rc.1", "2026_10+build.5"] { + assert!(validate_version(v).is_ok(), "{v}"); + } + } + + #[test] + fn path_like_versions_are_rejected() { + for v in [ + "", ".", "..", "../x", "a/b", r"a\b", "/abs", "x/../..", "a b", + ] { + assert!( + matches!(validate_version(v), Err(Error::InvalidVersion(_))), + "{v:?} must be rejected" + ); + } + } + + #[test] + fn install_rejects_an_escaping_version_before_touching_the_filesystem() { + let root = tempfile::tempdir().expect("tempdir"); + let victim = root.path().join("victim"); + std::fs::create_dir_all(&victim).expect("create victim"); + std::fs::write(victim.join("keep.txt"), "precious").expect("write"); + + let dest_root = root.path().join("install"); + std::fs::create_dir_all(&dest_root).expect("create dest"); + + let err = install(InstallOptions { + dest_root, + version_source: VersionSource::Explicit(release("../victim")), + platform: Some(Platform::LinuxX86_64), + force: true, + }) + .expect_err("an escaping version must be rejected"); + + assert!(matches!(err, Error::InvalidVersion(_)), "{err}"); + assert!(victim.join("keep.txt").exists(), "victim dir must survive"); + } +} From 5a51b33fac06ab16748b7c1a0c807d2ec1b6e511 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Thu, 8 Oct 2026 10:10:24 -0700 Subject: [PATCH 09/61] fix(api): do not delete a caller-supplied socket directory on HyperProcess drop Drop removed any socket directory whose name started with hyper-, wiping a user's domain_socket_directory such as ~/hyper-data. Only the temp directory the process created itself is removed now. --- hyperdb-api/CHANGELOG.md | 5 ++ hyperdb-api/src/process.rs | 69 ++++++++++--------- hyperdb-api/tests/socket_dir_cleanup_tests.rs | 49 +++++++++++++ 3 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 hyperdb-api/tests/socket_dir_cleanup_tests.rs diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index d3b790a2..56b882f9 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -21,6 +21,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). stored as `123.45` — a silent 10× magnitude error. The value is now rescaled to the column's declared scale; a value that would lose non-zero digits or overflow is rejected with `Error::Conversion` instead of being stored wrong. +- **`HyperProcess` no longer deletes a caller-supplied socket directory on + drop.** Drop removed any IPC socket directory whose basename started with + `hyper-`, so a `domain_socket_directory` such as `~/hyper-data` was wiped with + everything in it. Only the temp directory the process created itself is + removed now. ## [1.0.0-rc.4] - 2026-09-08 diff --git a/hyperdb-api/src/process.rs b/hyperdb-api/src/process.rs index 79b1f90d..e1e70365 100644 --- a/hyperdb-api/src/process.rs +++ b/hyperdb-api/src/process.rs @@ -212,9 +212,14 @@ pub struct HyperProcess { /// The transport mode this process was started with. transport_mode: TransportMode, /// The socket directory for UDS connections (Unix only). - /// This directory is automatically cleaned up on drop. + /// Removed on drop only when [`Self::owns_socket_directory`] is set. #[cfg(unix)] socket_directory: Option, + /// `true` when this process created `socket_directory` itself (the default + /// temp directory). A caller-supplied `domain_socket_directory` is never + /// ours to delete — it may hold unrelated data. + #[cfg(unix)] + owns_socket_directory: bool, /// The pipe name for Named Pipe connections (Windows only). #[cfg(windows)] pipe_name: Option, @@ -277,8 +282,8 @@ impl HyperProcess { /// process-wide [`IPC_INSTANCE_SEQ`] counter. The suffix is load-bearing: /// two concurrently-live IPC `HyperProcess` instances in one process must /// not share a socket path, or the second bind fails and surfaces as a - /// 60 s callback timeout. The `hyper-` prefix is also load-bearing — `Drop` - /// only cleans up directories whose basename `starts_with("hyper-")`. + /// 60 s callback timeout. `Drop` removes this directory + /// (and only this one) via `owns_socket_directory`. #[cfg(unix)] fn default_socket_dir() -> PathBuf { let seq = IPC_INSTANCE_SEQ.fetch_add(1, Ordering::Relaxed); @@ -417,27 +422,28 @@ impl HyperProcess { // Create socket directory for UDS if needed (Unix only) #[cfg(unix)] - let socket_directory: Option = if transport_mode == TransportMode::Ipc { - // Use custom directory if provided, otherwise create temp directory - let dir = if let Some(custom_dir) = - parameters.and_then(|p| p.domain_socket_directory.as_ref()) - { - custom_dir.clone() + let (socket_directory, owns_socket_directory): (Option, bool) = + if transport_mode == TransportMode::Ipc { + // Use custom directory if provided, otherwise create temp directory + let (dir, owned) = if let Some(custom_dir) = + parameters.and_then(|p| p.domain_socket_directory.as_ref()) + { + (custom_dir.clone(), false) + } else { + // Create a temp directory for the socket. The basename carries a + // per-process monotonic suffix (`hyper--`) so two + // concurrently-live IPC instances in one process never share a + // socket path — see `Self::default_socket_dir`. + let temp_dir = Self::default_socket_dir(); + std::fs::create_dir_all(&temp_dir).map_err(|e| { + Error::connection_with_io("Failed to create socket directory", e) + })?; + (temp_dir, true) + }; + (Some(dir), owned) } else { - // Create a temp directory for the socket. The basename carries a - // per-process monotonic suffix (`hyper--`) so two - // concurrently-live IPC instances in one process never share a - // socket path — see `Self::default_socket_dir`. - let temp_dir = Self::default_socket_dir(); - std::fs::create_dir_all(&temp_dir).map_err(|e| { - Error::connection_with_io("Failed to create socket directory", e) - })?; - temp_dir + (None, false) }; - Some(dir) - } else { - None - }; // On non-Unix platforms there is no UDS socket directory; the variable // is only referenced inside `#[cfg(unix)]` blocks so we do not need a @@ -738,6 +744,8 @@ impl HyperProcess { log_dir: resolved_log_dir, #[cfg(unix)] socket_directory, + #[cfg(unix)] + owns_socket_directory, #[cfg(windows)] pipe_name, }) @@ -1176,14 +1184,13 @@ impl Drop for HyperProcess { let _ = self.do_shutdown(Some(Duration::from_secs(5))); } - // Clean up socket directory if we created one + // Clean up the socket directory only if we created it. A caller-supplied + // directory is left alone, even if its name happens to start with `hyper-`. #[cfg(unix)] - if let Some(ref dir) = self.socket_directory { - // Only clean up if it's a temp directory we created (contains our PID) - let dir_name = dir.file_name().and_then(|n| n.to_str()).unwrap_or(""); - if dir_name.starts_with("hyper-") { - let _ = std::fs::remove_dir_all(dir); - } + if self.owns_socket_directory + && let Some(ref dir) = self.socket_directory + { + let _ = std::fs::remove_dir_all(dir); } } } @@ -1505,7 +1512,7 @@ mod tests { /// Two IPC instances in one process must derive *distinct* default socket /// directories, or their sockets collide and the second bind 60 s-timeouts. - /// Also guards the `hyper-` prefix that `Drop`'s cleanup keys on. + /// Also guards the `hyper-` prefix that marks these directories in the temp dir. #[cfg(unix)] #[test] fn default_socket_dir_names_are_unique_and_prefixed() { @@ -1516,7 +1523,7 @@ mod tests { let name = dir.file_name().and_then(|n| n.to_str()).unwrap_or(""); assert!( name.starts_with("hyper-"), - "socket dir basename must keep the `hyper-` prefix so Drop cleans it up: {name}" + "socket dir basename must keep the `hyper-` prefix: {name}" ); } } diff --git a/hyperdb-api/tests/socket_dir_cleanup_tests.rs b/hyperdb-api/tests/socket_dir_cleanup_tests.rs new file mode 100644 index 00000000..f656c7ae --- /dev/null +++ b/hyperdb-api/tests/socket_dir_cleanup_tests.rs @@ -0,0 +1,49 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! `HyperProcess::drop` must only remove a socket directory it created itself. + +#![cfg(unix)] + +use hyperdb_api::{HyperProcess, Parameters, TransportMode}; + +/// A caller-supplied `domain_socket_directory` whose basename happens to start +/// with `hyper-` (e.g. `~/hyper-data`) is user data, not scratch space. Drop +/// used to `remove_dir_all` it purely on the name prefix. +#[test] +fn user_supplied_socket_dir_survives_drop() { + // Keep the path short: Unix socket paths are limited to ~104 bytes. + let dir = std::path::PathBuf::from(format!("/tmp/hyper-keep-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("create user dir"); + let sentinel = dir.join("precious.txt"); + std::fs::write(&sentinel, b"do not delete").expect("write sentinel"); + + let mut params = Parameters::new(); + params.set_transport_mode(TransportMode::Ipc); + params.set_domain_socket_directory(&dir); + { + let hyper = HyperProcess::new(None, Some(¶ms)).expect("start hyperd over IPC"); + assert_eq!(hyper.socket_directory(), Some(dir.as_path())); + } + + let survived = sentinel.exists(); + let _ = std::fs::remove_dir_all(&dir); + assert!(survived, "Drop deleted a user-supplied socket directory"); +} + +/// The default temp socket directory is ours and must still be cleaned up. +#[test] +fn default_socket_dir_is_removed_on_drop() { + let mut params = Parameters::new(); + params.set_transport_mode(TransportMode::Ipc); + let dir = { + let hyper = HyperProcess::new(None, Some(¶ms)).expect("start hyperd over IPC"); + let dir = hyper + .socket_directory() + .expect("ipc socket dir") + .to_path_buf(); + assert!(dir.exists()); + dir + }; + assert!(!dir.exists(), "default socket dir should be cleaned up"); +} From 8d1f078f6ffa8c58c20bd026be3c233c24c18b24 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 20:47:34 -0700 Subject: [PATCH 10/61] fix(api): apply query_timeout and bind query_as!/query_scalar! arguments ConnectionBuilder/AsyncConnectionBuilder::query_timeout was stored and never used; it is now applied as the session's query_timeout (hyperd ignores it as a startup parameter), with zero and gRPC rejected at build(). query_as!/query_scalar! accepted $N arguments and discarded them. QueryAs::new and QueryScalar::new now take &[&dyn ToSqlParam] and bind through fetch_*_as_params. The compile-time dry-run now prepares parameterized SQL instead of failing it. --- hyperdb-api-derive/CHANGELOG.md | 6 + hyperdb-api-derive/README.md | 2 +- hyperdb-api/CHANGELOG.md | 19 +++ hyperdb-api/src/async_connection_builder.rs | 33 ++++- hyperdb-api/src/connection_builder.rs | 46 ++++++- hyperdb-api/src/query_as.rs | 126 +++++++++++++----- .../tests/compile_time_validation_tests.rs | 77 ++++++++++- hyperdb-api/tests/query_timeout_tests.rs | 80 +++++++++++ hyperdb-compile-check/CHANGELOG.md | 7 + hyperdb-compile-check/src/dry_run.rs | 76 +++++++++++ 10 files changed, 428 insertions(+), 44 deletions(-) create mode 100644 hyperdb-api/tests/query_timeout_tests.rs diff --git a/hyperdb-api-derive/CHANGELOG.md b/hyperdb-api-derive/CHANGELOG.md index 6d5b2ba6..cc69af3c 100644 --- a/hyperdb-api-derive/CHANGELOG.md +++ b/hyperdb-api-derive/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed + +- **`query_as!` and `query_scalar!` now bind their `$N` arguments.** They were + accepted and silently discarded. Each argument must implement + `hyperdb_api::ToSqlParam`; the macros now generate code that requires it. + ## [1.0.0-rc.3] - 2026-09-07 ### Changed diff --git a/hyperdb-api-derive/README.md b/hyperdb-api-derive/README.md index c623e5c8..8a915683 100644 --- a/hyperdb-api-derive/README.md +++ b/hyperdb-api-derive/README.md @@ -230,6 +230,6 @@ After RA finishes indexing you'll see squigglies on bad SQL strings and errors i ## Known limitations - **Type checking not yet implemented** — only column *names* are validated. Runtime `Error::Column { kind: TypeMismatch }` still catches type drift. -- **No parameter type checking** — bind parameters are opaque at compile time. +- **No parameter type checking** — bind arguments are bound at run time (each must implement `ToSqlParam`), but their types are not checked against the `$N` placeholders at compile time. - **Validates struct vs. SQL, not SQL vs. production DB** — struct/prod schema drift is still a runtime error. - **`INSERT`/`UPDATE`/`DELETE` without `RETURNING`** are not supported by `query_as!`; use `Connection::execute_command` directly. diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 56b882f9..699e4ba1 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -7,8 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **`QueryAs::new` and `QueryScalar::new` now take `&[&dyn ToSqlParam]`** + instead of `&[&dyn Debug]`. These constructors are emitted by the + `query_as!` / `query_scalar!` macros and are not meant to be called + directly, but code that did call them must now pass `ToSqlParam` values. + ### Fixed +- **`query_as!` / `query_scalar!` silently dropped their `$N` bind + arguments.** The arguments were stringified and never sent, so any query + with a placeholder failed at the server with "expected to have 0 + parameter(s)". They are now encoded and bound through the same path as + `Connection::fetch_all_as_params`. +- **`ConnectionBuilder::query_timeout` and + `AsyncConnectionBuilder::query_timeout` were recorded and never applied.** + The timeout is now set as the session's `query_timeout` right after + connecting, so `hyperd` cancels a statement that runs longer (SQLSTATE + 57014, "canceled"). A zero timeout and a timeout on a gRPC connection are + rejected at `build()` instead of being ignored. + - **`Date`, `Timestamp` and `OffsetTimestamp` query parameters were bound with the wrong epoch.** `ToSqlParam::encode_param` sent absolute Julian-based values, but Hyper's binary `Bind` expects PostgreSQL-epoch values (days / diff --git a/hyperdb-api/src/async_connection_builder.rs b/hyperdb-api/src/async_connection_builder.rs index 759d2468..c507456d 100644 --- a/hyperdb-api/src/async_connection_builder.rs +++ b/hyperdb-api/src/async_connection_builder.rs @@ -14,6 +14,7 @@ use std::time::Duration; use crate::async_connection::AsyncConnection; use crate::async_transport::AsyncTransport; use crate::connection::CreateMode; +use crate::connection_builder::query_timeout_statement; use crate::error::{Error, Result}; use crate::transport::{TransportType, detect_transport_type}; use hyperdb_api_core::client::{AsyncClient, Config}; @@ -112,8 +113,19 @@ impl AsyncConnectionBuilder { /// Sets the query timeout. /// - /// The value is recorded on the builder but no transport applies it, so - /// queries run until completion regardless of this setting. + /// Applied to the session as `hyperd`'s `query_timeout` setting right after + /// connecting, so the server cancels any statement on this connection that + /// runs longer than `timeout` and reports an error. Sub-millisecond + /// precision is rounded up to a whole millisecond. + /// + /// Applies to TCP, Unix-socket and named-pipe connections. `hyperd` + /// clamps the value to its own `query_timeout_max` ceiling. A gRPC + /// connection has no equivalent, so [`build`](Self::build) fails with + /// [`Error::FeatureNotSupported`] rather than ignoring the setting. + /// + /// # Errors + /// + /// [`build`](Self::build) returns an error if `timeout` is zero. #[must_use] pub fn query_timeout(mut self, timeout: Duration) -> Self { self.query_timeout = Some(timeout); @@ -222,6 +234,10 @@ impl AsyncConnectionBuilder { let client = AsyncClient::connect(&config).await?; let conn = AsyncConnection::from_async_client(client, db_path_str.clone()); + if let Some(timeout) = self.query_timeout { + conn.execute_command(&query_timeout_statement(timeout)?) + .await?; + } if let Some(db_path) = db_path_str { conn.handle_creation_mode_public(&db_path, self.create_mode) @@ -265,6 +281,10 @@ impl AsyncConnectionBuilder { let client = AsyncClient::connect_unix(&socket_path, &config).await?; let conn = AsyncConnection::from_async_client(client, db_path_str.clone()); + if let Some(timeout) = self.query_timeout { + conn.execute_command(&query_timeout_statement(timeout)?) + .await?; + } if let Some(db_path) = db_path_str { conn.handle_creation_mode_public(&db_path, self.create_mode) @@ -308,6 +328,10 @@ impl AsyncConnectionBuilder { let client = AsyncClient::connect_named_pipe(&pipe_path, &config).await?; let conn = AsyncConnection::from_async_client(client, db_path_str.clone()); + if let Some(timeout) = self.query_timeout { + conn.execute_command(&query_timeout_statement(timeout)?) + .await?; + } if let Some(db_path) = db_path_str { conn.handle_creation_mode_public(&db_path, self.create_mode) @@ -320,6 +344,11 @@ impl AsyncConnectionBuilder { /// Build a gRPC connection (async). async fn build_grpc(self) -> Result { + if self.query_timeout.is_some() { + return Err(Error::feature_not_supported( + "query_timeout is not supported on gRPC connections", + )); + } if self.create_mode != CreateMode::DoNotCreate { return Err(Error::feature_not_supported( "gRPC transport is read-only. Use CreateMode::DoNotCreate for gRPC connections.", diff --git a/hyperdb-api/src/connection_builder.rs b/hyperdb-api/src/connection_builder.rs index 5c01dcf7..99f0b589 100644 --- a/hyperdb-api/src/connection_builder.rs +++ b/hyperdb-api/src/connection_builder.rs @@ -57,8 +57,7 @@ pub struct ConnectionBuilder { user: Option, password: Option, login_timeout: Option, - /// Query timeout; recorded but not applied by any transport (see the - /// `query_timeout` setter). + /// Per-query timeout, sent to the server as the `query_timeout` setting. query_timeout: Option, /// Application name sent to the server during connection startup. application_name: Option, @@ -72,6 +71,19 @@ impl Default for ConnectionBuilder { } } +/// Renders the `SET` statement that applies `timeout` as the session's +/// `query_timeout` (whole milliseconds, rounded up). +/// +/// `hyperd` ignores `query_timeout` as a startup parameter, so it has to be +/// set on the established session. +pub(crate) fn query_timeout_statement(timeout: Duration) -> Result { + if timeout.is_zero() { + return Err(Error::config("query_timeout must be greater than zero")); + } + let millis = timeout.as_nanos().div_ceil(1_000_000); + Ok(format!("SET query_timeout = '{millis}ms'")) +} + impl ConnectionBuilder { /// Creates a new builder for the given endpoint. /// @@ -133,8 +145,19 @@ impl ConnectionBuilder { /// Sets the query timeout. /// - /// The value is recorded on the builder but no transport applies it, so - /// queries run until completion regardless of this setting. + /// Applied to the session as `hyperd`'s `query_timeout` setting right after + /// connecting, so the server cancels any statement on this connection that + /// runs longer than `timeout` and reports an error. Sub-millisecond + /// precision is rounded up to a whole millisecond. + /// + /// Applies to TCP, Unix-socket and named-pipe connections. `hyperd` + /// clamps the value to its own `query_timeout_max` ceiling. A gRPC + /// connection has no equivalent, so [`build`](Self::build) fails with + /// [`Error::FeatureNotSupported`] rather than ignoring the setting. + /// + /// # Errors + /// + /// [`build`](Self::build) returns an error if `timeout` is zero. #[must_use] pub fn query_timeout(mut self, timeout: Duration) -> Self { self.query_timeout = Some(timeout); @@ -272,6 +295,9 @@ impl ConnectionBuilder { let client = hyperdb_api_core::client::Client::connect(&config)?; let conn = Connection::from_client(client, db_path_str.clone()); + if let Some(timeout) = self.query_timeout { + conn.execute_command(&query_timeout_statement(timeout)?)?; + } // Handle database creation (TCP only - gRPC is read-only) if let Some(db_path) = db_path_str { @@ -322,6 +348,9 @@ impl ConnectionBuilder { let client = hyperdb_api_core::client::Client::connect_unix(&socket_path, &config)?; let conn = Connection::from_client(client, db_path_str.clone()); + if let Some(timeout) = self.query_timeout { + conn.execute_command(&query_timeout_statement(timeout)?)?; + } // Handle database creation if let Some(db_path) = db_path_str { @@ -372,6 +401,9 @@ impl ConnectionBuilder { let client = hyperdb_api_core::client::Client::connect_named_pipe(&pipe_path, &config)?; let conn = Connection::from_client(client, db_path_str.clone()); + if let Some(timeout) = self.query_timeout { + conn.execute_command(&query_timeout_statement(timeout)?)?; + } // Handle database creation if let Some(db_path) = db_path_str { @@ -384,6 +416,12 @@ impl ConnectionBuilder { /// Build a gRPC connection. fn build_grpc(self) -> Result { + if self.query_timeout.is_some() { + return Err(Error::feature_not_supported( + "query_timeout is not supported on gRPC connections", + )); + } + // Validate create_mode - gRPC is read-only if self.create_mode != CreateMode::DoNotCreate { return Err(Error::feature_not_supported( diff --git a/hyperdb-api/src/query_as.rs b/hyperdb-api/src/query_as.rs index 0c628704..bcd3c095 100644 --- a/hyperdb-api/src/query_as.rs +++ b/hyperdb-api/src/query_as.rs @@ -5,7 +5,59 @@ use std::marker::PhantomData; +use crate::params::{ParamFormat, ToSqlParam}; use crate::{Connection, FromRow, Result, RowValue}; +use hyperdb_api_core::types::Oid; + +/// A bind argument encoded eagerly so the query object owns it. +/// +/// `QueryAs` / `QueryScalar` are built from borrowed macro arguments but +/// executed later, so the arguments are captured as their wire encoding +/// rather than as borrows. +#[derive(Debug)] +struct BoundParam { + bytes: Option>, + format: ParamFormat, + oid: Oid, + literal: String, +} + +impl BoundParam { + fn capture(p: &dyn ToSqlParam) -> Self { + Self { + bytes: p.encode_param(), + format: p.param_format(), + oid: p.sql_oid(), + literal: p.to_sql_literal(), + } + } +} + +impl ToSqlParam for BoundParam { + fn encode_param(&self) -> Option> { + self.bytes.clone() + } + + fn param_format(&self) -> ParamFormat { + self.format + } + + fn sql_oid(&self) -> Oid { + self.oid + } + + fn to_sql_literal(&self) -> String { + self.literal.clone() + } +} + +fn capture_all(params: &[&dyn ToSqlParam]) -> Vec { + params.iter().map(|p| BoundParam::capture(*p)).collect() +} + +fn as_refs(params: &[BoundParam]) -> Vec<&dyn ToSqlParam> { + params.iter().map(|p| p as &dyn ToSqlParam).collect() +} /// A compiled, type-safe query. Created by the `query_as!` macro. /// @@ -14,18 +66,7 @@ use crate::{Connection, FromRow, Result, RowValue}; #[derive(Debug)] pub struct QueryAs { sql: String, - // Bind parameters are stored as formatted strings for now — the macro - // accepts `$N` args and validates the SQL, but binding is not yet wired - // (the `fetch_*` methods below forward to the NON-param `fetch_*_as`). - // - // To finish this: change `params` to hold `ToSqlParam` values and route - // through `Connection::fetch_*_as_params` (added in issue #137 — the - // parameterized FromRow methods are exactly the primitive this needs). - #[allow( - dead_code, - reason = "typed parameter binding not yet wired — see issue #137" - )] - params: Vec, + params: Vec, _phantom: PhantomData T>, } @@ -33,13 +74,12 @@ impl QueryAs { /// Construct a new `QueryAs`. Called by the `query_as!` macro; not intended /// for direct use. /// - /// `params` accepts `&dyn std::fmt::Debug` so the macro can pass any bind - /// arguments through — typed binding via `ToSqlParam` is not yet wired - /// (see the `TODO(#137)` on `fetch_all` below). - pub fn new(sql: &str, params: &[&dyn std::fmt::Debug]) -> Self { + /// `params` are the `$1`, `$2`, … bind arguments. Each is encoded + /// immediately and bound by the server at execution time. + pub fn new(sql: &str, params: &[&dyn ToSqlParam]) -> Self { Self { sql: sql.to_owned(), - params: params.iter().map(|p| format!("{p:?}")).collect(), + params: capture_all(params), _phantom: PhantomData, } } @@ -51,9 +91,7 @@ impl QueryAs { /// Returns a `hyperdb_api::Error` on connection failure, SQL error, or /// row-mapping failure. pub fn fetch_all(self, conn: &Connection) -> Result> { - // TODO(#137): forward to `conn.fetch_all_as_params(&self.sql, ¶ms)` - // once `params` holds `ToSqlParam` values, to actually bind `$N` args. - conn.fetch_all_as(&self.sql) + self.fetch_rows(conn) } /// Execute the query and return exactly one row. @@ -63,7 +101,11 @@ impl QueryAs { /// Returns `Error::Conversion` if the query returns zero rows. /// Returns a `hyperdb_api::Error` on connection or SQL failure. pub fn fetch_one(self, conn: &Connection) -> Result { - conn.fetch_one_as(&self.sql) + if self.params.is_empty() { + conn.fetch_one_as(&self.sql) + } else { + conn.fetch_one_as_params(&self.sql, &as_refs(&self.params)) + } } /// Execute the query and return `Some(row)` for the first row, or `None` @@ -73,8 +115,16 @@ impl QueryAs { /// /// Returns a `hyperdb_api::Error` on connection or SQL failure. pub fn fetch_optional(self, conn: &Connection) -> Result> { - let rows = conn.fetch_all_as::(&self.sql)?; - Ok(rows.into_iter().next()) + Ok(self.fetch_rows(conn)?.into_iter().next()) + } + + /// Runs the query, binding `$N` arguments when there are any. + fn fetch_rows(&self, conn: &Connection) -> Result> { + if self.params.is_empty() { + conn.fetch_all_as(&self.sql) + } else { + conn.fetch_all_as_params(&self.sql, &as_refs(&self.params)) + } } } @@ -92,23 +142,18 @@ impl QueryAs { #[derive(Debug)] pub struct QueryScalar { sql: String, - // Same gap as `QueryAs::params` — the macro validates the SQL and - // accepts args, but binding isn't wired yet. Route through - // `fetch_scalar_params` (or equivalent) once it exists. - #[allow( - dead_code, - reason = "typed parameter binding not yet wired — see issue #137" - )] - params: Vec, + params: Vec, _phantom: PhantomData T>, } impl QueryScalar { /// Construct a new `QueryScalar`. Called by the `query_scalar!` macro. - pub fn new(sql: &str, params: &[&dyn std::fmt::Debug]) -> Self { + /// + /// `params` are the `$1`, `$2`, … bind arguments, encoded immediately. + pub fn new(sql: &str, params: &[&dyn ToSqlParam]) -> Self { Self { sql: sql.to_owned(), - params: params.iter().map(|p| format!("{p:?}")).collect(), + params: capture_all(params), _phantom: PhantomData, } } @@ -120,7 +165,7 @@ impl QueryScalar { /// Returns a `hyperdb_api::Error` on connection failure, SQL error, or /// type conversion failure. pub fn fetch_all(self, conn: &Connection) -> Result> { - conn.fetch_all_as::>(&self.sql) + self.fetch_rows(conn) .map(|rows| rows.into_iter().map(|r| r.0).collect()) } @@ -130,7 +175,7 @@ impl QueryScalar { /// /// Returns `Error::Conversion` if the query returns zero rows. pub fn fetch_one(self, conn: &Connection) -> Result { - let rows = conn.fetch_all_as::>(&self.sql)?; + let rows = self.fetch_rows(conn)?; rows.into_iter() .next() .map(|r| r.0) @@ -143,9 +188,18 @@ impl QueryScalar { /// /// Returns a `hyperdb_api::Error` on connection or SQL failure. pub fn fetch_optional(self, conn: &Connection) -> Result> { - let rows = conn.fetch_all_as::>(&self.sql)?; + let rows = self.fetch_rows(conn)?; Ok(rows.into_iter().next().map(|r| r.0)) } + + /// Runs the query, binding `$N` arguments when there are any. + fn fetch_rows(&self, conn: &Connection) -> Result>> { + if self.params.is_empty() { + conn.fetch_all_as(&self.sql) + } else { + conn.fetch_all_as_params(&self.sql, &as_refs(&self.params)) + } + } } /// Internal single-column `FromRow` wrapper for `QueryScalar` methods. diff --git a/hyperdb-api/tests/compile_time_validation_tests.rs b/hyperdb-api/tests/compile_time_validation_tests.rs index 2af74eaf..9742f632 100644 --- a/hyperdb-api/tests/compile_time_validation_tests.rs +++ b/hyperdb-api/tests/compile_time_validation_tests.rs @@ -20,7 +20,7 @@ mod common; use common::TestConnection; use hyperdb_api::Table; -use hyperdb_api_derive::{FromRow, Table, query_as}; +use hyperdb_api_derive::{FromRow, Table, query_as, query_scalar}; // --------------------------------------------------------------------------- // Test structs — derive(Table) registers them at compile time when @@ -176,3 +176,78 @@ fn query_as_join_two_registered_tables() { assert_eq!(users.len(), 1); assert_eq!(users[0].name, "Eve"); } + +// --------------------------------------------------------------------------- +// Bind arguments +// --------------------------------------------------------------------------- + +/// `$N` arguments given to `query_as!` are bound, not dropped. +#[test] +fn query_as_binds_arguments() { + let test = TestConnection::new().expect("TestConnection"); + test.execute_command(CtUser::CREATE_SQL) + .expect("create ct_users"); + test.execute_command( + "INSERT INTO ct_users VALUES (1, 'Alice', 95.5), (2, 'Bob', 10.0), (3, 'Cy', NULL)", + ) + .expect("insert"); + + let name = String::from("Bob"); + let users: Vec = query_as!( + CtUser, + "SELECT id, name, score FROM ct_users WHERE name = $1 AND id >= $2", + name, + 1_i64 + ) + .fetch_all(&test.connection) + .expect("fetch_all with binds"); + assert_eq!(users.len(), 1); + assert_eq!(users[0].name, "Bob"); + + let one: CtUser = query_as!( + CtUser, + "SELECT id, name, score FROM ct_users WHERE id = $1", + 3_i64 + ) + .fetch_one(&test.connection) + .expect("fetch_one with binds"); + assert_eq!(one.name, "Cy"); + + let none: Option = query_as!( + CtUser, + "SELECT id, name, score FROM ct_users WHERE id = $1", + 99_i64 + ) + .fetch_optional(&test.connection) + .expect("fetch_optional with binds"); + assert!(none.is_none()); +} + +/// `$N` arguments given to `query_scalar!` are bound, not dropped. +#[test] +fn query_scalar_binds_arguments() { + let test = TestConnection::new().expect("TestConnection"); + test.execute_command(CtUser::CREATE_SQL) + .expect("create ct_users"); + test.execute_command("INSERT INTO ct_users VALUES (1, 'Alice', 95.5), (2, 'Bob', 10.0)") + .expect("insert"); + + let n: i64 = query_scalar!( + i64, + "SELECT COUNT(*) FROM ct_users WHERE score > $1", + 50.0_f64 + ) + .fetch_one(&test.connection) + .expect("scalar with bind"); + assert_eq!(n, 1); + + let names: Vec = query_scalar!( + String, + "SELECT name FROM ct_users WHERE id >= $1 AND id <= $2 ORDER BY id", + 1_i64, + 2_i64 + ) + .fetch_all(&test.connection) + .expect("scalar fetch_all with binds"); + assert_eq!(names, ["Alice", "Bob"]); +} diff --git a/hyperdb-api/tests/query_timeout_tests.rs b/hyperdb-api/tests/query_timeout_tests.rs new file mode 100644 index 00000000..17e9a5e1 --- /dev/null +++ b/hyperdb-api/tests/query_timeout_tests.rs @@ -0,0 +1,80 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! `query_timeout` on both connection builders is enforced by the server. + +mod common; + +use std::time::{Duration, Instant}; + +use common::test_hyper_params; +use hyperdb_api::{AsyncConnectionBuilder, ConnectionBuilder, HyperProcess}; + +/// A cross join large enough to run for far longer than the timeout. +const SLOW_QUERY: &str = "SELECT COUNT(*) FROM generate_series(1, 20000) AS a(x), \ + generate_series(1, 100000) AS b(y) WHERE (a.x::BIGINT * b.y) % 7 = 3"; + +#[test] +fn sync_query_timeout_cancels_slow_query() { + let params = test_hyper_params("sync_query_timeout").unwrap(); + let hyper = HyperProcess::new(None, Some(¶ms)).unwrap(); + let endpoint = hyper.require_endpoint().unwrap().to_string(); + + let conn = ConnectionBuilder::new(&endpoint) + .query_timeout(Duration::from_millis(100)) + .build() + .unwrap(); + + let start = Instant::now(); + let result: hyperdb_api::Result = conn.fetch_scalar(SLOW_QUERY); + eprintln!("ELAPSED {:?}", start.elapsed()); + let err = result.expect_err("slow query should be cancelled"); + assert!( + err.to_string().to_lowercase().contains("cancel"), + "unexpected error: {err}" + ); + assert!( + start.elapsed() < Duration::from_secs(30), + "query ran for {:?}, timeout not enforced", + start.elapsed() + ); + + // The connection remains usable and fast queries still succeed. + let one: i64 = conn.fetch_scalar("SELECT 1").unwrap(); + assert_eq!(one, 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn async_query_timeout_cancels_slow_query() { + let params = test_hyper_params("async_query_timeout").unwrap(); + let hyper = HyperProcess::new(None, Some(¶ms)).unwrap(); + let endpoint = hyper.require_endpoint().unwrap().to_string(); + + let conn = AsyncConnectionBuilder::new(&endpoint) + .query_timeout(Duration::from_millis(100)) + .build() + .await + .unwrap(); + + let start = Instant::now(); + let result: hyperdb_api::Result = conn.fetch_scalar(SLOW_QUERY).await; + eprintln!("ELAPSED {:?}", start.elapsed()); + let err = result.expect_err("slow query should be cancelled"); + assert!( + err.to_string().to_lowercase().contains("cancel"), + "unexpected error: {err}" + ); + assert!(start.elapsed() < Duration::from_secs(30)); +} + +#[test] +fn zero_query_timeout_is_rejected() { + let params = test_hyper_params("zero_query_timeout").unwrap(); + let hyper = HyperProcess::new(None, Some(¶ms)).unwrap(); + let endpoint = hyper.require_endpoint().unwrap().to_string(); + + let result = ConnectionBuilder::new(&endpoint) + .query_timeout(Duration::ZERO) + .build(); + assert!(result.is_err()); +} diff --git a/hyperdb-compile-check/CHANGELOG.md b/hyperdb-compile-check/CHANGELOG.md index 8c3f1484..755e7b8c 100644 --- a/hyperdb-compile-check/CHANGELOG.md +++ b/hyperdb-compile-check/CHANGELOG.md @@ -12,6 +12,13 @@ surface, and consumers should depend on `hyperdb-api-derive` instead. ## [Unreleased] +### Fixed + +- **SQL containing `$N` placeholders is now validated.** The dry-run executed + the statement with nothing bound, so Hyper rejected every parameterized + query with "expected to have 0 parameter(s)". Such queries are now + prepared with one unspecified-type parameter per placeholder. + ## [1.0.0-rc.3] - 2026-09-07 ### Fixed diff --git a/hyperdb-compile-check/src/dry_run.rs b/hyperdb-compile-check/src/dry_run.rs index db30ccdd..00492452 100644 --- a/hyperdb-compile-check/src/dry_run.rs +++ b/hyperdb-compile-check/src/dry_run.rs @@ -30,6 +30,15 @@ use crate::db::CompileTimeDb; /// [`crate::error_extract::classify`]). pub fn dry_run(db: &mut CompileTimeDb, user_sql: &str) -> Result { let wrapped = format!("WITH __hdb_q AS ({user_sql}) SELECT * FROM __hdb_q LIMIT 0"); + // A `$N` placeholder can't be executed without bound values, and Hyper + // rejects NULL in some positions. Preparing the statement (Parse/Describe, + // no Bind) lets the server infer the parameter types and still report the + // result schema, which is all validation needs. + let placeholders = placeholder_count(user_sql); + if placeholders > 0 { + let oids = vec![hyperdb_api::Oid::new(0); placeholders]; + return Ok(db.conn.prepare_typed(&wrapped, &oids)?.schema().clone()); + } let mut rowset = db.conn.execute_query(&wrapped)?; // Force execution: LIMIT 0 returns Ok(None) from next_chunk @@ -41,6 +50,58 @@ pub fn dry_run(db: &mut CompileTimeDb, user_sql: &str) -> Result { .ok_or_else(|| Error::Protocol("dry-run: schema missing after next_chunk".into())) } +/// The highest `$N` placeholder in `sql`, ignoring quoted strings and +/// identifiers. Returns 0 when there are none. +fn placeholder_count(sql: &str) -> usize { + let bytes = sql.as_bytes(); + let mut max = 0usize; + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + quote @ (b'\'' | b'"') => { + i += 1; + while i < bytes.len() && bytes[i] != quote { + i += 1; + } + } + b'$' => { + let digits = bytes[i + 1..] + .iter() + .take_while(|b| b.is_ascii_digit()) + .count(); + if digits > 0 + && let Ok(n) = sql[i + 1..i + 1 + digits].parse::() + { + max = max.max(n); + } + i += digits; + } + _ => {} + } + i += 1; + } + max +} + +#[cfg(test)] +mod placeholder_tests { + use super::placeholder_count; + + #[test] + fn counts_highest_placeholder() { + assert_eq!(placeholder_count("SELECT 1"), 0); + assert_eq!(placeholder_count("SELECT * FROM t WHERE a = $1"), 1); + assert_eq!(placeholder_count("WHERE a = $2 AND b = $1"), 2); + assert_eq!(placeholder_count("WHERE a = $10"), 10); + } + + #[test] + fn ignores_quoted_text() { + assert_eq!(placeholder_count("SELECT '$3', \"$4\""), 0); + assert_eq!(placeholder_count("SELECT 'it''s $5' WHERE a = $1"), 1); + } +} + #[cfg(test)] mod tests { use super::*; @@ -90,6 +151,21 @@ mod tests { assert_eq!(names, &["a", "b"]); } + #[test] + #[ignore = "requires HYPERD_PATH; run manually"] + fn dry_run_with_bind_placeholders() { + let mut db = get_or_init().lock(); + db.conn + .execute_command("CREATE TEMP TABLE IF NOT EXISTS _dr_bind (id BIGINT, name TEXT)") + .unwrap(); + let schema = dry_run( + &mut db, + "SELECT id, name FROM _dr_bind WHERE name = $1 AND id >= $2", + ) + .unwrap(); + assert_eq!(schema.column_count(), 2); + } + #[test] #[ignore = "requires HYPERD_PATH; run manually"] fn dry_run_bad_table_returns_error() { From 0860a3f234d6bb99b40df04aa9aa246c7773ee5b Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 22:14:12 -0700 Subject: [PATCH 11/61] fix(core): send gRPC JSON parameters in hyperd's typed format QueryParameters::json_positional and json_named emitted bare values and a JSON object, which hyperd rejects with 22023. hyperd requires an array of typed entries ({"type","value"} plus "name" for named parameters). Both now emit that format, inferring bool/bigint/numeric/float8/varchar from the JSON value and rejecting arrays and objects. Doc examples that passed raw value arrays through from_json_value, or used the keyword :name as a parameter, are corrected. --- hyperdb-api-core/CHANGELOG.md | 10 + hyperdb-api-core/src/client/grpc/client.rs | 7 +- hyperdb-api-core/src/client/grpc/mod.rs | 14 +- hyperdb-api-core/src/client/grpc/params.rs | 273 +++++++++++++++------ hyperdb-api-core/src/client/mod.rs | 9 +- hyperdb-api/tests/grpc_params_tests.rs | 167 +++++++++++++ 6 files changed, 389 insertions(+), 91 deletions(-) create mode 100644 hyperdb-api/tests/grpc_params_tests.rs diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index 092ccc8e..ed1ef272 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -24,6 +24,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **`AuthenticatedGrpcClient::get_table_labels` / `get_column_labels` escape their `schema` and `table` arguments** instead of interpolating them into the catalog query unescaped. +- **gRPC `QueryParameters::json_positional` and `json_named` produced JSON + that `hyperd` rejects** (`22023 invalid JSON query parameters`). They sent + bare values (`[42,"hi"]`) and an object (`{"id":42}`). `hyperd` requires an + array of typed entries (`[{"type":"bigint","value":"42"}]`, plus `"name"` for + named parameters), which both now emit. The Hyper type is inferred from the + JSON value (`bool`, `bigint`, `numeric` above `i64::MAX`, `float8`, + `varchar`; `null` as a nullable `varchar`); array and object values return + an error. The docs for `from_json_string` / `from_json_value`, which pass + JSON through unchanged, now describe the typed format instead of showing + examples `hyperd` would reject. ## [1.0.0-rc.3] - 2026-09-07 diff --git a/hyperdb-api-core/src/client/grpc/client.rs b/hyperdb-api-core/src/client/grpc/client.rs index 82d0eaf4..c3b44c2b 100644 --- a/hyperdb-api-core/src/client/grpc/client.rs +++ b/hyperdb-api-core/src/client/grpc/client.rs @@ -220,8 +220,11 @@ impl GrpcClient { /// # async fn example() -> Result<(), Box> { /// # let config = GrpcConfig::new("http://localhost:7484"); /// # let mut client = GrpcClient::connect(config).await?; - /// // Dollar-numbered parameters (mixed types use from_json_value) - /// let params = QueryParameters::from_json_value(&serde_json::json!([42, "Alice"]))?; + /// // Dollar-numbered parameters (mixed types use serde_json::json!) + /// let params = QueryParameters::json_positional(&[ + /// &serde_json::json!(42), + /// &serde_json::json!("Alice"), + /// ])?; /// let result = client.execute_query_with_params( /// "SELECT * FROM users WHERE id = $1 AND name = $2", /// params, diff --git a/hyperdb-api-core/src/client/grpc/mod.rs b/hyperdb-api-core/src/client/grpc/mod.rs index ad753e3b..2a89565f 100644 --- a/hyperdb-api-core/src/client/grpc/mod.rs +++ b/hyperdb-api-core/src/client/grpc/mod.rs @@ -55,21 +55,25 @@ //! # let config = GrpcConfig::new("http://localhost:7484"); //! let mut client = GrpcClient::connect(config).await?; //! -//! // Dollar-numbered parameters ($1, $2, ...) - use from_json_value for mixed types -//! let params = QueryParameters::from_json_value(&serde_json::json!([42, "Alice"]))?; +//! // Dollar-numbered parameters ($1, $2, ...) - use serde_json::json! for mixed types +//! let params = QueryParameters::json_positional(&[ +//! &serde_json::json!(42), +//! &serde_json::json!("Alice"), +//! ])?; //! let result = client.execute_query_with_params( //! "SELECT * FROM users WHERE id = $1 AND name = $2", //! params, //! ParameterStyle::DollarNumbered, //! ).await?; //! -//! // Named parameters (:id, :name, ...) +//! // Named parameters (:id, :user_name, ...). A keyword such as `name` +//! // must be quoted (`:"name"`) to be used as a parameter name. //! let params = QueryParameters::json_named() //! .add("id", &42i64)? -//! .add("name", &"Alice")? +//! .add("user_name", &"Alice")? //! .build(); //! let result = client.execute_query_with_params( -//! "SELECT * FROM users WHERE id = :id AND name = :name", +//! "SELECT * FROM users WHERE id = :id AND name = :user_name", //! params, //! ParameterStyle::Named, //! ).await?; diff --git a/hyperdb-api-core/src/client/grpc/params.rs b/hyperdb-api-core/src/client/grpc/params.rs index 2a34add7..2f045dcb 100644 --- a/hyperdb-api-core/src/client/grpc/params.rs +++ b/hyperdb-api-core/src/client/grpc/params.rs @@ -15,21 +15,25 @@ //! # let config = GrpcConfig::new("http://localhost:7484"); //! let mut client = GrpcClient::connect(config).await?; //! -//! // Using JSON parameters with $1, $2 style (use from_json_value for mixed types) -//! let params = QueryParameters::from_json_value(&serde_json::json!([42, "hello"]))?; +//! // Using JSON parameters with $1, $2 style (json! values for mixed types) +//! let params = QueryParameters::json_positional(&[ +//! &serde_json::json!(42), +//! &serde_json::json!("hello"), +//! ])?; //! let result = client.execute_query_with_params( //! "SELECT * FROM users WHERE id = $1 AND name = $2", //! params, //! ParameterStyle::DollarNumbered, //! ).await?; //! -//! // Using JSON parameters with named style +//! // Using JSON parameters with named style. A keyword such as `name` must +//! // be quoted (`:"name"`) to be used as a parameter name. //! let params = QueryParameters::json_named() //! .add("id", &42i64)? -//! .add("name", &"hello")? +//! .add("user_name", &"hello")? //! .build(); //! let result = client.execute_query_with_params( -//! "SELECT * FROM users WHERE id = :id AND name = :name", +//! "SELECT * FROM users WHERE id = :id AND name = :user_name", //! params, //! ParameterStyle::Named, //! ).await?; @@ -39,11 +43,69 @@ use bytes::Bytes; use serde::Serialize; -use serde_json::Value as JsonValue; +use serde::ser::Error as _; +use serde_json::{Map as JsonMap, Value as JsonValue}; use super::proto::hyper_service::query_param::{ParameterStyle as ProtoParameterStyle, Parameters}; use super::proto::hyper_service::{QueryParameterArrow, QueryParameterJson}; +/// Encodes one JSON value as a typed parameter entry in hyperd's JSON wire +/// format: `{"type": "", "value": ""}`. +/// +/// hyperd does not accept bare JSON values as parameters. It requires an +/// array of such entries, each carrying a Hyper type name and the value as +/// text (or `null`), which it then casts to that type. Values map to types as +/// follows: +/// +/// | JSON value | Hyper type | +/// |---|---| +/// | boolean | `bool` | +/// | integer that fits `i64` | `bigint` | +/// | integer above `i64::MAX` | `numeric(20, 0)` | +/// | other number | `float8` | +/// | string | `varchar` | +/// | `null` | nullable `varchar` | +/// +/// Arrays and objects have no unambiguous Hyper type and are rejected. +fn typed_entry(value: JsonValue) -> Result, serde_json::Error> { + let mut entry = JsonMap::new(); + let (type_name, text) = match value { + JsonValue::Null => return Ok(null_entry()), + JsonValue::Bool(b) => ("bool", JsonValue::String(b.to_string())), + JsonValue::Number(n) => { + let type_name = if n.is_i64() { + "bigint" + } else if n.is_u64() { + // u64::MAX has 20 decimal digits + entry.insert("precision".into(), 20.into()); + entry.insert("scale".into(), 0.into()); + "numeric" + } else { + "float8" + }; + (type_name, JsonValue::String(n.to_string())) + } + JsonValue::String(s) => ("varchar", JsonValue::String(s)), + JsonValue::Array(_) | JsonValue::Object(_) => { + return Err(serde_json::Error::custom( + "array and object values cannot be sent as JSON query parameters; \ + build the typed parameter JSON yourself and use from_json_string", + )); + } + }; + entry.insert("type".into(), type_name.into()); + entry.insert("value".into(), text); + Ok(entry) +} + +/// The typed entry for a SQL NULL: a nullable `varchar` with a `null` value. +fn null_entry() -> JsonMap { + let mut entry = JsonMap::new(); + entry.insert("type".into(), "varchar".into()); + entry.insert("value".into(), JsonValue::Null); + entry +} + /// Parameter style for SQL queries. /// /// This determines how parameters are referenced in the SQL query string. @@ -92,7 +154,14 @@ pub enum QueryParameters { } impl QueryParameters { - /// Creates JSON parameters from a JSON string. + /// Creates JSON parameters from a JSON string, sent to hyperd as-is. + /// + /// hyperd expects an array with one typed entry per parameter: + /// `{"type": , "value": }`, plus `"name"` for + /// named parameters. `value` is always a JSON string (or `null`) that + /// hyperd casts to `type`. Use this constructor when you need a type that + /// [`json_positional`](Self::json_positional) does not infer, such as + /// `date` or `numeric` with `precision`/`scale`. /// /// # Example /// @@ -100,23 +169,38 @@ impl QueryParameters { /// use hyperdb_api_core::client::grpc::QueryParameters; /// /// // For positional parameters ($1, $2 or ?) - /// let params = QueryParameters::from_json_string("[42, \"hello\"]"); + /// let params = QueryParameters::from_json_string( + /// r#"[{"type": "integer", "value": "42"}, {"type": "date", "value": "2026-01-31"}]"#, + /// ); /// /// // For named parameters (:id, :name) - /// let params = QueryParameters::from_json_string(r#"{"id": 42, "name": "hello"}"#); + /// let params = QueryParameters::from_json_string( + /// r#"[{"name": "id", "type": "bigint", "value": "42"}, + /// {"name": "name", "type": "varchar", "value": null}]"#, + /// ); /// ``` pub fn from_json_string(json: impl Into) -> Self { QueryParameters::Json(json.into()) } - /// Creates JSON parameters from a serializable value. + /// Creates JSON parameters from a serializable value, sent to hyperd + /// as-is. + /// + /// The value must serialize to the typed-entry array described on + /// [`from_json_string`](Self::from_json_string); it is not converted. + /// To pass plain values, use [`json_positional`](Self::json_positional) + /// or [`json_named`](Self::json_named) instead. /// /// # Example /// /// ``` /// use hyperdb_api_core::client::grpc::QueryParameters; + /// use serde_json::json; /// - /// let params = QueryParameters::from_json_value(&vec![42, 100, 200])?; + /// let params = QueryParameters::from_json_value(&json!([ + /// {"type": "smallint", "value": "7"}, + /// {"type": "numeric", "precision": 10, "scale": 2, "value": "12.50"}, + /// ]))?; /// # Ok::<(), serde_json::Error>(()) /// ``` /// @@ -131,34 +215,42 @@ impl QueryParameters { /// Creates JSON parameters for positional placeholders ($1, $2 or ?). /// - /// Values are serialized as a JSON array. All values must be the same type. - /// For mixed types, use [`from_json_value`](Self::from_json_value) with `serde_json::json!`. + /// Each value is serialized to JSON and encoded as a typed entry in the + /// array format hyperd expects (`[{"type":"bigint","value":"42"}, ...]`). + /// The Hyper type is inferred from the JSON value: booleans become + /// `bool`, integers `bigint` (or `numeric` above `i64::MAX`), other + /// numbers `float8`, strings `varchar`, and `null` a nullable `varchar`. + /// For a different type, cast in SQL (`CAST(? AS DATE)`). + /// + /// The slice is homogeneous in `T`; pass `serde_json::Value`s to mix + /// types. /// /// # Example /// /// ``` /// use hyperdb_api_core::client::grpc::QueryParameters; + /// use serde_json::json; /// /// // Same-type parameters /// let params = QueryParameters::json_positional(&[&42i64, &100i64])?; /// - /// // Mixed types: use from_json_value with serde_json::json! - /// let params = QueryParameters::from_json_value(&serde_json::json!([42, "hello", true]))?; + /// // Mixed types + /// let params = QueryParameters::json_positional(&[&json!(42), &json!("hello"), &json!(true)])?; /// # Ok::<(), serde_json::Error>(()) /// ``` /// /// # Errors /// /// Returns a [`serde_json::Error`] if any element of `values` fails - /// to serialize to JSON. + /// to serialize to JSON, or serializes to a JSON array or object. pub fn json_positional( values: &[&T], ) -> Result { - let json_values: Vec = values + let entries: Vec = values .iter() - .map(serde_json::to_value) + .map(|value| typed_entry(serde_json::to_value(value)?).map(JsonValue::Object)) .collect::>()?; - let json = serde_json::to_string(&json_values)?; + let json = serde_json::to_string(&entries)?; Ok(QueryParameters::Json(json)) } @@ -263,7 +355,8 @@ impl QueryParameters { /// ``` #[derive(Debug, Default)] pub struct JsonNamedParamsBuilder { - params: serde_json::Map, + /// Typed entries in insertion order, each carrying its `name`. + params: Vec, } impl JsonNamedParamsBuilder { @@ -278,26 +371,32 @@ impl JsonNamedParamsBuilder { /// # Arguments /// /// * `name` - Parameter name (without the colon prefix) - /// * `value` - Parameter value (must be JSON-serializable) + /// * `value` - Parameter value (must be JSON-serializable). Its Hyper + /// type is inferred as described on + /// [`QueryParameters::json_positional`]. /// /// # Errors /// /// Returns a [`serde_json::Error`] if `value` cannot be serialized - /// to JSON. + /// to JSON, or serializes to a JSON array or object. pub fn add( - mut self, + self, name: impl Into, value: &T, ) -> Result { - let json_value = serde_json::to_value(value)?; - self.params.insert(name.into(), json_value); - Ok(self) + let entry = typed_entry(serde_json::to_value(value)?)?; + Ok(self.push(name.into(), entry)) } #[must_use] - /// Adds a null parameter. - pub fn add_null(mut self, name: impl Into) -> Self { - self.params.insert(name.into(), JsonValue::Null); + /// Adds a null parameter, sent as a nullable `varchar`. + pub fn add_null(self, name: impl Into) -> Self { + self.push(name.into(), null_entry()) + } + + fn push(mut self, name: String, mut entry: JsonMap) -> Self { + entry.insert("name".into(), name.into()); + self.params.push(JsonValue::Object(entry)); self } @@ -306,14 +405,13 @@ impl JsonNamedParamsBuilder { /// # Panics /// /// Does not panic in practice. The `serde_json::to_string` call on a - /// `Map` is infallible for valid JSON trees — - /// serde_json only fails when a user-defined `Serialize` impl - /// returns an error, which cannot happen for the already-validated - /// `Value` payloads inserted via [`Self::add`] and [`Self::add_null`]. + /// `Value` tree is infallible — serde_json only fails when a + /// user-defined `Serialize` impl returns an error, which cannot happen + /// for the already-converted `Value` payloads inserted via + /// [`Self::add`] and [`Self::add_null`]. #[must_use] pub fn build(self) -> QueryParameters { - let json = serde_json::to_string(&JsonValue::Object(self.params)) - .expect("serializing Map never fails"); + let json = serde_json::to_string(&self.params).expect("serializing a Value never fails"); QueryParameters::Json(json) } } @@ -322,74 +420,87 @@ impl JsonNamedParamsBuilder { mod tests { use super::*; - #[test] - fn test_json_positional_integers() { - let params = QueryParameters::json_positional(&[&42i64, &100i64, &200i64]).unwrap(); + use serde_json::json; + + fn parsed(params: QueryParameters) -> JsonValue { match params { - QueryParameters::Json(json) => { - assert_eq!(json, r"[42,100,200]"); - } + QueryParameters::Json(json) => serde_json::from_str(&json).unwrap(), QueryParameters::Arrow(_) => panic!("Expected JSON parameters"), } } + #[test] + fn test_json_positional_integers() { + let params = QueryParameters::json_positional(&[&42i64, &-7i64]).unwrap(); + assert_eq!( + parsed(params), + json!([ + {"type": "bigint", "value": "42"}, + {"type": "bigint", "value": "-7"}, + ]) + ); + } + #[test] fn test_json_positional_strings() { let params = QueryParameters::json_positional(&[&"hello", &"world"]).unwrap(); - match params { - QueryParameters::Json(json) => { - assert_eq!(json, r#"["hello","world"]"#); - } - QueryParameters::Arrow(_) => panic!("Expected JSON parameters"), - } + assert_eq!( + parsed(params), + json!([ + {"type": "varchar", "value": "hello"}, + {"type": "varchar", "value": "world"}, + ]) + ); } #[test] - fn test_json_positional_mixed_via_value() { - // For mixed types, use from_json_value with serde_json::json! - let values = serde_json::json!([42, "hello", true]); - let params = QueryParameters::from_json_value(&values).unwrap(); - match params { - QueryParameters::Json(json) => { - assert_eq!(json, r#"[42,"hello",true]"#); - } - QueryParameters::Arrow(_) => panic!("Expected JSON parameters"), - } + fn test_json_positional_mixed() { + let params = QueryParameters::json_positional(&[ + &json!(true), + &json!(1.5), + &json!(u64::MAX), + &json!(null), + ]) + .unwrap(); + assert_eq!( + parsed(params), + json!([ + {"type": "bool", "value": "true"}, + {"type": "float8", "value": "1.5"}, + {"type": "numeric", "precision": 20, "scale": 0, "value": "18446744073709551615"}, + {"type": "varchar", "value": null}, + ]) + ); } #[test] - fn test_json_named() { - let params = QueryParameters::json_named() - .add("id", &42i64) - .unwrap() - .add("name", &"Alice") - .unwrap() - .build(); - match params { - QueryParameters::Json(json) => { - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["id"], 42); - assert_eq!(parsed["name"], "Alice"); - } - QueryParameters::Arrow(_) => panic!("Expected JSON parameters"), - } + fn test_json_positional_rejects_arrays_and_objects() { + assert!(QueryParameters::json_positional(&[&json!([1, 2])]).is_err()); + assert!(QueryParameters::json_positional(&[&json!({"a": 1})]).is_err()); + assert!( + QueryParameters::json_named() + .add("list", &vec![1, 2]) + .is_err() + ); } #[test] - fn test_json_named_with_null() { + fn test_json_named() { let params = QueryParameters::json_named() .add("id", &42i64) .unwrap() + .add("name", &"Alice") + .unwrap() .add_null("optional") .build(); - match params { - QueryParameters::Json(json) => { - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["id"], 42); - assert!(parsed["optional"].is_null()); - } - QueryParameters::Arrow(_) => panic!("Expected JSON parameters"), - } + assert_eq!( + parsed(params), + json!([ + {"name": "id", "type": "bigint", "value": "42"}, + {"name": "name", "type": "varchar", "value": "Alice"}, + {"name": "optional", "type": "varchar", "value": null}, + ]) + ); } #[test] diff --git a/hyperdb-api-core/src/client/mod.rs b/hyperdb-api-core/src/client/mod.rs index 4bbf4322..565054ad 100644 --- a/hyperdb-api-core/src/client/mod.rs +++ b/hyperdb-api-core/src/client/mod.rs @@ -226,7 +226,10 @@ //! let mut client = GrpcClient::connect(config).await?; //! //! // Dollar-numbered parameters ($1, $2, ...) - use serde_json::json! for mixed types -//! let params = QueryParameters::from_json_value(&serde_json::json!([42, "Alice"]))?; +//! let params = QueryParameters::json_positional(&[ +//! &serde_json::json!(42), +//! &serde_json::json!("Alice"), +//! ])?; //! let result = client.execute_query_with_params( //! "SELECT * FROM users WHERE id = $1 AND name = $2", //! params, @@ -236,10 +239,10 @@ //! // Named parameters using builder pattern //! let params = QueryParameters::json_named() //! .add("id", &42i64)? -//! .add("name", &"Alice")? +//! .add("user_name", &"Alice")? //! .build(); //! let result = client.execute_query_with_params( -//! "SELECT * FROM users WHERE id = :id AND name = :name", +//! "SELECT * FROM users WHERE id = :id AND name = :user_name", //! params, //! ParameterStyle::Named, //! ).await?; diff --git a/hyperdb-api/tests/grpc_params_tests.rs b/hyperdb-api/tests/grpc_params_tests.rs new file mode 100644 index 00000000..e02108f4 --- /dev/null +++ b/hyperdb-api/tests/grpc_params_tests.rs @@ -0,0 +1,167 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Integration tests for JSON-encoded gRPC query parameters. +//! +//! hyperd only accepts JSON parameters as an array of typed entries +//! (`[{"type":"bigint","value":"42"}, ...]`), so these tests send the +//! output of [`QueryParameters::json_positional`] and +//! [`QueryParameters::json_named`] to a live `hyperd` and check the values +//! that come back. + +use arrow::ipc::reader::StreamReader; +use arrow::util::display::array_value_to_string; +use hyperdb_api::{HyperProcess, ListenMode, Parameters, Result}; +use hyperdb_api_core::client::grpc::{GrpcClientSync, GrpcConfig, ParameterStyle, QueryParameters}; +use serde_json::json; + +/// Spins up `hyperd` in gRPC-only mode and returns the process and its gRPC URL. +fn grpc_hyperd() -> Result<(HyperProcess, String)> { + let mut params = Parameters::new(); + params.set("log_dir", "test_results"); + params.set_listen_mode(ListenMode::Grpc { port: 0 }); + let hyper = HyperProcess::new(None, Some(¶ms))?; + let url = hyper + .grpc_url() + .expect("hyperd should expose a gRPC URL when listen_mode=Grpc"); + Ok((hyper, url)) +} + +/// Runs `sql` with `params` and returns the single result row, each column +/// rendered as a display string. +fn query_one_row( + client: &mut GrpcClientSync, + sql: &str, + params: QueryParameters, + style: ParameterStyle, +) -> Result> { + let result = client.execute_query_with_params(sql, params, style)?; + let reader = StreamReader::try_new(std::io::Cursor::new(result.arrow_data()), None) + .expect("result should be an Arrow IPC stream"); + let batches: Vec<_> = reader + .collect::>() + .expect("Arrow batches should decode"); + let batch = batches + .iter() + .find(|b| b.num_rows() > 0) + .expect("query should return a row"); + assert_eq!(batch.num_rows(), 1, "expected exactly one row"); + Ok(batch + .columns() + .iter() + .map(|col| array_value_to_string(col, 0).expect("value should format")) + .collect()) +} + +#[test] +fn test_json_positional_question_mark_mixed_types() -> Result<()> { + let (_hyper, url) = grpc_hyperd()?; + let mut client = GrpcClientSync::connect(GrpcConfig::new(&url))?; + + let params = + QueryParameters::json_positional(&[&json!(41), &json!("hi"), &json!(true), &json!(1.5)]) + .expect("parameters should serialize"); + let row = query_one_row( + &mut client, + "SELECT ? + 1 AS a, ? AS b, ? AS c, ? * 2 AS d", + params, + ParameterStyle::QuestionMark, + )?; + assert_eq!(row, ["42", "hi", "true", "3.0"]); + Ok(()) +} + +#[test] +fn test_json_positional_dollar_numbered() -> Result<()> { + let (_hyper, url) = grpc_hyperd()?; + let mut client = GrpcClientSync::connect(GrpcConfig::new(&url))?; + + let params = + QueryParameters::json_positional(&[&40i64, &2i64]).expect("parameters should serialize"); + let row = query_one_row( + &mut client, + "SELECT $1 + $2 AS s, $2 AS second", + params, + ParameterStyle::DollarNumbered, + )?; + assert_eq!(row, ["42", "2"]); + Ok(()) +} + +#[test] +fn test_json_positional_null() -> Result<()> { + let (_hyper, url) = grpc_hyperd()?; + let mut client = GrpcClientSync::connect(GrpcConfig::new(&url))?; + + let params = QueryParameters::json_positional(&[&json!(null), &json!(null)]) + .expect("parameters should serialize"); + let row = query_one_row( + &mut client, + "SELECT ? IS NULL AS n, CAST(? AS BIGINT) IS NULL AS typed", + params, + ParameterStyle::QuestionMark, + )?; + assert_eq!(row, ["true", "true"]); + Ok(()) +} + +#[test] +fn test_json_positional_u64_above_i64_max() -> Result<()> { + let (_hyper, url) = grpc_hyperd()?; + let mut client = GrpcClientSync::connect(GrpcConfig::new(&url))?; + + let params = + QueryParameters::json_positional(&[&u64::MAX]).expect("parameters should serialize"); + let row = query_one_row( + &mut client, + "SELECT CAST(? AS TEXT) AS v", + params, + ParameterStyle::QuestionMark, + )?; + assert_eq!(row, ["18446744073709551615"]); + Ok(()) +} + +#[test] +fn test_from_json_string_explicit_types() -> Result<()> { + let (_hyper, url) = grpc_hyperd()?; + let mut client = GrpcClientSync::connect(GrpcConfig::new(&url))?; + + let params = QueryParameters::from_json_string( + r#"[{"type": "date", "value": "2026-01-31"}, + {"type": "numeric", "precision": 10, "scale": 2, "value": "12.50"}]"#, + ); + let row = query_one_row( + &mut client, + "SELECT CAST($1 + 1 AS TEXT) AS d, CAST($2 AS TEXT) AS n", + params, + ParameterStyle::DollarNumbered, + )?; + assert_eq!(row, ["2026-02-01", "12.50"]); + Ok(()) +} + +#[test] +fn test_json_named() -> Result<()> { + let (_hyper, url) = grpc_hyperd()?; + let mut client = GrpcClientSync::connect(GrpcConfig::new(&url))?; + + let params = QueryParameters::json_named() + .add("id", &42i64) + .expect("id should serialize") + .add("user_name", &"Alice") + .expect("user_name should serialize") + .add_null("missing") + // `name` is a keyword, so the SQL has to quote it + .add("name", &"Bob") + .expect("name should serialize") + .build(); + let row = query_one_row( + &mut client, + r#"SELECT :id AS id, :user_name AS user_name, :missing IS NULL AS missing, :"name" AS n"#, + params, + ParameterStyle::Named, + )?; + assert_eq!(row, ["42", "Alice", "true", "Bob"]); + Ok(()) +} From ac3bd4bdfac6f0732d71656e1c933ca4dbc4188d Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 22:26:54 -0700 Subject: [PATCH 12/61] docs(api): document OR/IN-over-parameters engine limitation hyperd rejects a single-column OR/IN/ANY/CASE filter that mixes a bound parameter with other values, at prepare time (XX000 "too few registered parameters"). Document the failing shapes, the affected APIs and the `IN (SELECT unnest(ARRAY[$1, $2]))` rewrite on Connection::query_params, the async and command_params variants, query_as!, and both READMEs. --- hyperdb-api-derive/README.md | 1 + hyperdb-api-derive/src/lib.rs | 8 +++++ hyperdb-api/README.md | 24 +++++++++++++++ hyperdb-api/src/async_connection.rs | 11 +++++-- hyperdb-api/src/connection.rs | 46 ++++++++++++++++++++++++++++- 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/hyperdb-api-derive/README.md b/hyperdb-api-derive/README.md index 8a915683..47bfa58e 100644 --- a/hyperdb-api-derive/README.md +++ b/hyperdb-api-derive/README.md @@ -233,3 +233,4 @@ After RA finishes indexing you'll see squigglies on bad SQL strings and errors i - **No parameter type checking** — bind arguments are bound at run time (each must implement `ToSqlParam`), but their types are not checked against the `$N` placeholders at compile time. - **Validates struct vs. SQL, not SQL vs. production DB** — struct/prod schema drift is still a runtime error. - **`INSERT`/`UPDATE`/`DELETE` without `RETURNING`** are not supported by `query_as!`; use `Connection::execute_command` directly. +- **`OR` / `IN` lists over arguments are rejected by `hyperd`** — an engine defect rejects a single-column filter such as `WHERE id IN ($1, $2)` or `WHERE id = $1 OR id = $2` with SQLSTATE `XX000`. Write `WHERE id IN (SELECT unnest(ARRAY[$1, $2]))` instead. See the `Connection::query_params` docs in `hyperdb-api`. diff --git a/hyperdb-api-derive/src/lib.rs b/hyperdb-api-derive/src/lib.rs index f55d4d63..1a025cd4 100644 --- a/hyperdb-api-derive/src/lib.rs +++ b/hyperdb-api-derive/src/lib.rs @@ -123,6 +123,14 @@ pub fn table_derive(input: TokenStream) -> TokenStream { /// cache — a `query_as!` can be re-expanded in a process where no derive has /// run. Validation therefore skips when the registry is entirely empty, so the /// editor does not report errors on code that compiles. +/// +/// # Known limitation: `OR` / `IN` lists over arguments +/// +/// Arguments are bound through the `*_as_params` methods, which use +/// `Connection::query_params`, so the `hyperd` defect documented there applies to `query_as!` and `query_scalar!`: a +/// single-column filter such as `WHERE id IN ($1, $2)` or +/// `WHERE id = $1 OR id = $2` fails with SQLSTATE `XX000`. Write +/// `WHERE id IN (SELECT unnest(ARRAY[$1, $2]))` instead. #[proc_macro] pub fn query_as(input: TokenStream) -> TokenStream { match expand_query_as(&input.into()) { diff --git a/hyperdb-api/README.md b/hyperdb-api/README.md index 51698ce9..e69606b5 100644 --- a/hyperdb-api/README.md +++ b/hyperdb-api/README.md @@ -216,6 +216,30 @@ let users: Vec = conn.fetch_all_as_params( )?; ``` +#### Known limitation: `OR` / `IN` lists over parameters + +A `hyperd` defect rejects a filter on a single column that combines a bound +parameter with other values through `OR`, `IN`, `NOT IN` or `= ANY(ARRAY[...])`, +for example `WHERE id = $1 OR id = $2` or `WHERE id IN ($1, $2)`. The statement +fails when it is prepared, with SQLSTATE `XX000` ("A parameter was accessed in an +execution target with too few registered parameters"). This affects every API +that binds parameters, including `command_params` for `UPDATE` and `DELETE`. +Filters that combine parameters with `AND`, and `OR` across two different +columns, are not affected. + +Rewrite the list as a subquery over an array built from the parameters. The +values stay bound, so the query is still injection-safe: + +```rust +// Fails: "SELECT * FROM users WHERE id IN ($1, $2)" +let mut result = conn.query_params( + "SELECT * FROM users WHERE id IN (SELECT unnest(ARRAY[$1, $2]))", + &[&1i64, &2i64], +)?; +``` + +See the `Connection::query_params` API docs for details. + #### Compile-time SQL validation (opt-in) With the `hyperdb-api-derive` crate's `compile-time` feature, the `query_as!` diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index 21b86487..e7c67632 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -709,9 +709,11 @@ impl AsyncConnection { /// Executes a parameterized query with binary-encoded parameters (async). /// /// Mirrors the sync [`Connection::query_params`](crate::Connection::query_params); - /// see that method for the design rationale. Parameters travel through the - /// extended query protocol (Parse/Bind/Execute) in HyperBinary format — no - /// SQL escaping, full SQL-injection safety regardless of parameter content. + /// see that method for the design rationale and for its known limitation + /// with `OR` / `IN` lists over parameters, which applies here too. + /// Parameters travel through the extended query protocol + /// (Parse/Bind/Execute) in HyperBinary format — no SQL escaping, full + /// SQL-injection safety regardless of parameter content. /// /// # Errors /// @@ -753,6 +755,9 @@ impl AsyncConnection { /// Executes a parameterized command (INSERT / UPDATE / DELETE) with /// binary-encoded parameters via Parse/Bind/Execute (async). /// + /// Subject to the known limitation with `OR` / `IN` lists over parameters + /// described on [`Connection::query_params`](crate::Connection::query_params). + /// /// # Errors /// /// - Returns [`Error::FeatureNotSupported`] on gRPC transports. diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index c951c68c..516ce195 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -1210,6 +1210,45 @@ impl Connection { /// } /// ``` /// + /// # Known limitation: `OR` / `IN` lists over parameters + /// + /// A `hyperd` defect rejects a filter on a **single column** that + /// combines a bound parameter with other values through `OR`, `IN`, + /// `NOT IN`, `= ANY(ARRAY[...])` or `CASE`, when the filter applies to a + /// table scan. Examples are `WHERE id = $1 OR id = $2`, + /// `WHERE id IN ($1, $2)` and `WHERE id = $1 OR id = 3`. The statement + /// fails when it is prepared, before any value is bound, with SQLSTATE + /// `XX000`: "A parameter was accessed in an execution target with too few + /// registered parameters." The same filter in an `UPDATE` or `DELETE` fails + /// the same way. + /// + /// The defect affects every API that binds parameters: this method, + /// [`command_params`](Self::command_params), the `*_as_params` methods, + /// [`prepare_typed`](Self::prepare_typed), + /// `query_as!` / `query_scalar!` with arguments, and their async + /// counterparts. Filters that combine parameters with `AND` (including + /// ranges such as `id >= $1 AND id <= $2`) are not affected, nor is an + /// `OR` across two different columns. + /// + /// Rewrite the list as a subquery over an array built from the + /// parameters. Values stay bound, so the query remains injection-safe: + /// + /// ```no_run + /// # use hyperdb_api::{Connection, Result}; + /// # fn example(conn: &Connection) -> Result<()> { + /// // Fails: "WHERE id IN ($1, $2)" + /// let result = conn.query_params( + /// "SELECT * FROM users WHERE id IN (SELECT unnest(ARRAY[$1, $2]))", + /// &[&1i64, &2i64], + /// )?; + /// # Ok(()) + /// # } + /// ``` + /// + /// Joining against `VALUES ($1), ($2)` also works. Inlining the values as + /// SQL literals works too, but only do that for values that are not user + /// input. + /// /// # Errors /// /// - Returns [`Error::FeatureNotSupported`] if the connection is using gRPC transport @@ -1259,7 +1298,8 @@ impl Connection { /// Returns the number of affected rows. /// /// See [`query_params`](Self::query_params) for details on parameter - /// handling and SQL injection prevention. + /// handling and SQL injection prevention, and for its known limitation + /// with `OR` / `IN` lists over parameters, which applies here too. /// /// # Example /// @@ -1768,6 +1808,10 @@ impl Connection { /// from the value. [`Geography`](crate::Geography) is unaffected and /// works normally here. /// + /// The known limitation with `OR` / `IN` lists over parameters described + /// on [`query_params`](Self::query_params) applies here too: such a + /// statement fails at `Parse` time. + /// /// # Errors /// /// - Returns [`Error::FeatureNotSupported`] if the connection is using gRPC transport From 48ea4e3238e93d564800a5db08c9ed9b3f9ce4bf Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 22:36:47 -0700 Subject: [PATCH 13/61] refactor(api)!: stop leaking core, tonic and deadpool types from the public API hyperdb_api::grpc no longer re-exports GrpcClient, GrpcClientSync or GrpcError (raw core clients carrying tonic/prost types); it exports GrpcColumnInfo instead. The escape hatches that returned hyperdb-api-core types are crate-private or removed: Connection::tcp_client, AsyncConnection::async_tcp_client and from_async_client, HyperProcess::connection_endpoint, ColumnDefinition::to_types_column_definition and the From impl. Replacements: HyperProcess::connection_endpoint_string and AsyncConnection::copy_in_hyperbinary, which the Node inserter now uses instead of a hand-written COPY loop. The async pool wraps deadpool: Pool and PooledConnection are hyperdb-api types, ConnectionManager is crate-private, Pool::get returns hyperdb_api::Result (Timeout / Connection / underlying error) and Pool::status returns PoolStatus. BREAKING CHANGE: see hyperdb-api/CHANGELOG.md [Unreleased]. --- hyperdb-api-node/src/inserter.rs | 40 +------ hyperdb-api/CHANGELOG.md | 35 ++++++ hyperdb-api/README.md | 2 +- hyperdb-api/src/async_connection.rs | 53 ++++++++- hyperdb-api/src/connection.rs | 2 +- hyperdb-api/src/lib.rs | 8 +- hyperdb-api/src/pool.rs | 124 +++++++++++++++++--- hyperdb-api/src/process.rs | 19 ++- hyperdb-api/src/table_definition.rs | 21 +--- hyperdb-api/tests/async_connection_tests.rs | 33 ++++++ hyperdb-api/tests/pool_tests.rs | 72 +++++++++++- hyperdb-mcp/src/daemon/run.rs | 18 ++- 12 files changed, 325 insertions(+), 102 deletions(-) diff --git a/hyperdb-api-node/src/inserter.rs b/hyperdb-api-node/src/inserter.rs index e869d5d2..5832cfff 100644 --- a/hyperdb-api-node/src/inserter.rs +++ b/hyperdb-api-node/src/inserter.rs @@ -232,43 +232,9 @@ impl RowInserter { return Ok(0); } - // Start a COPY IN session and stream the pre-encoded buffer. - let client = conn.async_tcp_client().ok_or_else(|| { - Error::from_reason( - "Inserter requires a TCP connection. \ - gRPC connections do not support COPY operations.", - ) - })?; - - let columns: Vec = table_def.columns().iter().map(|c| c.name.clone()).collect(); - let column_refs: Vec<&str> = columns.iter().map(std::string::String::as_str).collect(); - let table_name = table_def.qualified_name(); - - let mut writer = client - .copy_in_arc_with_format(&table_name, &column_refs, "HYPERBINARY") - .await - .map_err(|e| Error::from_reason(e.to_string()))?; - - // hyperd caps COPY packets at ~150 MB; slice the pre-encoded - // buffer so large inserts don't trigger `packet with length - // > 157286400` on the server side. - const MAX_COPY_CHUNK: usize = 64 * 1024 * 1024; - let mut cursor = 0; - while cursor < encoded.len() { - let end = (cursor + MAX_COPY_CHUNK).min(encoded.len()); - writer - .send_direct(&encoded[cursor..end]) - .await - .map_err(|e| Error::from_reason(e.to_string()))?; - writer - .flush_stream() - .await - .map_err(|e| Error::from_reason(e.to_string()))?; - cursor = end; - } - - let count = writer - .finish() + // Stream the pre-encoded buffer as one COPY IN session. + let count = conn + .copy_in_hyperbinary(&table_def, &encoded) .await .map_err(|e| Error::from_reason(e.to_string()))?; i64::try_from(count).map_err(|_| Error::from_reason("row count exceeds i64::MAX")) diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 699e4ba1..dc84716f 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -13,6 +13,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/). instead of `&[&dyn Debug]`. These constructors are emitted by the `query_as!` / `query_scalar!` macros and are not meant to be called directly, but code that did call them must now pass `ToSqlParam` values. +- **BREAKING: `hyperdb_api::grpc` no longer re-exports `GrpcClient`, + `GrpcClientSync` or `GrpcError`.** They are the raw `hyperdb-api-core` + clients and carry `tonic` / `prost` types, which the core crate declares + unstable. Use `GrpcConnection` / `GrpcConnectionAsync` (or the unified + `Connection`); failures surface as `hyperdb_api::Error`. The module now also + exports `GrpcColumnInfo`, which `GrpcQueryResult::columns` returns. +- **BREAKING: escape hatches that returned `hyperdb-api-core` types are no + longer public.** `Connection::tcp_client`, `AsyncConnection::async_tcp_client`, + `AsyncConnection::from_async_client`, `HyperProcess::connection_endpoint`, + `ColumnDefinition::to_types_column_definition` and + `impl From for ColumnDefinition` + are removed from the public API. Replacements: + `HyperProcess::connection_endpoint_string()` (the endpoint + `Connection::new` connects to, as a `String`) and + `AsyncConnection::copy_in_hyperbinary` (below), which replaces hand-written + COPY loops over `async_tcp_client`. +- **BREAKING: the async pool no longer exposes `deadpool` types.** + `pool::Pool` and `pool::PooledConnection` were aliases of + `deadpool::managed::Pool` / `Object`; they are now `hyperdb-api` types, and + `pool::ConnectionManager` is crate-private. `Pool::get` returns + `hyperdb_api::Result` — a wait/create/recycle timeout is + `Error::Timeout`, a closed pool is `Error::Connection`, and a failed connect + is the underlying error — so the `.map_err(|e| Error::internal(e.to_string()))` + adapter is no longer needed. `Pool::status` returns `PoolStatus` (as + `ConnectionPool` does), and `PooledConnection::take` frees the slot. The + `deadpool` version is no longer part of this crate's semver surface. + +### Added + +- **`AsyncConnection::copy_in_hyperbinary`** streams a pre-encoded + `HyperBinary` buffer (for example from `InsertChunk`) into a table with + `COPY ... FROM STDIN`, slicing it under `hyperd`'s COPY packet cap, and + returns the inserted row count. +- **`HyperProcess::connection_endpoint_string`**, and `Pool::close` / + `Pool::is_closed` on the async pool. ### Fixed diff --git a/hyperdb-api/README.md b/hyperdb-api/README.md index e69606b5..84835d4e 100644 --- a/hyperdb-api/README.md +++ b/hyperdb-api/README.md @@ -114,7 +114,7 @@ async fn main() -> Result<()> { let pool = create_pool(config)?; // Get connections from the pool — returned automatically when dropped - let conn = pool.get().await.map_err(|e| hyperdb_api::Error::internal(e.to_string()))?; + let conn = pool.get().await?; conn.execute_command("SELECT 1").await?; Ok(()) diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index e7c67632..c5ad879f 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -16,6 +16,7 @@ use crate::error::{Error, Result}; use crate::names::escape_sql_path; use crate::query_stats::{QueryStats, QueryStatsProvider}; use crate::result::{Row, RowValue}; +use crate::table_definition::TableDefinition; /// An async connection to a Hyper database. /// @@ -143,7 +144,7 @@ impl AsyncConnection { /// Builds an `AsyncConnection` from a pre-existing `AsyncClient` (TCP only). #[must_use] - pub fn from_async_client( + pub(crate) fn from_async_client( client: hyperdb_api_core::client::AsyncClient, database: Option, ) -> Self { @@ -1116,12 +1117,52 @@ impl AsyncConnection { Ok(()) } - /// Returns a reference to the underlying async TCP client (`None` for gRPC). + /// Streams a pre-encoded `HyperBinary` buffer into `table` with `COPY ... FROM STDIN`. + /// + /// `data` must hold rows in the `HyperBinary` format for the table's columns, + /// in definition order — for example the output of + /// [`InsertChunk`](crate::InsertChunk). The buffer is sent in 64 MiB slices + /// because `hyperd` rejects COPY packets above ~150 MB. Returns the number of + /// rows the server reports as inserted; an empty buffer inserts nothing and + /// returns `0`. /// - /// Prefer the high-level `AsyncConnection` methods; this escape hatch - /// remains for code that needs direct protocol access (e.g. custom - /// COPY loops). - pub fn async_tcp_client(&self) -> Option<&hyperdb_api_core::client::AsyncClient> { + /// Most callers want [`AsyncInserter`](crate::AsyncInserter), which encodes + /// rows for you. Use this when the bytes are already encoded. + /// + /// # Errors + /// + /// - Returns [`Error::feature_not_supported`] on a gRPC connection, which + /// cannot run COPY. + /// - Returns an error if the server rejects the COPY or the connection + /// fails while sending. + pub async fn copy_in_hyperbinary(&self, table: &TableDefinition, data: &[u8]) -> Result { + // hyperd caps COPY packets at ~150 MB; slice well under that. + const MAX_COPY_CHUNK: usize = 64 * 1024 * 1024; + + if data.is_empty() { + return Ok(0); + } + let client = self.async_tcp_client().ok_or_else(|| { + Error::feature_not_supported( + "copy_in_hyperbinary requires a TCP connection. \ + gRPC connections do not support COPY operations.", + ) + })?; + let columns: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect(); + let table_name = table.qualified_name(); + + let mut writer = client + .copy_in_arc_with_format(&table_name, &columns, "HYPERBINARY") + .await?; + for slice in data.chunks(MAX_COPY_CHUNK) { + writer.send_direct(slice).await?; + writer.flush_stream().await?; + } + Ok(writer.finish().await?) + } + + /// Returns a reference to the underlying async TCP client (`None` for gRPC). + pub(crate) fn async_tcp_client(&self) -> Option<&hyperdb_api_core::client::AsyncClient> { self.transport.async_tcp_client() } diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index 516ce195..1bd4c30a 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -1739,7 +1739,7 @@ impl Connection { /// Returns the underlying wire-protocol client, or `None` for gRPC connections. /// /// TCP, Unix-socket and named-pipe connections all return `Some`. - pub fn tcp_client(&self) -> Option<&Client> { + pub(crate) fn tcp_client(&self) -> Option<&Client> { match &self.transport { Transport::Tcp(tcp) => Some(&tcp.client), Transport::Grpc(_) => None, diff --git a/hyperdb-api/src/lib.rs b/hyperdb-api/src/lib.rs index a8a23433..6924d1f0 100644 --- a/hyperdb-api/src/lib.rs +++ b/hyperdb-api/src/lib.rs @@ -278,10 +278,12 @@ pub mod grpc { // Re-export connection types from grpc_connection module pub use crate::grpc_connection::{GrpcConnection, GrpcConnectionAsync}; - // Re-export types from hyperdb_api_core::client::grpc + // Configuration and result types. The raw core clients (`GrpcClient`, + // `GrpcClientSync`) and `GrpcError` are deliberately not re-exported: they + // would put tonic and prost types into this crate's public API. Errors + // surface as `hyperdb_api::Error`. pub use hyperdb_api_core::client::grpc::{ - GrpcClient, GrpcClientSync, GrpcConfig, GrpcError, GrpcQueryResult, GrpcResultChunk, - TransferMode, + GrpcColumnInfo, GrpcConfig, GrpcQueryResult, GrpcResultChunk, TransferMode, }; } diff --git a/hyperdb-api/src/pool.rs b/hyperdb-api/src/pool.rs index 850514dd..40e964a6 100644 --- a/hyperdb-api/src/pool.rs +++ b/hyperdb-api/src/pool.rs @@ -5,8 +5,8 @@ //! //! This module provides two pools that share a common configuration surface: //! -//! - [`Pool`] — an async pool built on [`deadpool`], for `async`/`await` -//! applications. Created via [`create_pool`]. +//! - [`Pool`] — an async pool for `async`/`await` applications. Created via +//! [`create_pool`]. //! - [`ConnectionPool`] — a synchronous, r2d2-style pool with **no Tokio //! dependency** on its hot path, for blocking applications. Created via //! [`SyncPoolConfig::build`]. @@ -33,7 +33,7 @@ //! let pool = create_pool(config)?; //! //! // Get a connection from the pool -//! let conn = pool.get().await.map_err(|e| hyperdb_api::Error::internal(e.to_string()))?; +//! let conn = pool.get().await?; //! //! // Use the connection //! conn.execute_command("SELECT 1").await?; @@ -80,8 +80,8 @@ //! - **`idle_timeout`** retires connections that have sat idle too long (down to //! `min_idle`, which is kept warm). //! - **Timeouts** (`wait_timeout`, `create_timeout`, `recycle_timeout`) bound how -//! long an acquire may block. The async pool enforces all three via deadpool's -//! Tokio runtime; the sync pool enforces `wait_timeout` natively (see +//! long an acquire may block. The async pool enforces all three on the Tokio +//! runtime; the sync pool enforces `wait_timeout` natively (see //! [`SyncPoolConfig`] for the create/recycle caveat). //! //! # Lifecycle hooks (async pool only) @@ -461,6 +461,9 @@ impl PoolConfig { /// Connection pool manager for `AsyncConnection`. /// +/// Crate-private: the pool's public face is [`Pool`] and [`PooledConnection`], +/// so the `deadpool` types behind them are not part of this crate's API. +/// /// The first call to [`Manager::create`] holds an async mutex while attempting /// to open a connection with the configured [`CreateMode`]. Concurrent callers /// wait for that attempt to finish, then use `CreateMode::DoNotCreate`. If the @@ -468,7 +471,7 @@ impl PoolConfig { /// (for idempotent modes only — `Create` is not retried because a sibling /// connection may have already created the database). #[derive(Debug)] -pub struct ConnectionManager { +pub(crate) struct ConnectionManager { config: Arc, /// Synchronizes the first-connection attempt across concurrent callers. /// `Some(())` after the first successful attempt; held while a first @@ -479,8 +482,7 @@ pub struct ConnectionManager { impl ConnectionManager { /// Creates a new connection manager. - #[must_use] - pub fn new(config: PoolConfig) -> Self { + fn new(config: PoolConfig) -> Self { Self { config: Arc::new(config), init_lock: Arc::new(AsyncMutex::new(false)), @@ -597,21 +599,104 @@ impl Manager for ConnectionManager { /// A pool of async connections to a Hyper database. /// -/// This pool manages a set of reusable connections, automatically creating -/// new connections when needed and recycling them after use. -pub type Pool = managed::Pool; +/// Created by [`create_pool`]. Connections are opened lazily, reused across +/// checkouts and recycled according to the configured [`RecycleStrategy`]. +/// `Pool` is a cheap handle: clone it to share one pool between tasks. +#[derive(Clone, Debug)] +pub struct Pool { + inner: managed::Pool, +} + +impl Pool { + /// Checks a connection out of the pool, opening one if none is idle and + /// the pool is below its maximum size. + /// + /// The connection returns to the pool when the guard is dropped. + /// + /// # Errors + /// + /// - Returns [`Error::Timeout`] if `wait_timeout`, `create_timeout` or + /// `recycle_timeout` elapses. + /// - Returns [`Error::Connection`] if the pool has been [closed](Self::close). + /// - Returns the underlying error if opening a connection fails or an + /// `after_connect` hook rejects it. + pub async fn get(&self) -> Result { + match self.inner.get().await { + Ok(inner) => Ok(PooledConnection { inner }), + Err(managed::PoolError::Backend(e)) => Err(e), + Err(e @ managed::PoolError::Timeout(_)) => Err(Error::timeout(e.to_string())), + Err(managed::PoolError::Closed) => { + Err(Error::connection("the connection pool is closed")) + } + Err(e) => Err(Error::internal(e.to_string())), + } + } -/// A pooled connection wrapper. -pub type PooledConnection = managed::Object; + /// Returns a snapshot of the pool's occupancy. + #[must_use] + pub fn status(&self) -> PoolStatus { + let status = self.inner.status(); + PoolStatus { + idle: status.available, + size: status.size, + max_size: status.max_size, + } + } + + /// Closes the pool: idle connections are dropped and further + /// [`get`](Self::get) calls fail. Connections already checked out are + /// dropped, not returned, when their guards go out of scope. + pub fn close(&self) { + self.inner.close(); + } + + /// Returns `true` once [`close`](Self::close) has been called. + #[must_use] + pub fn is_closed(&self) -> bool { + self.inner.is_closed() + } +} + +/// A connection checked out of an async [`Pool`]. +/// +/// Derefs to [`AsyncConnection`]. Returns to the pool when dropped; the pool +/// recycles it before the next checkout. +#[derive(Debug)] +pub struct PooledConnection { + inner: managed::Object, +} + +impl PooledConnection { + /// Removes the connection from the pool's management, taking ownership. + /// + /// The pool slot is freed; the returned connection will not be recycled. + #[must_use] + pub fn take(self) -> AsyncConnection { + managed::Object::take(self.inner) + } +} + +impl std::ops::Deref for PooledConnection { + type Target = AsyncConnection; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for PooledConnection { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} /// Creates a new connection pool from configuration. /// /// # Errors /// -/// Returns [`Error::Config`] wrapping the `deadpool` builder failure if -/// the pool cannot be constructed (e.g. invalid `max_size`). Connections -/// themselves are opened lazily on first use, so endpoint/auth errors -/// surface from [`Pool::get`](managed::Pool::get), not here. +/// Returns [`Error::Config`] if the pool cannot be constructed (e.g. invalid +/// `max_size`). Connections themselves are opened lazily on first use, so +/// endpoint/auth errors surface from [`Pool::get`], not here. pub fn create_pool(config: PoolConfig) -> Result { let max_size = config.max_size; let timeouts = Timeouts { @@ -623,12 +708,15 @@ pub fn create_pool(config: PoolConfig) -> Result { // a timeout is actually configured so the zero-config path stays untouched. let needs_runtime = config.has_timeout(); let manager = ConnectionManager::new(config); - let mut builder = Pool::builder(manager).max_size(max_size).timeouts(timeouts); + let mut builder = managed::Pool::builder(manager) + .max_size(max_size) + .timeouts(timeouts); if needs_runtime { builder = builder.runtime(Runtime::Tokio1); } builder .build() + .map(|inner| Pool { inner }) .map_err(|e| Error::config(format!("Failed to create pool: {e}"))) } diff --git a/hyperdb-api/src/process.rs b/hyperdb-api/src/process.rs index e1e70365..e42dc42f 100644 --- a/hyperdb-api/src/process.rs +++ b/hyperdb-api/src/process.rs @@ -987,13 +987,24 @@ impl HyperProcess { /// Returns the connection endpoint for this process. /// - /// This returns a [`ConnectionEndpoint`] that can be used to connect - /// to this Hyper instance via TCP, Unix Domain Socket, or Named Pipe. - #[must_use] - pub fn connection_endpoint(&self) -> Option<&ConnectionEndpoint> { + /// This is the endpoint [`Connection::new`](crate::Connection::new) connects + /// to: TCP, Unix Domain Socket, or Named Pipe. + pub(crate) fn connection_endpoint(&self) -> Option<&ConnectionEndpoint> { self.connection_endpoint.as_ref() } + /// Returns the endpoint to connect to this process, rendered as a string. + /// + /// Unlike [`endpoint`](Self::endpoint), which is always the TCP + /// `host:port`, this is the endpoint [`Connection::new`](crate::Connection::new) + /// actually uses: a Unix socket path or named pipe when the process was + /// started with an IPC listen mode. Returns `None` if the process has no + /// connectable endpoint yet. + #[must_use] + pub fn connection_endpoint_string(&self) -> Option { + self.connection_endpoint.as_ref().map(ToString::to_string) + } + /// Returns the log directory where hyperd writes its log files. /// /// The log file is typically `hyperd.log` within this directory. diff --git a/hyperdb-api/src/table_definition.rs b/hyperdb-api/src/table_definition.rs index ad4a5514..51c1fc55 100644 --- a/hyperdb-api/src/table_definition.rs +++ b/hyperdb-api/src/table_definition.rs @@ -6,7 +6,7 @@ use std::borrow::Cow; use crate::error::{Error, Result}; -use hyperdb_api_core::types::{ColumnDefinition as TypesColumnDefinition, Nullability, SqlType}; +use hyperdb_api_core::types::{Nullability, SqlType}; /// Possible persistence levels for database objects. /// @@ -289,25 +289,6 @@ impl ColumnDefinition { pub fn set_sql_type(&mut self, sql_type: SqlType) { self.sql_type_or_name = SqlTypeOrName::SqlType(sql_type); } - - /// Converts to the hyper-types `ColumnDefinition` (if `SqlType` is set). - #[must_use] - pub fn to_types_column_definition(&self) -> Option { - self.sql_type() - .map(|sql_type| TypesColumnDefinition::new(&self.name, sql_type, self.nullability())) - } -} - -impl From for ColumnDefinition { - fn from(col: TypesColumnDefinition) -> Self { - ColumnDefinition { - name: col.name.clone(), - sql_type_or_name: SqlTypeOrName::SqlType(col.sql_type), - nullable: col.nullability.is_nullable(), - collation: None, - default_expr: None, - } - } } /// A table definition. diff --git a/hyperdb-api/tests/async_connection_tests.rs b/hyperdb-api/tests/async_connection_tests.rs index 95d34a23..68f37d93 100644 --- a/hyperdb-api/tests/async_connection_tests.rs +++ b/hyperdb-api/tests/async_connection_tests.rs @@ -505,3 +505,36 @@ async fn stream_as_params_async() { conn.close().await.unwrap(); } + +/// `copy_in_hyperbinary` streams a pre-encoded `HyperBinary` buffer into a +/// table and reports the inserted row count; an empty buffer is a no-op. +#[tokio::test(flavor = "current_thread")] +async fn copy_in_hyperbinary_inserts_encoded_rows() { + use hyperdb_api::{InsertChunk, SqlType, TableDefinition}; + + let (_hyper, conn) = fresh_async_conn("async_copy_in_hyperbinary").await.unwrap(); + let table_def = TableDefinition::from("copied") + .add_required_column("id", SqlType::int()) + .add_required_column("name", SqlType::text()); + conn.execute_command("CREATE TABLE copied (id INT NOT NULL, name TEXT NOT NULL)") + .await + .unwrap(); + + let mut chunk = InsertChunk::from_table_definition(&table_def); + for (id, name) in [(1, "a"), (2, "b"), (3, "c")] { + chunk.add_i32(id).unwrap(); + chunk.add_str(name).unwrap(); + chunk.end_row().unwrap(); + } + let buffer = chunk.take().expect("rows were buffered"); + + assert_eq!(conn.copy_in_hyperbinary(&table_def, &[]).await.unwrap(), 0); + let inserted = conn.copy_in_hyperbinary(&table_def, &buffer).await.unwrap(); + assert_eq!(inserted, 3); + + let count: i64 = conn + .fetch_scalar("SELECT COUNT(*) FROM copied") + .await + .unwrap(); + assert_eq!(count, 3); +} diff --git a/hyperdb-api/tests/pool_tests.rs b/hyperdb-api/tests/pool_tests.rs index 152f71ee..0e3991a4 100644 --- a/hyperdb-api/tests/pool_tests.rs +++ b/hyperdb-api/tests/pool_tests.rs @@ -100,7 +100,11 @@ async fn async_pool_wait_timeout_fires() { let result = pool.get().await; let elapsed = start.elapsed(); - assert!(result.is_err(), "second acquire should time out"); + assert!( + matches!(result, Err(hyperdb_api::Error::Timeout(_))), + "second acquire should time out with Error::Timeout, got {:?}", + result.as_ref().err() + ); assert!( elapsed < Duration::from_secs(5), "should fail fast via wait_timeout, took {elapsed:?}" @@ -108,6 +112,72 @@ async fn async_pool_wait_timeout_fires() { drop(held); } +/// `Pool::status` reports occupancy through `PoolStatus`, mirroring the sync +/// pool, and `PooledConnection::take` frees the slot. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_pool_status_and_take() { + let (_hyper, endpoint) = fresh_server("pool_async_status").unwrap(); + let config = PoolConfig::new(&endpoint, db_path("pool_async_status")) + .create_mode(CreateMode::CreateAndReplace) + .max_size(3); + let pool = create_pool(config).unwrap(); + + let status = pool.status(); + assert_eq!((status.size, status.idle, status.max_size), (0, 0, 3)); + + let conn = pool.get().await.expect("get"); + let status = pool.status(); + assert_eq!((status.size, status.idle), (1, 0), "one checked out"); + + drop(conn); + let status = pool.status(); + assert_eq!((status.size, status.idle), (1, 1), "returned to the pool"); + + let owned = pool.get().await.expect("get again").take(); + owned.execute_command("SELECT 1").await.unwrap(); + assert_eq!(pool.status().size, 0, "take() should free the pool slot"); +} + +/// A closed pool refuses checkouts with a `hyperdb_api::Error`, not a +/// deadpool error type. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_pool_get_after_close_is_connection_error() { + let (_hyper, endpoint) = fresh_server("pool_async_closed").unwrap(); + let config = PoolConfig::new(&endpoint, db_path("pool_async_closed")) + .create_mode(CreateMode::CreateAndReplace) + .max_size(2); + let pool = create_pool(config).unwrap(); + assert!(!pool.is_closed()); + + pool.close(); + assert!(pool.is_closed()); + let err = pool + .get() + .await + .expect_err("closed pool must refuse checkout"); + assert!( + matches!(err, hyperdb_api::Error::Connection { .. }), + "expected Error::Connection, got {err:?}" + ); +} + +/// A failure to open a connection surfaces as the underlying +/// `hyperdb_api::Error`, not a wrapped pool error. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_pool_get_surfaces_connect_error() { + // Nothing listens on port 1; the first checkout has to open a connection. + let config = PoolConfig::new("127.0.0.1:1", db_path("pool_async_connect_err")) + .create_mode(CreateMode::CreateAndReplace) + .max_size(1); + let pool = create_pool(config).unwrap(); + + let err = pool.get().await.expect_err("nothing is listening"); + assert!( + matches!(err, hyperdb_api::Error::Connection { .. }), + "expected Error::Connection, got {err:?}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn async_pool_recycle_ping_strategy_works() { let (_hyper, endpoint) = fresh_server("pool_async_ping").unwrap(); diff --git a/hyperdb-mcp/src/daemon/run.rs b/hyperdb-mcp/src/daemon/run.rs index 5db9e55f..6bd98ec2 100644 --- a/hyperdb-mcp/src/daemon/run.rs +++ b/hyperdb-mcp/src/daemon/run.rs @@ -102,12 +102,11 @@ pub async fn run_daemon(config: DaemonConfig) -> Result<(), Box/domain/hyper` (a `tab.domain://` scheme artifact), whereas hyperd - // actually binds `/hyper`; `connection_endpoint()` carries the path a + // actually binds `/hyper`; `connection_endpoint_string()` carries the path a // client can connect to. For TCP and Windows named pipes the two agree. let endpoint = hyper - .connection_endpoint() - .ok_or("hyperd did not report a connection endpoint")? - .to_string(); + .connection_endpoint_string() + .ok_or("hyperd did not report a connection endpoint")?; info!(endpoint = %endpoint, "hyperd started"); // Step 3: Build DaemonInfo and write discovery file @@ -388,15 +387,12 @@ fn try_restart_hyperd( let params = build_params(&state_dir).map_err(|e| RestartError::SpawnFailed(e.to_string()))?; let new_hyper = HyperProcess::new(None, Some(¶ms)) .map_err(|e| RestartError::SpawnFailed(e.to_string()))?; - // See `run_daemon`: publish the connectable `connection_endpoint()`, not + // See `run_daemon`: publish the connectable `connection_endpoint_string()`, not // the raw callback descriptor, so a Unix socket path is the one hyperd // actually bound. - let new_endpoint = new_hyper - .connection_endpoint() - .ok_or_else(|| { - RestartError::SpawnFailed("hyperd did not report a connection endpoint".into()) - })? - .to_string(); + let new_endpoint = new_hyper.connection_endpoint_string().ok_or_else(|| { + RestartError::SpawnFailed("hyperd did not report a connection endpoint".into()) + })?; // Publish the new endpoint, discovery file first and STATUS second. // From 96ead748dffebfffccc0553600327105aa40402f Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 23:22:46 -0700 Subject: [PATCH 14/61] fix(api,core)!: address review of the public-API leak cleanup Follow-up to 48ea4e3. - grpc::TransferMode is now an owned enum (Sync, Async, Adaptive) rather than the prost type; GrpcColumnInfo fields are no longer public. - core: a cancelled COPY write no longer wedges the async connection. A write-in-flight flag marks the connection unhealthy after an interrupted write, no CopyFail is queued behind a partial frame, finish/cancel_copy refuse to run on it, and the post-cancel drain is bounded. - copy_in_hyperbinary requires exactly one HyperBinary header (InvalidOperation otherwise), uses the borrowed writer, and documents that it is not cancel-safe. - pool: a closed pool is Error::InvalidOperation; docs state that recycle failures are discarded and replaced; PoolStatus docs corrected. - HyperProcess absolutizes a custom domain socket directory so connection_endpoint_string is always routable. - Drop remaining deadpool/core mentions from user-facing docs and examples. Tests added: core cancel-safety unit tests, copy_in_hyperbinary header, server-error and gRPC cases, pool create-timeout, relative socket dir. --- README.md | 4 +- docs/RUST_GUIDELINES.md | 10 +- hyperdb-api-core/CHANGELOG.md | 16 ++ hyperdb-api-core/docs/IPC_IMPLEMENTATION.md | 2 +- .../src/client/async_connection.rs | 157 +++++++++++++++--- .../src/client/grpc/authenticated_client.rs | 2 +- hyperdb-api-core/src/client/grpc/client.rs | 4 +- hyperdb-api-core/src/client/grpc/config.rs | 36 +++- hyperdb-api-core/src/client/grpc/executor.rs | 4 +- hyperdb-api-core/src/client/grpc/mod.rs | 2 +- hyperdb-api-core/src/client/grpc/result.rs | 7 +- hyperdb-api/CHANGELOG.md | 34 +++- .../additional_examples/async_usage.rs | 2 +- .../additional_examples/connection_pool.rs | 15 +- hyperdb-api/examples/async_parity_smoke.rs | 2 +- hyperdb-api/src/async_connection.rs | 39 ++++- hyperdb-api/src/async_connection_builder.rs | 4 +- hyperdb-api/src/connection_builder.rs | 4 +- hyperdb-api/src/lib.rs | 2 +- hyperdb-api/src/pool.rs | 25 ++- hyperdb-api/src/process.rs | 8 +- hyperdb-api/tests/async_connection_tests.rs | 90 ++++++++++ hyperdb-api/tests/pool_tests.rs | 35 +++- hyperdb-api/tests/socket_dir_cleanup_tests.rs | 38 +++++ 24 files changed, 454 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 97b96a48..2c87e8cc 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ executable, obtained with `make download-hyperd`. See - **Dual Architecture** — sync (`Connection`) and async (`AsyncConnection`) APIs - **Typed Row Mapping** — `#[derive(FromRow)]` structs, including streaming `stream_as` for constant-memory typed queries - **Compile-time SQL Validation** — opt-in `query_as!` macro checks SQL against your schema at build time (red squigglies in VS Code) -- **Connection Pooling** — async pooling via `deadpool` for high-concurrency applications +- **Connection Pooling** — async connection pooling for high-concurrency applications - **Key-Value Store** — string-native `KvStore` / `AsyncKvStore` backed by a single fixed table - **Arrow Integration** — insert and read data in Arrow IPC stream format - **gRPC Transport** — read-only access with Arrow IPC and load balancing support @@ -261,7 +261,7 @@ The API ships 14 examples in `hyperdb-api/examples/` plus 2 companion crate exam | `async_usage` | `AsyncConnection` and Tokio patterns | | `threaded_inserter` | Multi-threaded bulk insertion with `InsertChunk`/`ChunkSender` | | `grpc_query` | gRPC transport, Arrow IPC results | -| `connection_pool` | Async connection pooling with deadpool | +| `connection_pool` | Async connection pooling | | `transactions` | RAII guards, multi-table rollback, DDL, reconnect semantics | ### Running Examples diff --git a/docs/RUST_GUIDELINES.md b/docs/RUST_GUIDELINES.md index 0d3f96b3..11660cdb 100644 --- a/docs/RUST_GUIDELINES.md +++ b/docs/RUST_GUIDELINES.md @@ -72,11 +72,11 @@ review; point to them when requesting changes. ### API design - **M-WEASEL-WORDS.** Avoid weasel words (`Service`, `Manager`, `Factory`, - `Helper`). Prefer names that describe what the type *is* or *does*. One - legitimate exception in this repo: `ConnectionManager` in - [hyperdb-api/src/pool.rs](../hyperdb-api/src/pool.rs), which matches - `deadpool::Manager` trait nomenclature. Upstream also rules out accepting - builders as parameters: where repeatable instantiation is needed, take + `Helper`). Prefer names that describe what the type *is* or *does*. The + crate-private `ConnectionManager` in + [hyperdb-api/src/pool.rs](../hyperdb-api/src/pool.rs) is the one place the + word appears; it implements `deadpool::Manager` and is not public API. + Upstream also rules out accepting builders as parameters: where repeatable instantiation is needed, take `impl Fn() -> Foo` rather than a `FooBuilder`. - **M-SHORT-NAMES.** At most two short words per identifier (`AppConfig`, not `GlobalApplicationConfig`); no crate or module prefix baked into the name diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index ed1ef272..a202ae43 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -13,8 +13,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **`client::grpc::TransferMode` is now a crate-owned enum** (`Sync`, `Async`, + `Adaptive`) with `From` conversions to the generated protobuf type, replacing + the re-export of the prost type. `GrpcColumnInfo`'s `name` and `sql_type` + fields are now `pub(crate)`; use the accessors. + ### Fixed +- **A cancelled COPY write could wedge the async connection.** Dropping a COPY + write future mid-frame left a partial `CopyData` frame on the wire, so the + queued `CopyFail` was read as part of that frame and + `drain_pending_copy_cancel` waited forever for `ReadyForQuery`. + `AsyncRawConnection` now tracks an in-flight write: after an interrupted + write the connection reports unhealthy, no `CopyFail` is queued, and + `finish_copy` / `cancel_copy` refuse to run on it. The post-cancel drain is + also bounded and marks the connection desynchronized instead of looping. + - **`Interval` `Display` kept the sign only when the time component was an hour or more.** A negative interval under one hour (for example −30 minutes) rendered as a positive `00:30:00`; it now renders `-00:30:00`. diff --git a/hyperdb-api-core/docs/IPC_IMPLEMENTATION.md b/hyperdb-api-core/docs/IPC_IMPLEMENTATION.md index da050d01..fe76adc1 100644 --- a/hyperdb-api-core/docs/IPC_IMPLEMENTATION.md +++ b/hyperdb-api-core/docs/IPC_IMPLEMENTATION.md @@ -138,7 +138,7 @@ In `hyperdb-api/src/process.rs`: When connecting via `Connection::new(&HyperProcess, ...)`: -- Uses `connection_endpoint()` if available — UDS on Unix, Named Pipe on Windows +- Uses the process's connection endpoint if available — UDS on Unix, Named Pipe on Windows - Falls back to string endpoint parsing (TCP) --- diff --git a/hyperdb-api-core/src/client/async_connection.rs b/hyperdb-api-core/src/client/async_connection.rs index 319ad559..bd9c42a8 100644 --- a/hyperdb-api-core/src/client/async_connection.rs +++ b/hyperdb-api-core/src/client/async_connection.rs @@ -52,6 +52,14 @@ pub struct AsyncRawConnection { /// [`Self::is_healthy`] and [`Self::ensure_healthy`] for the /// consumer-facing API. desynchronized: bool, + /// `true` while a write to the stream is in progress. It is set before + /// the first `await` of a write and cleared only when the write + /// completes, so it is still `true` if the future was dropped mid-write + /// (cancellation) or the write failed. In that state a partial frame may + /// be on the wire and nothing appended after it can be parsed as a + /// message boundary, so the connection is unusable. See + /// [`Self::is_healthy`]. + write_in_flight: bool, } impl AsyncRawConnection @@ -73,6 +81,7 @@ where server_params: HashMap::new(), pending_copy_cancel: false, desynchronized: false, + write_in_flight: false, } } @@ -81,7 +90,7 @@ where /// [`super::connection::RawConnection::is_healthy`] for the full /// semantics — this is the async mirror with identical behavior. pub fn is_healthy(&self) -> bool { - !self.desynchronized + !self.desynchronized && !self.write_in_flight } /// Marks this connection as desynchronized. @@ -101,7 +110,7 @@ where /// server request to short-circuit operations on a desynchronized /// connection before any bytes hit the wire. pub(crate) fn ensure_healthy(&self) -> Result<()> { - if self.desynchronized { + if !self.is_healthy() { return Err(Error::connection( "connection is desynchronized from the server and cannot be reused; \ discard it and open a new one", @@ -144,7 +153,18 @@ where /// but NOT flushed (we can't do async I/O from `Drop`). The next async /// operation will call [`drain_pending_copy_cancel`](Self::drain_pending_copy_cancel) to flush and drain /// the server's `ErrorResponse` + `ReadyForQuery` before proceeding. + /// + /// If a write was interrupted (the COPY future was dropped or failed + /// mid-`write_all`), a partial frame is on the wire and the server would + /// read the `CopyFail` bytes as the rest of that frame, so it never sees + /// the cancel. The `CopyFail` is not queued then; the connection is marked + /// desynchronized instead, so the next operation fails fast rather than + /// waiting for a `ReadyForQuery` that cannot arrive. pub fn queue_copy_fail(&mut self, reason: &str) { + if self.write_in_flight { + self.desynchronized = true; + return; + } frontend::copy_fail(reason, &mut self.write_buf); self.pending_copy_cancel = true; } @@ -169,25 +189,24 @@ where // Flush the queued CopyFail message self.flush().await?; - // Drain messages until the connection is back in ReadyForQuery state - loop { - let msg = self.read_message().await?; - match msg { - Message::ReadyForQuery(_) => { - self.pending_copy_cancel = false; - debug!( - target: "hyperdb_api_core::client", - "drained pending COPY cancel — connection restored" - ); - return Ok(()); - } - Message::ErrorResponse(_) => { - // Expected — server confirms the cancel - } - _ => { - // Ignore other messages (e.g., NoticeResponse) - } - } + // Drain messages (the expected ErrorResponse, notices) until the + // connection is back in ReadyForQuery state. Bounded: a server that + // never answers marks the connection desynchronized instead of + // hanging every later operation. + if self + .drain_until_ready_bounded(super::connection::POST_ERROR_DRAIN_CAP) + .await + { + self.pending_copy_cancel = false; + debug!( + target: "hyperdb_api_core::client", + "drained pending COPY cancel — connection restored" + ); + Ok(()) + } else { + Err(Error::connection( + "could not drain the abandoned COPY; connection marked desynchronized", + )) } } @@ -624,9 +643,12 @@ where /// the underlying async transport fails. pub async fn flush(&mut self) -> Result<()> { if !self.write_buf.is_empty() { + // Left set if this future is dropped or fails mid-write. + self.write_in_flight = true; self.stream.write_all(&self.write_buf).await?; self.stream.flush().await?; self.write_buf.clear(); + self.write_in_flight = false; } Ok(()) } @@ -731,20 +753,25 @@ where /// - Returns [`Error`] (I/O) if flushing buffered bytes or writing /// the header / payload to the async transport fails. pub async fn send_copy_data_direct(&mut self, data: &[u8]) -> Result<()> { - // First flush any pending buffered data - if !self.write_buf.is_empty() { - self.stream.write_all(&self.write_buf).await?; - self.write_buf.clear(); - } - // Write CopyData message header + data directly to stream // Message format: 'd' (1 byte) + length (4 bytes BigEndian) + data let msg_len = u32::try_from(4 + data.len()) .map_err(|_| Error::protocol("CopyData payload exceeds u32::MAX bytes"))?; let len_be = msg_len.to_be_bytes(); let header = [b'd', len_be[0], len_be[1], len_be[2], len_be[3]]; + + // `write_all` is not cancel-safe: if this future is dropped between + // the awaits below, a partial frame stays on the wire. The flag is + // left set in that case (see `write_in_flight`). + self.write_in_flight = true; + // First flush any pending buffered data + if !self.write_buf.is_empty() { + self.stream.write_all(&self.write_buf).await?; + self.write_buf.clear(); + } self.stream.write_all(&header).await?; self.stream.write_all(data).await?; + self.write_in_flight = false; Ok(()) } @@ -772,6 +799,7 @@ where /// - Returns [`Error`] (I/O) / [`Error`] (closed) on transport /// read/write failure. pub async fn finish_copy(&mut self) -> Result { + self.ensure_healthy()?; self.flush().await?; frontend::copy_done(&mut self.write_buf); @@ -808,6 +836,7 @@ where /// `CopyFail` frame fails, or [`Error`] (closed) if the server /// drops the connection before returning `ReadyForQuery`. pub async fn cancel_copy(&mut self, reason: &str) -> Result<()> { + self.ensure_healthy()?; self.flush().await?; frontend::copy_fail(reason, &mut self.write_buf); @@ -1154,3 +1183,77 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A write future dropped mid-`write_all` (cancellation) must poison the + /// connection. The peer end of the duplex pipe is never read, and its + /// 8-byte capacity guarantees `write_all` parks after a partial write. + #[tokio::test] + async fn cancelled_copy_write_marks_connection_unhealthy() { + let (client, _peer) = tokio::io::duplex(8); + let mut conn = AsyncRawConnection::new(client); + assert!(conn.is_healthy()); + + let payload = [0u8; 1024]; + tokio::select! { + biased; + res = conn.send_copy_data_direct(&payload) => { + panic!("write into a full pipe cannot complete: {res:?}"); + } + () = std::future::ready(()) => {} + } + + assert!( + !conn.is_healthy(), + "an interrupted write must poison the connection" + ); + assert!(conn.ensure_healthy().is_err()); + } + + /// With a partial frame on the wire, a queued `CopyFail` would be read by + /// the server as part of that frame and never seen. `queue_copy_fail` + /// must not queue it and must leave the connection desynchronized. + #[tokio::test] + async fn queue_copy_fail_after_interrupted_write_desynchronizes() { + let (client, _peer) = tokio::io::duplex(8); + let mut conn = AsyncRawConnection::new(client); + + tokio::select! { + biased; + _ = conn.send_copy_data_direct(&[0u8; 1024]) => unreachable!(), + () = std::future::ready(()) => {} + } + conn.queue_copy_fail("dropped"); + + assert!( + !conn.pending_copy_cancel, + "CopyFail must not follow a partial frame" + ); + assert!(conn.desynchronized); + } + + /// A completed write leaves the connection healthy and `queue_copy_fail` + /// behaves as before. + #[tokio::test] + async fn completed_copy_write_stays_healthy() { + let (client, mut peer) = tokio::io::duplex(4096); + let mut conn = AsyncRawConnection::new(client); + + conn.send_copy_data_direct(b"abc").await.unwrap(); + assert!(conn.is_healthy()); + + conn.queue_copy_fail("abandoned"); + assert!(conn.pending_copy_cancel); + assert!(conn.is_healthy()); + + // 'd' + len(7) + payload reached the peer intact. + let mut got = [0u8; 8]; + tokio::io::AsyncReadExt::read_exact(&mut peer, &mut got) + .await + .unwrap(); + assert_eq!(&got, &[b'd', 0, 0, 0, 7, b'a', b'b', b'c']); + } +} diff --git a/hyperdb-api-core/src/client/grpc/authenticated_client.rs b/hyperdb-api-core/src/client/grpc/authenticated_client.rs index 034a7e0b..6f937e5d 100644 --- a/hyperdb-api-core/src/client/grpc/authenticated_client.rs +++ b/hyperdb-api-core/src/client/grpc/authenticated_client.rs @@ -37,10 +37,10 @@ use std::collections::HashMap; use crate::client::error::{Error, Result}; use hyperdb_api_salesforce::{DataCloudToken, SharedTokenProvider}; +use super::config::TransferMode; use super::error::from_grpc_status; use super::executor::GrpcQueryExecutor; use super::params::{ParameterStyle, QueryParameters}; -use super::proto::hyper_service::query_param::TransferMode; use super::proto::{ AttachedDatabase, CancelQueryParam, HyperServiceClient, OutputFormat, QueryParam, }; diff --git a/hyperdb-api-core/src/client/grpc/client.rs b/hyperdb-api-core/src/client/grpc/client.rs index c3b44c2b..0152a30d 100644 --- a/hyperdb-api-core/src/client/grpc/client.rs +++ b/hyperdb-api-core/src/client/grpc/client.rs @@ -14,10 +14,10 @@ use tracing::{debug, info, warn}; use crate::client::error::{Error, Result}; use super::config::GrpcConfig; +use super::config::TransferMode; use super::error::from_grpc_status; use super::executor::{GrpcChunkStream, GrpcQueryExecutor}; use super::params::{ParameterStyle, QueryParameters}; -use super::proto::hyper_service::query_param::TransferMode; use super::proto::{ AttachedDatabase, CancelQueryParam, HyperServiceClient, OutputFormat, QueryParam, }; @@ -555,7 +555,7 @@ impl GrpcClient { /// # When do you have a `query_id`? /// /// The server assigns a `query_id` for queries started in - /// [`TransferMode::Async`](super::proto::hyper_service::query_param::TransferMode) + /// [`TransferMode::Async`] /// (long-running queries that the client polls). Grab it from /// [`GrpcQueryResult::query_id`](super::result::GrpcQueryResult::query_id) /// after `execute_query_with_options(..., TransferMode::Async)` returns. diff --git a/hyperdb-api-core/src/client/grpc/config.rs b/hyperdb-api-core/src/client/grpc/config.rs index 574408bf..e17cb390 100644 --- a/hyperdb-api-core/src/client/grpc/config.rs +++ b/hyperdb-api-core/src/client/grpc/config.rs @@ -9,7 +9,41 @@ use std::collections::HashMap; use std::time::Duration; -use super::proto::hyper_service::query_param::TransferMode; +use super::proto::hyper_service::query_param::TransferMode as ProtoTransferMode; + +/// How the server delivers query results. +/// +/// This is the crate's own enum rather than the generated protobuf one, so +/// regenerating the proto does not change the public API. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TransferMode { + /// All results arrive in the `ExecuteQuery` response (simple; subject to + /// the server's 100 s timeout). + Sync, + /// The `ExecuteQuery` response carries only a header; results are fetched + /// with `GetQueryResult`. + Async, + /// The first chunk arrives inline and the rest is streamed through + /// `GetQueryResult` (the default, and recommended). + #[default] + Adaptive, +} + +impl From for ProtoTransferMode { + fn from(mode: TransferMode) -> Self { + match mode { + TransferMode::Sync => ProtoTransferMode::Sync, + TransferMode::Async => ProtoTransferMode::Async, + TransferMode::Adaptive => ProtoTransferMode::Adaptive, + } + } +} + +impl From for i32 { + fn from(mode: TransferMode) -> Self { + ProtoTransferMode::from(mode).into() + } +} /// Default maximum message size for gRPC requests/responses (64 MB). /// diff --git a/hyperdb-api-core/src/client/grpc/executor.rs b/hyperdb-api-core/src/client/grpc/executor.rs index 2214a398..19d59699 100644 --- a/hyperdb-api-core/src/client/grpc/executor.rs +++ b/hyperdb-api-core/src/client/grpc/executor.rs @@ -50,8 +50,8 @@ use tracing::{debug, trace, warn}; use crate::client::error::{Error, Result}; +use super::config::TransferMode; use super::error::from_grpc_status; -use super::proto::hyper_service::query_param::TransferMode; use super::proto::hyper_service::query_result::Result as QueryResultPayload; use super::proto::hyper_service::query_status::CompletionStatus; use super::proto::{ @@ -277,7 +277,7 @@ where // chunks 1..N still need to be fetched for ADAPTIVE // (and 0..N for ASYNC). match self.transfer_mode { - TransferMode::Sync | TransferMode::Unspecified => { + TransferMode::Sync => { debug!( query_id = ?self.query_id, "ExecuteQuery stream closed; SYNC mode complete", diff --git a/hyperdb-api-core/src/client/grpc/mod.rs b/hyperdb-api-core/src/client/grpc/mod.rs index 2a89565f..30f8e9b3 100644 --- a/hyperdb-api-core/src/client/grpc/mod.rs +++ b/hyperdb-api-core/src/client/grpc/mod.rs @@ -126,7 +126,7 @@ pub use result::{GrpcColumnInfo, GrpcQueryResult, GrpcResultChunk}; pub use authenticated_client::{AuthenticatedGrpcClient, AuthenticatedGrpcClientSync, TableInfo}; // Re-export transfer mode for users who want to specify it -pub use proto::hyper_service::query_param::TransferMode; +pub use config::TransferMode; // Re-export output format for advanced users pub use proto::hyper_service::OutputFormat; diff --git a/hyperdb-api-core/src/client/grpc/result.rs b/hyperdb-api-core/src/client/grpc/result.rs index 0d596413..a72a9e35 100644 --- a/hyperdb-api-core/src/client/grpc/result.rs +++ b/hyperdb-api-core/src/client/grpc/result.rs @@ -247,9 +247,10 @@ impl GrpcResultChunk { #[derive(Debug)] pub struct GrpcColumnInfo<'a> { /// Column name - pub name: &'a str, - /// SQL type information - pub sql_type: Option, + pub(crate) name: &'a str, + /// SQL type information (the generated protobuf message, so it is not + /// part of the public API; see [`Self::type_name`]) + pub(crate) sql_type: Option, } impl GrpcColumnInfo<'_> { diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index dc84716f..afdafa38 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -33,19 +33,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `pool::Pool` and `pool::PooledConnection` were aliases of `deadpool::managed::Pool` / `Object`; they are now `hyperdb-api` types, and `pool::ConnectionManager` is crate-private. `Pool::get` returns - `hyperdb_api::Result` — a wait/create/recycle timeout is - `Error::Timeout`, a closed pool is `Error::Connection`, and a failed connect - is the underlying error — so the `.map_err(|e| Error::internal(e.to_string()))` - adapter is no longer needed. `Pool::status` returns `PoolStatus` (as - `ConnectionPool` does), and `PooledConnection::take` frees the slot. The - `deadpool` version is no longer part of this crate's semver surface. + `hyperdb_api::Result` — a wait or create timeout is + `Error::Timeout`, a closed pool is `Error::InvalidOperation`, and a failed + connect is the underlying error — so the + `.map_err(|e| Error::internal(e.to_string()))` adapter is no longer needed. + A connection that fails its recycle check (or whose recycle times out) is + discarded and replaced, never reported to the caller. `Pool::status` returns + `PoolStatus` (as `ConnectionPool` does), and `PooledConnection::take` frees + the slot. The `deadpool` version is no longer part of this crate's semver + surface. The deadpool-only API is gone with it: `Pool::resize`, `retain`, + `timeout_get`, `manager` and `weak`, `status().waiting`, and + `Object::id`, `metrics`, `AsRef` and `AsMut`. +- **BREAKING: `hyperdb_api::grpc::TransferMode` is now a `hyperdb-api-core` + enum (`Sync`, `Async`, `Adaptive`)** instead of the prost-generated type, so + no protobuf type appears in `GrpcConfig::transfer_mode` or + `ConnectionBuilder::transfer_mode`. `Adaptive` stays the default; the + protocol's `Unspecified` value is no longer constructible. +- **BREAKING: `GrpcColumnInfo` no longer has public fields.** Use + `GrpcColumnInfo::name()` and `type_name()`. +- **`HyperProcess` makes a caller-supplied `domain_socket_directory` + absolute.** A relative directory produced an endpoint string such as + `rs-123/hyper` that `Connection::new` could not resolve; the endpoint now + always starts with `/` on Unix. ### Added - **`AsyncConnection::copy_in_hyperbinary`** streams a pre-encoded `HyperBinary` buffer (for example from `InsertChunk`) into a table with `COPY ... FROM STDIN`, slicing it under `hyperd`'s COPY packet cap, and - returns the inserted row count. + returns the inserted row count. The buffer must start with exactly one + HyperBinary header (the first `InsertChunk::take()` of a fresh or cleared + chunk); a headerless buffer is rejected with `Error::InvalidOperation`. The + future is not cancel-safe: dropping it mid-write leaves the connection + reporting unhealthy, so a pool evicts it on the next checkout. - **`HyperProcess::connection_endpoint_string`**, and `Pool::close` / `Pool::is_closed` on the async pool. diff --git a/hyperdb-api/examples/additional_examples/async_usage.rs b/hyperdb-api/examples/additional_examples/async_usage.rs index 85fe1e24..d99fdb59 100644 --- a/hyperdb-api/examples/additional_examples/async_usage.rs +++ b/hyperdb-api/examples/additional_examples/async_usage.rs @@ -368,5 +368,5 @@ fn print_best_practices() { println!(" Once data is retrieved, process it with async code (HTTP, file I/O).\n"); println!("7. Consider connection pooling for high load"); - println!(" See the connection_pool example for deadpool integration.\n"); + println!(" See the connection_pool example for async connection pooling.\n"); } diff --git a/hyperdb-api/examples/additional_examples/connection_pool.rs b/hyperdb-api/examples/additional_examples/connection_pool.rs index 41269bbd..3408f8c3 100644 --- a/hyperdb-api/examples/additional_examples/connection_pool.rs +++ b/hyperdb-api/examples/additional_examples/connection_pool.rs @@ -1,12 +1,12 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Connection pool example demonstrating async connection pooling with deadpool. +//! Connection pool example demonstrating async connection pooling. //! //! This example shows how to use the connection pool for efficient connection //! reuse in async applications with multiple concurrent tasks. //! -//! Run with: cargo run --example `connection_pool` --features pool +//! Run with: cargo run -p hyperdb-api --example connection_pool use std::time::Duration; @@ -44,10 +44,7 @@ async fn main() -> Result<()> { // Get a connection to set up the schema { - let conn = pool - .get() - .await - .map_err(|e| hyperdb_api::Error::internal(e.to_string()))?; + let conn = pool.get().await?; conn.execute_command( "CREATE TABLE counters ( id INT NOT NULL, @@ -102,7 +99,11 @@ async fn main() -> Result<()> { } println!("\nAll tasks completed!"); - println!("Pool status: {} connections in use", pool.status().size); + let status = pool.status(); + println!( + "Pool status: {} connections open, {} idle", + status.size, status.idle + ); Ok(()) } diff --git a/hyperdb-api/examples/async_parity_smoke.rs b/hyperdb-api/examples/async_parity_smoke.rs index 7cb5db86..2cdc23f1 100644 --- a/hyperdb-api/examples/async_parity_smoke.rs +++ b/hyperdb-api/examples/async_parity_smoke.rs @@ -5,7 +5,7 @@ //! //! Exercises the full async surface — connect, DDL, parameterized //! DML, streaming reads, struct mapping, transactions — **without** -//! calling `async_tcp_client()` or importing anything from +//! reaching for the underlying protocol client or importing anything from //! `hyperdb_api_core::client`. If this example compiles and runs clean, the //! "async is a first-class equivalent of sync" contract holds. diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index c5ad879f..b2012fe1 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -1119,20 +1119,37 @@ impl AsyncConnection { /// Streams a pre-encoded `HyperBinary` buffer into `table` with `COPY ... FROM STDIN`. /// - /// `data` must hold rows in the `HyperBinary` format for the table's columns, - /// in definition order — for example the output of - /// [`InsertChunk`](crate::InsertChunk). The buffer is sent in 64 MiB slices - /// because `hyperd` rejects COPY packets above ~150 MB. Returns the number of - /// rows the server reports as inserted; an empty buffer inserts nothing and - /// returns `0`. + /// `data` must be one complete `HyperBinary` stream for the table's + /// columns, in definition order: exactly one `HyperBinary` header followed + /// by the rows. That is the buffer from the **first** + /// [`InsertChunk::take`](crate::InsertChunk::take) of a fresh (or + /// [`clear`](crate::InsertChunk::clear)ed) chunk. Later `take()` calls on + /// the same chunk return rows without a header, and two buffers + /// concatenated carry two headers; both are rejected. The buffer is sent + /// in 64 MiB slices because `hyperd` rejects COPY packets above ~150 MB. + /// Returns the number of rows the server reports as inserted; an empty + /// buffer inserts nothing and returns `0`. /// /// Most callers want [`AsyncInserter`](crate::AsyncInserter), which encodes /// rows for you. Use this when the bytes are already encoded. /// + /// # Cancel safety + /// + /// Not cancel-safe. Dropping the returned future before it resolves (for + /// example through `tokio::time::timeout` or `select!`) can leave a + /// partial message on the wire. The connection then reports itself + /// unhealthy and every later call on it fails with + /// [`Error::Connection`]; discard it and open a new one. A pooled + /// connection in that state is evicted by the default + /// [`RecycleStrategy::SelectOne`](crate::pool::RecycleStrategy::SelectOne) + /// probe the next time it is checked out. + /// /// # Errors /// /// - Returns [`Error::feature_not_supported`] on a gRPC connection, which /// cannot run COPY. + /// - Returns [`Error::invalid_operation`] if `data` does not start with a + /// `HyperBinary` header. /// - Returns an error if the server rejects the COPY or the connection /// fails while sending. pub async fn copy_in_hyperbinary(&self, table: &TableDefinition, data: &[u8]) -> Result { @@ -1148,12 +1165,16 @@ impl AsyncConnection { gRPC connections do not support COPY operations.", ) })?; + if !data.starts_with(hyperdb_api_core::protocol::copy::HYPER_BINARY_HEADER) { + return Err(Error::invalid_operation( + "copy_in_hyperbinary: data must start with a HyperBinary header \ + (use the first InsertChunk::take() of a fresh chunk)", + )); + } let columns: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect(); let table_name = table.qualified_name(); - let mut writer = client - .copy_in_arc_with_format(&table_name, &columns, "HYPERBINARY") - .await?; + let mut writer = client.copy_in(&table_name, &columns).await?; for slice in data.chunks(MAX_COPY_CHUNK) { writer.send_direct(slice).await?; writer.flush_stream().await?; diff --git a/hyperdb-api/src/async_connection_builder.rs b/hyperdb-api/src/async_connection_builder.rs index c507456d..cef6d1ae 100644 --- a/hyperdb-api/src/async_connection_builder.rs +++ b/hyperdb-api/src/async_connection_builder.rs @@ -51,7 +51,7 @@ pub struct AsyncConnectionBuilder { login_timeout: Option, query_timeout: Option, application_name: Option, - transfer_mode: Option, + transfer_mode: Option, } impl Default for AsyncConnectionBuilder { @@ -173,7 +173,7 @@ impl AsyncConnectionBuilder { /// Sets the transfer mode for gRPC connections (ignored for TCP). #[must_use] - pub fn transfer_mode(mut self, mode: hyperdb_api_core::client::grpc::TransferMode) -> Self { + pub fn transfer_mode(mut self, mode: crate::grpc::TransferMode) -> Self { self.transfer_mode = Some(mode); self } diff --git a/hyperdb-api/src/connection_builder.rs b/hyperdb-api/src/connection_builder.rs index 99f0b589..72c935d7 100644 --- a/hyperdb-api/src/connection_builder.rs +++ b/hyperdb-api/src/connection_builder.rs @@ -62,7 +62,7 @@ pub struct ConnectionBuilder { /// Application name sent to the server during connection startup. application_name: Option, /// Transfer mode for gRPC connections (ignored for TCP) - transfer_mode: Option, + transfer_mode: Option, } impl Default for ConnectionBuilder { @@ -229,7 +229,7 @@ impl ConnectionBuilder { /// } /// ``` #[must_use] - pub fn transfer_mode(mut self, mode: hyperdb_api_core::client::grpc::TransferMode) -> Self { + pub fn transfer_mode(mut self, mode: crate::grpc::TransferMode) -> Self { self.transfer_mode = Some(mode); self } diff --git a/hyperdb-api/src/lib.rs b/hyperdb-api/src/lib.rs index 6924d1f0..2805759e 100644 --- a/hyperdb-api/src/lib.rs +++ b/hyperdb-api/src/lib.rs @@ -114,7 +114,7 @@ //! # Public Modules //! //! - [`copy`] — CSV/text export and import via COPY protocol -//! - [`pool`] — Async connection pooling (deadpool-based) +//! - [`pool`] — Async connection pooling //! - [`grpc`] — gRPC transport types for Arrow IPC queries //! //! # Bulk Data Loading diff --git a/hyperdb-api/src/pool.rs b/hyperdb-api/src/pool.rs index 40e964a6..17a9337c 100644 --- a/hyperdb-api/src/pool.rs +++ b/hyperdb-api/src/pool.rs @@ -251,7 +251,7 @@ pub struct PoolConfig { /// [`RecycleStrategy::SelectOne`]. pub recycle: RecycleStrategy, /// Maximum time to wait for a slot to become available on - /// [`get`](managed::Pool::get). `None` waits indefinitely (the default). + /// [`get`](Pool::get). `None` waits indefinitely (the default). pub wait_timeout: Option, /// Maximum time to wait for a new connection to be created. `None` disables /// the cap (the default). @@ -615,18 +615,23 @@ impl Pool { /// /// # Errors /// - /// - Returns [`Error::Timeout`] if `wait_timeout`, `create_timeout` or - /// `recycle_timeout` elapses. - /// - Returns [`Error::Connection`] if the pool has been [closed](Self::close). + /// - Returns [`Error::Timeout`] if `wait_timeout` or `create_timeout` + /// elapses. + /// - Returns [`Error::InvalidOperation`] if the pool has been + /// [closed](Self::close). Retrying cannot succeed. /// - Returns the underlying error if opening a connection fails or an /// `after_connect` hook rejects it. + /// + /// A connection that fails, or exceeds `recycle_timeout`, during its + /// recycle probe is not reported: the pool discards it and hands out + /// another one (opening a new connection if needed). pub async fn get(&self) -> Result { match self.inner.get().await { Ok(inner) => Ok(PooledConnection { inner }), Err(managed::PoolError::Backend(e)) => Err(e), Err(e @ managed::PoolError::Timeout(_)) => Err(Error::timeout(e.to_string())), Err(managed::PoolError::Closed) => { - Err(Error::connection("the connection pool is closed")) + Err(Error::invalid_operation("the connection pool is closed")) } Err(e) => Err(Error::internal(e.to_string())), } @@ -1214,14 +1219,20 @@ impl ConnectionPool { } } -/// A snapshot of [`ConnectionPool`] occupancy. +/// A snapshot of pool occupancy, returned by [`ConnectionPool::status`] and +/// [`Pool::status`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct PoolStatus { /// Number of idle connections currently available. + /// + /// For the async [`Pool`] this is the number of idle connections not + /// already claimed by a task blocked in [`get`](Pool::get), so it can read + /// lower than the number physically idle while callers are waiting. pub idle: usize, /// Total live connections (idle + checked out). pub size: usize, - /// Configured maximum pool size. + /// Configured maximum pool size. The async [`Pool`] reports `0` once it + /// has been [closed](Pool::close). pub max_size: usize, } diff --git a/hyperdb-api/src/process.rs b/hyperdb-api/src/process.rs index e42dc42f..309c7a7a 100644 --- a/hyperdb-api/src/process.rs +++ b/hyperdb-api/src/process.rs @@ -428,7 +428,13 @@ impl HyperProcess { let (dir, owned) = if let Some(custom_dir) = parameters.and_then(|p| p.domain_socket_directory.as_ref()) { - (custom_dir.clone(), false) + // Absolutized: the directory is rendered into the endpoint + // string, and a relative path would be taken for a TCP + // host when handed back to `Connection::new`. + ( + std::path::absolute(custom_dir).unwrap_or_else(|_| custom_dir.clone()), + false, + ) } else { // Create a temp directory for the socket. The basename carries a // per-process monotonic suffix (`hyper--`) so two diff --git a/hyperdb-api/tests/async_connection_tests.rs b/hyperdb-api/tests/async_connection_tests.rs index 68f37d93..37bf2737 100644 --- a/hyperdb-api/tests/async_connection_tests.rs +++ b/hyperdb-api/tests/async_connection_tests.rs @@ -538,3 +538,93 @@ async fn copy_in_hyperbinary_inserts_encoded_rows() { .unwrap(); assert_eq!(count, 3); } + +/// Only the first `InsertChunk::take()` carries the `HyperBinary` header, so +/// a later buffer is not a complete COPY stream. It is refused up front with +/// `InvalidOperation` rather than failing mid-COPY on the server. +#[tokio::test(flavor = "current_thread")] +async fn copy_in_hyperbinary_rejects_headerless_buffer() { + use hyperdb_api::{Error, InsertChunk, SqlType, TableDefinition}; + + let (_hyper, conn) = fresh_async_conn("async_copy_in_headerless").await.unwrap(); + let table_def = TableDefinition::from("headerless").add_required_column("id", SqlType::int()); + conn.execute_command("CREATE TABLE headerless (id INT NOT NULL)") + .await + .unwrap(); + + let mut chunk = InsertChunk::from_table_definition(&table_def); + chunk.add_i32(1).unwrap(); + chunk.end_row().unwrap(); + let first = chunk.take().expect("first buffer"); + chunk.add_i32(2).unwrap(); + chunk.end_row().unwrap(); + let second = chunk.take().expect("second buffer"); + + let err = conn + .copy_in_hyperbinary(&table_def, &second) + .await + .expect_err("a headerless buffer is not a COPY stream"); + assert!( + matches!(err, Error::InvalidOperation(_)), + "expected InvalidOperation, got {err:?}" + ); + + // The connection is untouched, and the first buffer still works. + assert_eq!( + conn.copy_in_hyperbinary(&table_def, &first).await.unwrap(), + 1 + ); +} + +/// A server-side COPY failure surfaces as an error and leaves the connection +/// usable for the next statement. +#[tokio::test(flavor = "current_thread")] +async fn copy_in_hyperbinary_server_error_leaves_connection_usable() { + use hyperdb_api::{SqlType, TableDefinition}; + use hyperdb_api_core::protocol::copy::HYPER_BINARY_HEADER; + + let (_hyper, conn) = fresh_async_conn("async_copy_in_server_error") + .await + .unwrap(); + let table_def = TableDefinition::from("broken").add_required_column("id", SqlType::int()); + conn.execute_command("CREATE TABLE broken (id INT NOT NULL)") + .await + .unwrap(); + + // A valid header followed by 3 bytes: a truncated `INT` row. + let mut data = HYPER_BINARY_HEADER.to_vec(); + data.extend_from_slice(&[0x01; 3]); + conn.copy_in_hyperbinary(&table_def, &data) + .await + .expect_err("a truncated row must be rejected"); + + let one: i64 = conn.fetch_scalar("SELECT 1").await.unwrap(); + assert_eq!(one, 1, "connection must stay usable after a failed COPY"); +} + +/// COPY needs the TCP wire protocol; a gRPC connection reports that instead +/// of attempting it. +#[tokio::test(flavor = "current_thread")] +async fn copy_in_hyperbinary_on_grpc_is_feature_not_supported() { + use hyperdb_api::{Error, ListenMode, Parameters, SqlType, TableDefinition}; + use hyperdb_api_core::protocol::copy::HYPER_BINARY_HEADER; + + let mut params = Parameters::new(); + params.set("log_dir", "test_results"); + params.set_listen_mode(ListenMode::Grpc { port: 0 }); + let hyper = HyperProcess::new(None, Some(¶ms)).unwrap(); + let url = hyper.grpc_url().expect("gRPC URL"); + let conn = AsyncConnection::connect(&url, "", CreateMode::DoNotCreate) + .await + .unwrap(); + + let table_def = TableDefinition::from("t").add_required_column("id", SqlType::int()); + let err = conn + .copy_in_hyperbinary(&table_def, HYPER_BINARY_HEADER) + .await + .expect_err("COPY over gRPC is not supported"); + assert!( + matches!(err, Error::FeatureNotSupported(_)), + "expected FeatureNotSupported, got {err:?}" + ); +} diff --git a/hyperdb-api/tests/pool_tests.rs b/hyperdb-api/tests/pool_tests.rs index 0e3991a4..b2f0dc3d 100644 --- a/hyperdb-api/tests/pool_tests.rs +++ b/hyperdb-api/tests/pool_tests.rs @@ -139,9 +139,10 @@ async fn async_pool_status_and_take() { } /// A closed pool refuses checkouts with a `hyperdb_api::Error`, not a -/// deadpool error type. +/// deadpool error type. It is `InvalidOperation` (retrying cannot succeed), +/// not `Connection` (which callers treat as transient). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn async_pool_get_after_close_is_connection_error() { +async fn async_pool_get_after_close_is_invalid_operation() { let (_hyper, endpoint) = fresh_server("pool_async_closed").unwrap(); let config = PoolConfig::new(&endpoint, db_path("pool_async_closed")) .create_mode(CreateMode::CreateAndReplace) @@ -156,8 +157,8 @@ async fn async_pool_get_after_close_is_connection_error() { .await .expect_err("closed pool must refuse checkout"); assert!( - matches!(err, hyperdb_api::Error::Connection { .. }), - "expected Error::Connection, got {err:?}" + matches!(err, hyperdb_api::Error::InvalidOperation(_)), + "expected Error::InvalidOperation, got {err:?}" ); } @@ -168,7 +169,10 @@ async fn async_pool_get_surfaces_connect_error() { // Nothing listens on port 1; the first checkout has to open a connection. let config = PoolConfig::new("127.0.0.1:1", db_path("pool_async_connect_err")) .create_mode(CreateMode::CreateAndReplace) - .max_size(1); + .max_size(1) + // Guard only: refusal is immediate on Linux and macOS, but a + // filtering firewall or Windows' SYN retries would otherwise stall. + .create_timeout(Some(Duration::from_secs(30))); let pool = create_pool(config).unwrap(); let err = pool.get().await.expect_err("nothing is listening"); @@ -178,6 +182,27 @@ async fn async_pool_get_surfaces_connect_error() { ); } +/// A `create_timeout` that elapses is `Error::Timeout`. The listener accepts +/// the TCP connection (the kernel does, via the backlog) but never answers +/// the startup message, so the connect hangs until the timeout fires. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_pool_create_timeout_is_timeout_error() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let config = PoolConfig::new(addr.to_string(), db_path("pool_async_create_timeout")) + .create_mode(CreateMode::CreateAndReplace) + .max_size(1) + .create_timeout(Some(Duration::from_millis(300))); + let pool = create_pool(config).unwrap(); + + let err = pool.get().await.expect_err("the server never answers"); + assert!( + matches!(err, hyperdb_api::Error::Timeout(_)), + "expected Error::Timeout, got {err:?}" + ); + drop(listener); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn async_pool_recycle_ping_strategy_works() { let (_hyper, endpoint) = fresh_server("pool_async_ping").unwrap(); diff --git a/hyperdb-api/tests/socket_dir_cleanup_tests.rs b/hyperdb-api/tests/socket_dir_cleanup_tests.rs index f656c7ae..5b431e35 100644 --- a/hyperdb-api/tests/socket_dir_cleanup_tests.rs +++ b/hyperdb-api/tests/socket_dir_cleanup_tests.rs @@ -47,3 +47,41 @@ fn default_socket_dir_is_removed_on_drop() { }; assert!(!dir.exists(), "default socket dir should be cleaned up"); } + +/// `connection_endpoint_string` must be connectable even when the caller gave +/// a relative socket directory: a relative path would be taken for a TCP host +/// by `Connection::new`, so the directory is made absolute up front. +#[test] +fn relative_socket_directory_yields_connectable_endpoint_string() { + let rel = std::path::PathBuf::from(format!("rs-{}", std::process::id())); + std::fs::create_dir_all(&rel).expect("create relative dir"); + + /// Removes the test's leftovers even when an assertion panics. + struct Cleanup(std::path::PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + let _ = std::fs::remove_file("rel_socket.hyper"); + } + } + let _cleanup = Cleanup(rel.clone()); + + let mut params = Parameters::new(); + params.set_transport_mode(TransportMode::Ipc); + params.set_domain_socket_directory(&rel); + let hyper = HyperProcess::new(None, Some(¶ms)).expect("start hyperd over IPC"); + let endpoint = hyper + .connection_endpoint_string() + .expect("IPC process has an endpoint"); + assert!( + endpoint.starts_with('/'), + "endpoint must be absolute so it is routed to the Unix socket: {endpoint}" + ); + let conn = hyperdb_api::Connection::new( + &hyper, + "rel_socket.hyper", + hyperdb_api::CreateMode::CreateAndReplace, + ) + .expect("connect over the relative-dir socket"); + conn.execute_command("SELECT 1").expect("query"); +} From daa217e17b87bad54a2bca5bf4a3f13f97ffd019 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 23:33:58 -0700 Subject: [PATCH 15/61] docs(api): transport failures are Error::Connection, not Error::Io The core client's Io errors map to hyperdb_api::Error::Connection, and Error::Io is for direct SDK-boundary I/O (as error.rs already says), but about twenty `# Errors` sections promised Error::Io for network failures. Also correct DEVELOPMENT-client.md, which said pooling is not exposed. --- hyperdb-api-core/docs/DEVELOPMENT-client.md | 5 +++-- hyperdb-api/src/arrow_result.rs | 2 +- hyperdb-api/src/async_connection.rs | 22 ++++++++++----------- hyperdb-api/src/async_connection_builder.rs | 2 +- hyperdb-api/src/async_prepared.rs | 8 ++++---- hyperdb-api/src/async_result.rs | 2 +- hyperdb-api/src/connection.rs | 8 ++++---- hyperdb-api/src/prepared.rs | 4 ++-- 8 files changed, 27 insertions(+), 26 deletions(-) diff --git a/hyperdb-api-core/docs/DEVELOPMENT-client.md b/hyperdb-api-core/docs/DEVELOPMENT-client.md index 8170d88e..8438aff6 100644 --- a/hyperdb-api-core/docs/DEVELOPMENT-client.md +++ b/hyperdb-api-core/docs/DEVELOPMENT-client.md @@ -295,8 +295,9 @@ See `cancel.rs` for the full rationale. - `GrpcClient` duplicates query-building logic between `execute_query_with_options` and `execute_query_with_params_and_options` -- should be unified - `AsyncClient` lacks a streaming query mode equivalent to `QueryStream` -- Connection pooling exists as `deadpool` integration but is not yet exposed - as a first-class API in this crate +- Connection pooling is not part of this crate. It lives in `hyperdb-api`'s + `pool` module, which wraps `deadpool` behind its own `Pool` and + `PooledConnection` types - `TlsConfig` / `TlsMode` are defined but not yet wired into `Config`'s builder (TLS is configured at a lower level currently) diff --git a/hyperdb-api/src/arrow_result.rs b/hyperdb-api/src/arrow_result.rs index 73ecccff..da1ed738 100644 --- a/hyperdb-api/src/arrow_result.rs +++ b/hyperdb-api/src/arrow_result.rs @@ -387,7 +387,7 @@ pub trait ChunkSource: Send { /// /// Implementations return whatever transport error the underlying /// source produces (typically [`Error::Server`] from a gRPC stream or - /// [`Error::Io`] on network failures). + /// [`Error::Connection`] on network failures). fn next_chunk(&mut self) -> Result>; } diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index b2012fe1..9590ace3 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -74,7 +74,7 @@ impl AsyncConnection { /// /// # Errors /// - /// - Returns [`Error::Io`] / [`Error::Connection`] if the handshake with + /// - Returns [`Error::Connection`] if the handshake with /// the server fails. /// - Returns [`Error::Server`] if the `CreateMode` SQL (`CREATE` /// / `DROP` / `ATTACH`) is rejected by the server. @@ -100,7 +100,7 @@ impl AsyncConnection { /// # Errors /// /// - Returns [`Error::Authentication`] if authentication is rejected. - /// - Returns [`Error::Io`] if the endpoint cannot be reached. + /// - Returns [`Error::Connection`] if the endpoint cannot be reached. /// - Returns [`Error::Server`] if the `CreateMode` SQL is rejected. pub async fn connect_with_auth( endpoint: &str, @@ -130,7 +130,7 @@ impl AsyncConnection { /// /// # Errors /// - /// Returns [`Error::Io`] or [`Error::Connection`] if the TCP handshake + /// Returns [`Error::Connection`] if the TCP handshake /// with `endpoint` fails. pub async fn without_database(endpoint: &str) -> Result { let transport = AsyncTransport::connect_tcp(endpoint).await?; @@ -253,7 +253,7 @@ impl AsyncConnection { /// - Returns [`Error::FeatureNotSupported`] on gRPC transports that do not yet /// support write operations. /// - Returns [`Error::Server`] if the SQL fails to parse or execute. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn execute_command(&self, sql: &str) -> Result { let token = self.stats_before_query(sql); let result = self.transport.execute_command(sql).await; @@ -303,7 +303,7 @@ impl AsyncConnection { /// # Errors /// /// - Returns [`Error::Server`] if the SQL is rejected by the server. - /// - Returns [`Error::Io`] on transport-level I/O failures while + /// - Returns [`Error::Connection`] on transport-level I/O failures while /// opening the stream. pub async fn execute_query(&self, query: &str) -> Result> { let token = self.stats_before_query(query); @@ -722,7 +722,7 @@ impl AsyncConnection { /// are TCP-only). /// - Returns [`Error::Server`] if the server rejects the statement at /// `Parse`, `Bind`, or `Execute` time. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn query_params( &self, query: &str, @@ -764,7 +764,7 @@ impl AsyncConnection { /// - Returns [`Error::FeatureNotSupported`] on gRPC transports. /// - Returns [`Error::Server`] if the server rejects the statement at /// `Parse`, `Bind`, or `Execute` time. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn command_params( &self, query: &str, @@ -1018,7 +1018,7 @@ impl AsyncConnection { /// /// # Errors /// - /// Returns [`Error::Server`] or [`Error::Io`] if the `SELECT 1` + /// Returns [`Error::Server`] or [`Error::Connection`] if the `SELECT 1` /// round-trip fails — i.e. the connection is no longer usable. pub async fn ping(&self) -> Result<()> { self.execute_command("SELECT 1").await?; @@ -1072,7 +1072,7 @@ impl AsyncConnection { /// /// - Returns [`Error::FeatureNotSupported`] on gRPC transports — cancellation is not /// yet implemented for gRPC. - /// - Returns [`Error::Connection`] or [`Error::Io`] if the cancel-request + /// - Returns [`Error::Connection`] if the cancel-request /// connection to the server fails. pub async fn cancel(&self) -> Result<()> { self.transport.cancel().await @@ -1217,7 +1217,7 @@ impl AsyncConnection { /// are TCP-only). /// - Returns [`Error::Server`] if the server rejects the `Parse` /// message (SQL syntax error, unknown OID). - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn prepare_typed( &self, query: &str, @@ -1260,7 +1260,7 @@ impl AsyncConnection { /// - Returns [`Error::FeatureNotSupported`] on gRPC transports. /// - Returns [`Error::Server`] if the server rejects the `Parse` /// message. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn prepare_typed_arc( self: &Arc, query: &str, diff --git a/hyperdb-api/src/async_connection_builder.rs b/hyperdb-api/src/async_connection_builder.rs index cef6d1ae..718ce5ea 100644 --- a/hyperdb-api/src/async_connection_builder.rs +++ b/hyperdb-api/src/async_connection_builder.rs @@ -189,7 +189,7 @@ impl AsyncConnectionBuilder { /// /// # Errors /// - /// - Returns [`Error::Io`] or [`Error::Connection`] if the transport + /// - Returns [`Error::Connection`] if the transport /// handshake fails (TCP refused, TLS rejected, named-pipe not /// found, gRPC channel setup failure). /// - Returns [`Error::Authentication`] if authentication is rejected. diff --git a/hyperdb-api/src/async_prepared.rs b/hyperdb-api/src/async_prepared.rs index c652b3cb..9366895f 100644 --- a/hyperdb-api/src/async_prepared.rs +++ b/hyperdb-api/src/async_prepared.rs @@ -83,7 +83,7 @@ impl<'conn> AsyncPreparedStatement<'conn> { /// - Returns [`Error::FeatureNotSupported`] on gRPC transport. /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn query(&self, params: &[&dyn ToSqlParam]) -> Result> { let (encoded, formats) = encode_params(params); let client = async_tcp_client(self.connection)?; @@ -106,7 +106,7 @@ impl<'conn> AsyncPreparedStatement<'conn> { /// - Returns [`Error::FeatureNotSupported`] on gRPC transport. /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn execute(&self, params: &[&dyn ToSqlParam]) -> Result { let (encoded, formats) = encode_params(params); let client = async_tcp_client(self.connection)?; @@ -259,7 +259,7 @@ impl AsyncPreparedStatementOwned { /// - Returns [`Error::FeatureNotSupported`] on gRPC transport. /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`, or raises a runtime error while streaming. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn fetch_all(&self, params: &[&dyn ToSqlParam]) -> Result> { let (encoded, formats) = encode_params(params); let client = async_tcp_client_arc(&self.connection)?; @@ -282,7 +282,7 @@ impl AsyncPreparedStatementOwned { /// - Returns [`Error::FeatureNotSupported`] on gRPC transport. /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn execute(&self, params: &[&dyn ToSqlParam]) -> Result { let (encoded, formats) = encode_params(params); let client = async_tcp_client_arc(&self.connection)?; diff --git a/hyperdb-api/src/async_result.rs b/hyperdb-api/src/async_result.rs index a0fe457a..7c3a09d8 100644 --- a/hyperdb-api/src/async_result.rs +++ b/hyperdb-api/src/async_result.rs @@ -179,7 +179,7 @@ impl<'conn> AsyncRowset<'conn> { /// /// - Returns [`crate::Error::Server`] if the server sends an `ErrorResponse` /// while streaming the result set. - /// - Returns [`crate::Error::Io`] on transport-level I/O failures. + /// - Returns [`crate::Error::Connection`] on transport-level I/O failures. /// - Returns [`crate::Error::Conversion`] if an Arrow IPC chunk cannot be decoded. pub async fn next_chunk(&mut self) -> Result>> { enum TransportChunk { diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index 1bd4c30a..8b00f16f 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -392,8 +392,8 @@ impl Connection { /// /// # Errors /// - /// Returns [`Error::Connection`] if the TCP or gRPC handshake fails, and - /// [`Error::Io`] if the endpoint cannot be reached. + /// Returns [`Error::Connection`] if the TCP or gRPC handshake fails or the + /// endpoint cannot be reached. pub fn without_database(endpoint: &str) -> Result { crate::ConnectionBuilder::new(endpoint).build() } @@ -1868,7 +1868,7 @@ impl Connection { /// /// # Errors /// - /// Returns [`Error::Server`] or [`Error::Io`] if the `SELECT 1` + /// Returns [`Error::Server`] or [`Error::Connection`] if the `SELECT 1` /// round-trip fails — i.e. the connection is no longer usable. pub fn ping(&self) -> Result<()> { self.execute_command("SELECT 1")?; @@ -1951,7 +1951,7 @@ impl Connection { /// /// - Returns [`Error::FeatureNotSupported`] on gRPC connections, which do not /// support query cancellation. - /// - Returns [`Error::Connection`] or [`Error::Io`] if the separate + /// - Returns [`Error::Connection`] if the separate /// cancel-request connection to the server fails. pub fn cancel(&self) -> Result<()> { match &self.transport { diff --git a/hyperdb-api/src/prepared.rs b/hyperdb-api/src/prepared.rs index f4426591..fd06e941 100644 --- a/hyperdb-api/src/prepared.rs +++ b/hyperdb-api/src/prepared.rs @@ -100,7 +100,7 @@ impl<'conn> PreparedStatement<'conn> { /// gRPC transport (prepared statements are TCP-only). /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute` (type mismatch, runtime error while streaming). - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub fn query(&self, params: &[&dyn ToSqlParam]) -> Result> { let (encoded, formats) = encode_params(params); let client = tcp_client(self.connection)?; @@ -121,7 +121,7 @@ impl<'conn> PreparedStatement<'conn> { /// - Returns [`Error::FeatureNotSupported`] on gRPC transport. /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`. - /// - Returns [`Error::Io`] on transport-level I/O failures. + /// - Returns [`Error::Connection`] on transport-level I/O failures. pub fn execute(&self, params: &[&dyn ToSqlParam]) -> Result { let (encoded, formats) = encode_params(params); let client = tcp_client(self.connection)?; From 23173ab37f6372879e9b4279d8f09ae9fe045ac0 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Fri, 9 Oct 2026 23:44:58 -0700 Subject: [PATCH 16/61] feat(api,core)!: mark SqlType and Error struct variants non_exhaustive SqlType, its struct variants (Numeric, Varchar, Char), the struct variants of hyperdb_api::Error and ColumnErrorKind::TypeMismatch are #[non_exhaustive], so a minor release can add a type or a field. Callers match with a wildcard, end field patterns with `..`, and construct via the existing constructors. Add SqlType::try_numeric. The Arrow decimal mapping built SqlType::Numeric by struct expression, which could yield an invalid NUMERIC (Decimal256 reaches precision 76); it now falls back to Text outside 1..=38. --- hyperdb-api-core/CHANGELOG.md | 3 ++ hyperdb-api-core/src/types/sql_type.rs | 28 +++++++++++++++++++ hyperdb-api/CHANGELOG.md | 11 ++++++++ hyperdb-api/src/arrow_result.rs | 13 ++++----- hyperdb-api/src/error.rs | 23 +++++++++------ hyperdb-api/tests/numeric_typemod_tests.rs | 16 ++++++++--- hyperdb-api/tests/remaining_features_tests.rs | 2 +- 7 files changed, 75 insertions(+), 21 deletions(-) diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index a202ae43..8b15ea57 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **`SqlType` and its struct variants (`Numeric`, `Varchar`, `Char`) are + `#[non_exhaustive]`.** `SqlType::try_numeric` is the non-panicking + counterpart of `SqlType::numeric` for untrusted precision and scale. - **`client::grpc::TransferMode` is now a crate-owned enum** (`Sync`, `Async`, `Adaptive`) with `From` conversions to the generated protobuf type, replacing the re-export of the prost type. `GrpcColumnInfo`'s `name` and `sql_type` diff --git a/hyperdb-api-core/src/types/sql_type.rs b/hyperdb-api-core/src/types/sql_type.rs index bdfa2252..2d926dd0 100644 --- a/hyperdb-api-core/src/types/sql_type.rs +++ b/hyperdb-api-core/src/types/sql_type.rs @@ -36,6 +36,11 @@ use super::special::Numeric; /// /// Each variant corresponds to a Hyper SQL type and has an associated internal OID. /// +/// The enum and its struct variants (`Numeric`, `Varchar`, `Char`) are +/// `#[non_exhaustive]`: match with a wildcard arm, end field patterns with +/// `..`, and build values with the constructors (for example +/// [`SqlType::numeric`]). +/// /// # Example /// /// ``` @@ -48,6 +53,7 @@ use super::special::Numeric; /// assert_eq!(numeric_type.to_string(), "NUMERIC(18, 2)"); /// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] pub enum SqlType { /// Boolean type. Bool, @@ -59,6 +65,7 @@ pub enum SqlType { #[default] Int, /// Numeric/decimal type with precision and scale. + #[non_exhaustive] Numeric { /// Total number of digits (1-38). precision: u32, @@ -76,11 +83,13 @@ pub enum SqlType { /// Variable-length text. Text, /// Variable-length character string with max length. + #[non_exhaustive] Varchar { /// Maximum length, or None for unlimited. max_length: Option, }, /// Fixed-length character string. + #[non_exhaustive] Char { /// Fixed length. length: u32, @@ -187,6 +196,25 @@ impl SqlType { SqlType::Numeric { precision, scale } } + /// Creates a Numeric type, or `None` if `precision` is outside + /// `1..=38` or `scale` exceeds `precision`. + /// + /// The non-panicking counterpart of [`SqlType::numeric`], for values that + /// come from outside the program (for example an Arrow `Decimal256` + /// schema). + /// + /// # Example + /// + /// ``` + /// use hyperdb_api_core::types::SqlType; + /// assert_eq!(SqlType::try_numeric(18, 2), Some(SqlType::numeric(18, 2))); + /// assert_eq!(SqlType::try_numeric(76, 2), None); + /// ``` + pub fn try_numeric(precision: u32, scale: u32) -> Option { + (precision >= 1 && precision <= u32::from(Numeric::MAX_PRECISION) && scale <= precision) + .then_some(SqlType::Numeric { precision, scale }) + } + /// Creates a Double type. /// /// # Example diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index afdafa38..2549e948 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **BREAKING: `SqlType` is `#[non_exhaustive]`, and so are the struct variants + of `SqlType` (`Numeric`, `Varchar`, `Char`) and of `Error` (`Connection`, + `Server`, `Closed`, `Cancelled`, `Column`, `ColumnIndexOutOfBounds`, + `Internal`) and `ColumnErrorKind::TypeMismatch`.** Match `SqlType` with a + wildcard arm, end any pattern that names variant fields with `..`, and build + these values with the constructors (`SqlType::numeric`, `Error::server`, ...) + instead of struct expressions. This lets a minor release add a type or a + field without a major bump. +- **Arrow `Decimal128` / `Decimal256` columns whose precision is outside + `1..=38` (or whose scale exceeds the precision) now map to `SqlType::Text`** + rather than an invalid `NUMERIC`, like other unmapped Arrow types. - **`QueryAs::new` and `QueryScalar::new` now take `&[&dyn ToSqlParam]`** instead of `&[&dyn Debug]`. These constructors are emitted by the `query_as!` / `query_scalar!` macros and are not meant to be called diff --git a/hyperdb-api/src/arrow_result.rs b/hyperdb-api/src/arrow_result.rs index da1ed738..34dc6ab8 100644 --- a/hyperdb-api/src/arrow_result.rs +++ b/hyperdb-api/src/arrow_result.rs @@ -819,14 +819,11 @@ pub(crate) fn arrow_type_to_sql_type(dt: &DataType) -> SqlType { DataType::Time32(_) | DataType::Time64(_) => SqlType::Time, DataType::Timestamp(_, None) => SqlType::Timestamp, DataType::Timestamp(_, Some(_)) => SqlType::TimestampTz, - DataType::Decimal128(p, s) => SqlType::Numeric { - precision: u32::from(*p), - scale: decimal_scale_to_u32(*s), - }, - DataType::Decimal256(p, s) => SqlType::Numeric { - precision: u32::from(*p), - scale: decimal_scale_to_u32(*s), - }, + // Hyper NUMERIC holds at most 38 digits; a wider (or malformed) + // decimal falls back to text like any other unmapped type. + DataType::Decimal128(p, s) | DataType::Decimal256(p, s) => { + SqlType::try_numeric(u32::from(*p), decimal_scale_to_u32(*s)).unwrap_or(SqlType::Text) + } DataType::Interval(_) => SqlType::Interval, _ => SqlType::Text, // Fallback to text for unknown types } diff --git a/hyperdb-api/src/error.rs b/hyperdb-api/src/error.rs index 8122a787..433f9b94 100644 --- a/hyperdb-api/src/error.rs +++ b/hyperdb-api/src/error.rs @@ -20,20 +20,19 @@ use thiserror::Error as ThisError; /// This enum is `#[non_exhaustive]`: new variants may be added in minor /// releases, so match arms must include a wildcard `_ =>` pattern. /// -/// Struct variants (`Connection`, `Server`, `Column`, -/// `ColumnIndexOutOfBounds`, `Internal`) cannot use Rust's -/// `#[non_exhaustive]` (E0639), so forward-compatibility for new fields -/// relies on construction via the provided constructors: +/// Every struct variant (`Connection`, `Server`, `Closed`, `Cancelled`, +/// `Column`, `ColumnIndexOutOfBounds`, `Internal`) is `#[non_exhaustive]` +/// too, so a field can be added in a minor release. Outside this crate you +/// cannot build them with struct-expression syntax (E0639) and a pattern +/// that names fields must end in `..`. Use the constructors: /// /// - [`Self::internal`] for [`Self::Internal`] /// - [`Self::connection`] / [`Self::connection_with_io`] for [`Self::Connection`] /// - [`Self::server`] for [`Self::Server`] /// - [`Self::column`] for [`Self::Column`] /// - [`Self::column_index_out_of_bounds`] for [`Self::ColumnIndexOutOfBounds`] -/// -/// Downstream code that uses struct-expression syntax for these -/// variants will fail to compile if a new field is added in a minor -/// release; using the constructors keeps callers source-compatible. +/// - [`Self::closed`] / [`Self::closed_with_sqlstate`] for [`Self::Closed`] +/// - [`Self::cancelled`] / [`Self::cancelled_with_sqlstate`] for [`Self::Cancelled`] #[derive(Debug, ThisError)] #[non_exhaustive] pub enum Error { @@ -51,6 +50,7 @@ pub enum Error { "connection error{}: {message}", sqlstate.as_ref().map(|s| format!(" ({s})")).unwrap_or_default(), )] + #[non_exhaustive] Connection { /// Human-readable description. message: String, @@ -82,6 +82,7 @@ pub enum Error { detail.as_ref().map(|d| format!("\nDETAIL: {d}")).unwrap_or_default(), hint.as_ref().map(|h| format!("\nHINT: {h}")).unwrap_or_default(), )] + #[non_exhaustive] Server { /// The 5-character `PostgreSQL` SQLSTATE code, if reported. sqlstate: Option, @@ -113,6 +114,7 @@ pub enum Error { "connection closed{}: {message}", sqlstate.as_ref().map(|s| format!(" ({s})")).unwrap_or_default(), )] + #[non_exhaustive] Closed { /// Human-readable description. message: String, @@ -131,6 +133,7 @@ pub enum Error { "operation cancelled{}: {message}", sqlstate.as_ref().map(|s| format!(" ({s})")).unwrap_or_default(), )] + #[non_exhaustive] Cancelled { /// Human-readable description. message: String, @@ -193,6 +196,7 @@ pub enum Error { /// by `FromRow` impls (through `RowAccessor`) and `Row::get_by_name` to /// signal which column failed and why. #[error("column {name}: {kind}")] + #[non_exhaustive] Column { /// The column name. name: String, @@ -205,6 +209,7 @@ pub enum Error { /// access; named access uses [`Self::Column`] with /// [`ColumnErrorKind::Missing`]. #[error("column index {idx} out of bounds (row has {column_count} columns)")] + #[non_exhaustive] ColumnIndexOutOfBounds { /// The requested 0-based column index. idx: usize, @@ -225,6 +230,7 @@ pub enum Error { /// /// Construct via [`Self::internal`]. #[error("internal error: {message}")] + #[non_exhaustive] Internal { /// Human-readable description of what invariant was violated. message: String, @@ -245,6 +251,7 @@ pub enum ColumnErrorKind { /// Column value could not be decoded as the target type. #[error("type mismatch: expected {expected}, got {actual}")] + #[non_exhaustive] TypeMismatch { /// Rust type name the caller asked for. expected: String, diff --git a/hyperdb-api/tests/numeric_typemod_tests.rs b/hyperdb-api/tests/numeric_typemod_tests.rs index d7aeaa7b..fccea995 100644 --- a/hyperdb-api/tests/numeric_typemod_tests.rs +++ b/hyperdb-api/tests/numeric_typemod_tests.rs @@ -59,7 +59,9 @@ fn avg_integer_schema_has_precision_16_scale_6() { assert_eq!(schema.column_count(), 1); let col = schema.column(0); match col.sql_type() { - SqlType::Numeric { precision, scale } => { + SqlType::Numeric { + precision, scale, .. + } => { assert_eq!( precision, 16, "AVG(INTEGER) precision should be 16 per Hyper's \ @@ -96,7 +98,9 @@ fn avg_bigint_schema_has_precision_25_scale_6() { let schema = result.schema().expect("schema after first chunk"); let col = schema.column(0); match col.sql_type() { - SqlType::Numeric { precision, scale } => { + SqlType::Numeric { + precision, scale, .. + } => { assert_eq!(precision, 25, "AVG(BIGINT) precision should be 25"); assert_eq!(scale, 6, "AVG(BIGINT) scale should be 6"); } @@ -149,7 +153,9 @@ fn avg_integer_via_row_get_numeric() { // And row.sql_type() exposes the column's SqlType from the schema. match row.sql_type(0) { - Some(SqlType::Numeric { precision, scale }) => { + Some(SqlType::Numeric { + precision, scale, .. + }) => { assert_eq!(precision, 16); assert_eq!(scale, 6); } @@ -225,7 +231,9 @@ fn explicit_numeric_column_preserves_precision_and_scale() { let _ = result.next_chunk().unwrap(); let schema = result.schema().expect("schema"); match schema.column(0).sql_type() { - SqlType::Numeric { precision, scale } => { + SqlType::Numeric { + precision, scale, .. + } => { assert_eq!(precision, 10); assert_eq!(scale, 2); } diff --git a/hyperdb-api/tests/remaining_features_tests.rs b/hyperdb-api/tests/remaining_features_tests.rs index fc34c940..1cdeeb4a 100644 --- a/hyperdb-api/tests/remaining_features_tests.rs +++ b/hyperdb-api/tests/remaining_features_tests.rs @@ -459,7 +459,7 @@ fn test_derive_from_row_missing_column_errors() { .expect_err("expected missing-column error"); match err { - Error::Column { name, kind } => { + Error::Column { name, kind, .. } => { assert_eq!(name, "not_in_query"); assert!( matches!(kind, ColumnErrorKind::Missing), From 624ce958e885b58cc78d6d1479de941bd75c435b Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 00:12:22 -0700 Subject: [PATCH 17/61] fix(api,core,derive)!: always quote identifiers so reserved-word tables work SqlIdentifier, format_table_name, TableDefinition::qualified_name (and the drop/staging SQL built on them) and derive(Table)'s CREATE_SQL left lowercase names bare, so tables or columns named order/user/table broke every COPY. Identifiers are now always double-quoted; generated SQL text changes but the resolved identifiers do not. Co-Authored-By: Claude Sonnet 5.5 --- hyperdb-api-core/CHANGELOG.md | 5 + hyperdb-api-core/docs/DEVELOPMENT-protocol.md | 9 +- hyperdb-api-core/src/protocol/escape.rs | 112 ++++++----------- hyperdb-api-derive/CHANGELOG.md | 4 + hyperdb-api-derive/README.md | 2 +- hyperdb-api-derive/src/table_derive.rs | 15 ++- .../ui/pass/derive_table_rename_field.rs | 6 +- hyperdb-api/CHANGELOG.md | 7 ++ hyperdb-api/src/table_definition.rs | 42 ++----- .../tests/compile_time_validation_tests.rs | 8 +- hyperdb-api/tests/reserved_word_tests.rs | 117 ++++++++++++++++++ 11 files changed, 203 insertions(+), 124 deletions(-) create mode 100644 hyperdb-api/tests/reserved_word_tests.rs diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index 8b15ea57..32c2c849 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -25,6 +25,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- `protocol::escape::SqlIdentifier`, `format_table_name` and `escape_identifier` + now always quote identifiers. They previously left lowercase names bare, so + reserved words such as `order` produced invalid SQL. Output changes from + `users` to `"users"`. + - **A cancelled COPY write could wedge the async connection.** Dropping a COPY write future mid-frame left a partial `CopyData` frame on the wire, so the queued `CopyFail` was read as part of that frame and diff --git a/hyperdb-api-core/docs/DEVELOPMENT-protocol.md b/hyperdb-api-core/docs/DEVELOPMENT-protocol.md index 87e21dbc..c8ae4854 100644 --- a/hyperdb-api-core/docs/DEVELOPMENT-protocol.md +++ b/hyperdb-api-core/docs/DEVELOPMENT-protocol.md @@ -187,11 +187,10 @@ This catches broken intra-doc links and missing docs (the crate enables functions), `ParseError` is in `types.rs` (used by type conversions). A future cleanup could unify them, but they are stable and the separation matches the module boundary. -- **`is_valid_unquoted_identifier` does not check reserved words.** The current - implementation checks syntax only (letter/underscore start, alphanumeric - body). SQL reserved words like `select` are not detected, so they will be - emitted unquoted. In practice this is harmless for Hyper's parser, but could - be tightened. +- **`is_valid_unquoted_identifier` does not check reserved words.** It checks + syntax only, so `select` or `order` pass. This is why `SqlIdentifier` quotes + unconditionally rather than consulting it: a bare reserved word is a syntax + error in Hyper, and `"users"` and `users` are the same identifier. --- diff --git a/hyperdb-api-core/src/protocol/escape.rs b/hyperdb-api-core/src/protocol/escape.rs index 6800aa6c..3c49a2a6 100644 --- a/hyperdb-api-core/src/protocol/escape.rs +++ b/hyperdb-api-core/src/protocol/escape.rs @@ -22,28 +22,23 @@ use std::fmt; /// A wrapper that ensures a SQL identifier is properly escaped when formatted. /// -/// This is a zero-cost abstraction that performs escaping lazily during formatting. -/// Identifiers are conditionally quoted: -/// - Simple lowercase identifiers (`users`, `my_table`) are not quoted -/// - Identifiers with uppercase letters are quoted to preserve case -/// - Identifiers with special characters are quoted +/// The identifier is **always** quoted, with embedded `"` doubled. Quoting +/// unconditionally is deliberate: the engine reserves keywords such as `order`, +/// `user` and `table`, and a bare reserved word is a syntax error (it broke +/// every `COPY` into such a table). Reserved words cannot be detected reliably +/// from the client, and for a plain lowercase name `"users"` and `users` are the +/// same relation, so the extra quotes never change meaning. /// /// # Example /// /// ``` /// use hyperdb_api_core::protocol::escape::SqlIdentifier; /// -/// // Simple identifiers are not quoted -/// assert_eq!(format!("{}", SqlIdentifier("users")), "users"); -/// assert_eq!(format!("{}", SqlIdentifier("my_table")), "my_table"); -/// -/// // Uppercase letters are quoted to preserve case +/// assert_eq!(format!("{}", SqlIdentifier("users")), "\"users\""); +/// // Reserved words survive +/// assert_eq!(format!("{}", SqlIdentifier("order")), "\"order\""); +/// // Case is preserved /// assert_eq!(format!("{}", SqlIdentifier("Segment")), "\"Segment\""); -/// -/// // Special characters require quoting -/// assert_eq!(format!("{}", SqlIdentifier("my-table")), "\"my-table\""); -/// assert_eq!(format!("{}", SqlIdentifier("my table")), "\"my table\""); -/// /// // Internal quotes are escaped /// assert_eq!(format!("{}", SqlIdentifier("my\"table")), "\"my\"\"table\""); /// ``` @@ -52,25 +47,7 @@ pub struct SqlIdentifier<'a>(pub &'a str); impl fmt::Display for SqlIdentifier<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Check if identifier needs quoting: - // 1. Not a valid unquoted identifier (has spaces, hyphens, etc.) - // 2. Contains uppercase letters (to preserve case - PostgreSQL case-folds unquoted identifiers) - let needs_quoting = - !is_valid_unquoted_identifier(self.0) || self.0.chars().any(char::is_uppercase); - - if needs_quoting { - f.write_str("\"")?; - for c in self.0.chars() { - if c == '"' { - f.write_str("\"\"")?; - } else { - write!(f, "{c}")?; - } - } - f.write_str("\"") - } else { - f.write_str(self.0) - } + fmt::Display::fmt(&QuotedIdentifier(self.0), f) } } @@ -144,10 +121,13 @@ pub fn is_valid_unquoted_identifier(s: &str) -> bool { /// ``` /// use hyperdb_api_core::protocol::escape::format_table_name; /// -/// assert_eq!(format_table_name(None, None, "users"), "users"); -/// assert_eq!(format_table_name(None, Some("public"), "users"), "public.users"); -/// assert_eq!(format_table_name(Some("mydb"), Some("public"), "users"), "mydb.public.users"); -/// assert_eq!(format_table_name(None, None, "my-table"), "\"my-table\""); +/// assert_eq!(format_table_name(None, None, "users"), "\"users\""); +/// assert_eq!(format_table_name(None, Some("public"), "users"), "\"public\".\"users\""); +/// assert_eq!( +/// format_table_name(Some("mydb"), Some("public"), "users"), +/// "\"mydb\".\"public\".\"users\"" +/// ); +/// assert_eq!(format_table_name(None, None, "order"), "\"order\""); /// ``` #[must_use] pub fn format_table_name(database: Option<&str>, schema: Option<&str>, table: &str) -> String { @@ -176,7 +156,7 @@ pub fn format_table_name(database: Option<&str>, schema: Option<&str>, table: &s /// ``` /// use hyperdb_api_core::protocol::escape::escape_identifier; /// -/// assert_eq!(escape_identifier("table"), "table"); +/// assert_eq!(escape_identifier("table"), "\"table\""); /// assert_eq!(escape_identifier("Segment"), "\"Segment\""); /// ``` #[must_use] @@ -186,22 +166,17 @@ pub fn escape_identifier(identifier: &str) -> String { /// A SQL identifier that is **always** quoted, whatever it contains. /// -/// [`SqlIdentifier`] omits the quotes when a name is already a legal bare -/// identifier, which is fine for display but unsafe for generated DDL: -/// [`is_valid_unquoted_identifier`] deliberately does not know the reserved -/// word list, so an all-lowercase keyword such as `select` or `order` passes -/// the check and is emitted bare, producing a syntax error. Quoting -/// unconditionally sidesteps the whole question — `"users"` and `users` name -/// the same relation, so the extra quotes never change meaning. -/// -/// Use this for any identifier written into SQL that the engine must parse. +/// Since [`SqlIdentifier`] now quotes unconditionally too, the two render +/// identically; this type remains as the explicit spelling of that intent. +/// `"users"` and `users` name the same relation, so the extra quotes never +/// change meaning, and reserved words such as `order` stay legal. /// /// # Example /// /// ``` /// use hyperdb_api_core::protocol::escape::QuotedIdentifier; /// -/// // Reserved words survive, where SqlIdentifier would emit them bare +/// // Reserved words survive /// assert_eq!(format!("{}", QuotedIdentifier("select")), "\"select\""); /// assert_eq!(format!("{}", QuotedIdentifier("users")), "\"users\""); /// // Internal quotes are doubled @@ -248,21 +223,12 @@ mod tests { #[test] fn test_sql_identifier_display() { - // Valid unquoted identifiers with only lowercase should not be quoted - assert_eq!(format!("{}", SqlIdentifier("table")), "table"); - assert_eq!(format!("{}", SqlIdentifier("my_table")), "my_table"); - assert_eq!(format!("{}", SqlIdentifier("table1")), "table1"); - assert_eq!(format!("{}", SqlIdentifier("_private")), "_private"); - assert_eq!(format!("{}", SqlIdentifier("my$var")), "my$var"); - - // Identifiers with uppercase letters should be quoted to preserve case + // Always quoted, including reserved words and plain lowercase names + assert_eq!(format!("{}", SqlIdentifier("table")), "\"table\""); + assert_eq!(format!("{}", SqlIdentifier("order")), "\"order\""); + assert_eq!(format!("{}", SqlIdentifier("my_table")), "\"my_table\""); assert_eq!(format!("{}", SqlIdentifier("Segment")), "\"Segment\""); - assert_eq!(format!("{}", SqlIdentifier("CustomerID")), "\"CustomerID\""); - assert_eq!(format!("{}", SqlIdentifier("Table")), "\"Table\""); - - // Invalid unquoted identifiers should be quoted assert_eq!(format!("{}", SqlIdentifier("my-table")), "\"my-table\""); - assert_eq!(format!("{}", SqlIdentifier("my table")), "\"my table\""); assert_eq!(format!("{}", SqlIdentifier("1table")), "\"1table\""); assert_eq!(format!("{}", SqlIdentifier("my\"table")), "\"my\"\"table\""); assert_eq!(format!("{}", SqlIdentifier("")), "\"\""); @@ -290,40 +256,36 @@ mod tests { #[test] fn test_format_table_name() { - assert_eq!(format_table_name(None, None, "users"), "users"); + assert_eq!(format_table_name(None, None, "users"), "\"users\""); + assert_eq!(format_table_name(None, None, "order"), "\"order\""); assert_eq!( format_table_name(None, Some("public"), "users"), - "public.users" + "\"public\".\"users\"" ); assert_eq!( format_table_name(Some("mydb"), Some("public"), "users"), - "mydb.public.users" + "\"mydb\".\"public\".\"users\"" ); - // Test with names that need quoting - assert_eq!(format_table_name(None, None, "my-table"), "\"my-table\""); assert_eq!( format_table_name(None, Some("my schema"), "users"), - "\"my schema\".users" + "\"my schema\".\"users\"" ); } #[test] fn test_sql_identifier_in_format() { - // Demonstrate zero-allocation composability - let table = "users"; - let column = "Customer ID"; let sql = format!( "SELECT {} FROM {}", - SqlIdentifier(column), - SqlIdentifier(table) + SqlIdentifier("Customer ID"), + SqlIdentifier("users") ); - assert_eq!(sql, "SELECT \"Customer ID\" FROM users"); + assert_eq!(sql, "SELECT \"Customer ID\" FROM \"users\""); } // Backward compat function tests #[test] fn test_escape_identifier() { - assert_eq!(escape_identifier("table"), "table"); + assert_eq!(escape_identifier("table"), "\"table\""); assert_eq!(escape_identifier("Segment"), "\"Segment\""); } diff --git a/hyperdb-api-derive/CHANGELOG.md b/hyperdb-api-derive/CHANGELOG.md index cc69af3c..172b4491 100644 --- a/hyperdb-api-derive/CHANGELOG.md +++ b/hyperdb-api-derive/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- `derive(Table)` now double-quotes the table and column names in `CREATE_SQL`, + so structs named or fields called after reserved words (`Order`, `User`, + `order`) produce valid DDL. `CREATE_SQL` text changes accordingly. + - **`query_as!` and `query_scalar!` now bind their `$N` arguments.** They were accepted and silently discarded. Each argument must implement `hyperdb_api::ToSqlParam`; the macros now generate code that requires it. diff --git a/hyperdb-api-derive/README.md b/hyperdb-api-derive/README.md index 47bfa58e..fd1b5d12 100644 --- a/hyperdb-api-derive/README.md +++ b/hyperdb-api-derive/README.md @@ -89,7 +89,7 @@ struct User { // Use the derived CREATE_SQL to create the table at runtime: conn.execute_command(User::CREATE_SQL)?; println!("{}", User::NAME); // "users" -println!("{}", User::CREATE_SQL); // "CREATE TABLE IF NOT EXISTS users (id BIGINT NOT NULL, ...)" +println!("{}", User::CREATE_SQL); // "CREATE TABLE IF NOT EXISTS \"users\" (\"id\" BIGINT NOT NULL, ...)" ``` ### Struct-level attributes diff --git a/hyperdb-api-derive/src/table_derive.rs b/hyperdb-api-derive/src/table_derive.rs index 21b0943c..c1c0a6bd 100644 --- a/hyperdb-api-derive/src/table_derive.rs +++ b/hyperdb-api-derive/src/table_derive.rs @@ -58,7 +58,7 @@ pub(crate) fn expand(input: &DeriveInput) -> syn::Result { let create_sql = format!( "CREATE TABLE IF NOT EXISTS {} ({})", - table_name, + quote_ident(&table_name), col_defs.join(", ") ); @@ -245,7 +245,18 @@ fn column_def(field: &Field, _table_name: &str) -> syn::Result { " NOT NULL" }; - Ok(format!("{col_name} {sql_type}{nullability}")) + Ok(format!( + "{} {sql_type}{nullability}", + quote_ident(&col_name) + )) +} + +/// Double-quote an identifier for the generated `CREATE TABLE`. +/// +/// Always quoting keeps reserved words (`order`, `user`, ...) legal; for a +/// plain lowercase name the quoted and bare forms are the same identifier. +fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) } /// Map a Rust type to a SQL type string. diff --git a/hyperdb-api-derive/tests/ui/pass/derive_table_rename_field.rs b/hyperdb-api-derive/tests/ui/pass/derive_table_rename_field.rs index 34eb7be0..1ef6fca8 100644 --- a/hyperdb-api-derive/tests/ui/pass/derive_table_rename_field.rs +++ b/hyperdb-api-derive/tests/ui/pass/derive_table_rename_field.rs @@ -9,9 +9,9 @@ struct User { } fn main() { - assert!(User::CREATE_SQL.contains("email_address")); + assert!(User::CREATE_SQL.contains("\"email_address\"")); // The original field name "email" must not appear as a standalone column identifier - assert!(!User::CREATE_SQL.contains("email BIGINT")); + assert!(!User::CREATE_SQL.contains("\"email\" BIGINT")); // "email_address" should be the column name for the renamed field - assert!(User::CREATE_SQL.contains("email_address TEXT NOT NULL")); + assert!(User::CREATE_SQL.contains("\"email_address\" TEXT NOT NULL")); } diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 2549e948..18de3351 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -82,6 +82,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- **Reserved-word table and column names** (`order`, `user`, `table`, ...) no + longer break generated SQL. `TableDefinition::qualified_name`, `table_name`, + `schema_name`, `database_name`, `to_drop_sql` and the `MappedInserter` + staging statements now always double-quote identifiers, so every inserter's + `COPY` works on such tables. The generated SQL text changes (`users` becomes + `"users"`); the identifiers resolved are the same. + - **`query_as!` / `query_scalar!` silently dropped their `$N` bind arguments.** The arguments were stringified and never sent, so any query with a placeholder failed at the server with "expected to have 0 diff --git a/hyperdb-api/src/table_definition.rs b/hyperdb-api/src/table_definition.rs index 51c1fc55..5a237f81 100644 --- a/hyperdb-api/src/table_definition.rs +++ b/hyperdb-api/src/table_definition.rs @@ -739,7 +739,7 @@ impl TableDefinition { /// use hyperdb_api::TableDefinition; /// /// let table = TableDefinition::new("Extract").with_schema("Extract"); - /// // "Extract" is quoted because it contains uppercase letters (to preserve case) + /// // Names are always quoted, preserving case and allowing reserved words /// assert_eq!(table.table_name(), "\"Extract\""); /// ``` #[must_use] @@ -763,36 +763,10 @@ impl TableDefinition { .map(|s| format!("{}", SqlIdentifier(s))) } - /// Returns the qualified table name with every part quoted. - /// - /// [`qualified_name`](Self::qualified_name) leaves a name bare when it is - /// already a legal unquoted identifier, which is not safe for generated - /// DDL: the underlying check does not know the reserved word list, so a - /// table reflected out of the catalog as `order` would be emitted bare and - /// rejected. Statements this type generates use this instead. - fn quoted_qualified_name(&self) -> String { - match (&self.database, &self.schema) { - (Some(db), Some(schema)) => format!( - "{}.{}.{}", - QuotedIdentifier(db), - QuotedIdentifier(schema), - QuotedIdentifier(&self.name) - ), - (None, Some(schema)) => format!( - "{}.{}", - QuotedIdentifier(schema), - QuotedIdentifier(&self.name) - ), - (Some(db), None) => { - format!("{}.{}", QuotedIdentifier(db), QuotedIdentifier(&self.name)) - } - (None, None) => format!("{}", QuotedIdentifier(&self.name)), - } - } - /// Returns the qualified table name (escaped). /// - /// Format: `database.schema.table` (if all parts are set, unquoted if valid identifiers) + /// Format: `database.schema.table`, with every part double-quoted so that + /// reserved words such as `order` and mixed-case names are valid SQL. #[must_use] pub fn qualified_name(&self) -> String { match (&self.database, &self.schema) { @@ -907,7 +881,7 @@ impl TableDefinition { }; sql.push_str(create_keyword); - sql.push_str(&self.quoted_qualified_name()); + sql.push_str(&self.qualified_name()); sql.push_str(" ("); for (i, col) in self.columns.iter().enumerate() { @@ -1048,21 +1022,21 @@ mod tests { let table = TableDefinition::new("users") .with_schema("public") .with_database("mydb"); - assert_eq!(table.qualified_name(), r"mydb.public.users"); + assert_eq!(table.qualified_name(), r#""mydb"."public"."users""#); } #[test] fn test_table_name() { let table = TableDefinition::new("Extract").with_schema("Extract"); - // "Extract" is quoted because it contains uppercase letters (to preserve case) + // Names are always quoted, preserving case and allowing reserved words assert_eq!(table.table_name(), r#""Extract""#); } #[test] fn test_drop_sql() { let table = TableDefinition::new("users"); - assert_eq!(table.to_drop_sql(true), r"DROP TABLE users"); - assert_eq!(table.to_drop_sql(false), r"DROP TABLE IF EXISTS users"); + assert_eq!(table.to_drop_sql(true), r#"DROP TABLE "users""#); + assert_eq!(table.to_drop_sql(false), r#"DROP TABLE IF EXISTS "users""#); } #[test] diff --git a/hyperdb-api/tests/compile_time_validation_tests.rs b/hyperdb-api/tests/compile_time_validation_tests.rs index 9742f632..b433a0e5 100644 --- a/hyperdb-api/tests/compile_time_validation_tests.rs +++ b/hyperdb-api/tests/compile_time_validation_tests.rs @@ -55,21 +55,21 @@ fn table_derive_creates_correct_sql() { "CREATE_SQL must contain table name" ); assert!( - CtUser::CREATE_SQL.contains("id BIGINT"), + CtUser::CREATE_SQL.contains("\"id\" BIGINT"), "i64 maps to BIGINT" ); assert!( - CtUser::CREATE_SQL.contains("name TEXT"), + CtUser::CREATE_SQL.contains("\"name\" TEXT"), "String maps to TEXT" ); // score is Option → nullable DOUBLE PRECISION assert!( - CtUser::CREATE_SQL.contains("score DOUBLE PRECISION"), + CtUser::CREATE_SQL.contains("\"score\" DOUBLE PRECISION"), "f64 maps to DOUBLE PRECISION" ); // Option → no NOT NULL constraint assert!( - !CtUser::CREATE_SQL.contains("score DOUBLE PRECISION NOT NULL"), + !CtUser::CREATE_SQL.contains("\"score\" DOUBLE PRECISION NOT NULL"), "Option must not have NOT NULL" ); assert_eq!(CtUser::NAME, "ct_users"); diff --git a/hyperdb-api/tests/reserved_word_tests.rs b/hyperdb-api/tests/reserved_word_tests.rs new file mode 100644 index 00000000..ca3a97bc --- /dev/null +++ b/hyperdb-api/tests/reserved_word_tests.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Tables and columns named after SQL reserved words (`order`, `user`, ...) +//! must work through every generated statement: CREATE, COPY, the mapped +//! INSERT...SELECT, DROP, and `derive(Table)`'s `CREATE_SQL`. + +mod common; +use common::{TestConnection, test_hyper_params, test_result_path}; + +use hyperdb_api::{ + AsyncConnection, AsyncInserter, Catalog, ColumnMapping, CreateMode, HyperProcess, Inserter, + SqlType, Table, TableDefinition, +}; +use hyperdb_api_derive::Table; + +fn reserved_def(table: &str) -> TableDefinition { + TableDefinition::new(table) + .add_required_column("order", SqlType::int()) + .add_nullable_column("user", SqlType::text()) +} + +fn count(test: &TestConnection, table: &str) -> i64 { + test.connection + .fetch_scalar::(&format!("SELECT COUNT(*) FROM \"{table}\"")) + .expect("count") +} + +#[test] +fn sync_inserter_into_reserved_word_table() { + let test = TestConnection::new().expect("connection"); + let def = reserved_def("order"); + Catalog::new(&test.connection) + .create_table(&def) + .expect("create"); + + let mut ins = Inserter::new(&test.connection, &def).expect("inserter"); + ins.add_row(&[&1i32, &"a"]).expect("row"); + ins.add_row(&[&2i32, &"b"]).expect("row"); + assert_eq!(ins.execute().expect("COPY into \"order\""), 2); + assert_eq!(count(&test, "order"), 2); + + Catalog::new(&test.connection) + .drop_table("order") + .expect("drop"); +} + +#[test] +fn drop_sql_quotes_reserved_word() { + let def = reserved_def("user"); + assert_eq!(def.to_drop_sql(true), r#"DROP TABLE "user""#); + assert_eq!(def.qualified_name(), r#""user""#); +} + +#[test] +fn mapped_inserter_into_reserved_word_table() { + let test = TestConnection::new().expect("connection"); + let target = reserved_def("table"); + Catalog::new(&test.connection) + .create_table(&target) + .expect("create"); + + let stage = TableDefinition::new("_stage") + .add_required_column("order", SqlType::int()) + .add_nullable_column("user", SqlType::text()); + let mappings = vec![ColumnMapping::new("order"), ColumnMapping::new("user")]; + let mut ins = Inserter::with_column_mappings(&test.connection, &stage, "table", &mappings) + .expect("mapped inserter"); + ins.add_row(&[&7i32, &"x"]).expect("row"); + assert_eq!(ins.execute().expect("mapped insert"), 1); + assert_eq!(count(&test, "table"), 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn async_inserter_into_reserved_word_table() { + let name = "reserved_word_async"; + let db_path = test_result_path(name, "hyper").expect("path"); + let params = test_hyper_params(name).expect("params"); + let hyper = HyperProcess::new(None, Some(¶ms)).expect("hyperd"); + let endpoint = hyper.require_endpoint().expect("endpoint").to_string(); + let conn = AsyncConnection::connect( + &endpoint, + db_path.to_str().expect("utf8"), + CreateMode::CreateAndReplace, + ) + .await + .expect("connect"); + + let def = reserved_def("select"); + conn.execute_command(&def.to_create_sql(true).expect("ddl")) + .await + .expect("create"); + let mut ins = AsyncInserter::new(&conn, &def).expect("inserter"); + ins.add_i32(1).unwrap(); + ins.add_str("a").unwrap(); + ins.end_row().await.expect("row"); + assert_eq!(ins.execute().await.expect("COPY into \"select\""), 1); +} + +#[derive(Debug, Table)] +#[allow(dead_code, reason = "only the derived consts are used")] +#[hyperdb(table = "group")] +struct Reserved { + order: i64, + #[hyperdb(rename = "user")] + who: Option, +} + +#[test] +fn derive_table_create_sql_quotes_reserved_words() { + let test = TestConnection::new().expect("connection"); + assert_eq!(Reserved::NAME, "group"); + test.connection + .execute_command(Reserved::CREATE_SQL) + .expect("derive CREATE_SQL with reserved names must parse"); + assert_eq!(count(&test, "group"), 0); +} From 90f35127aaf4b8ce032e094424910986a4963f53 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 00:20:35 -0700 Subject: [PATCH 18/61] fix(api)!: tie PreparedStatement::query rowset to the statement borrow The rowset holds the connection lock while streaming; dropping its statement first closes it on the same connection and deadlocks. Return Rowset<'s> / AsyncRowset<'s> so the borrow checker rejects it, with a compile_fail doctest. Co-Authored-By: Claude Sonnet 5.5 --- hyperdb-api/CHANGELOG.md | 5 +++++ hyperdb-api/src/async_prepared.rs | 2 +- hyperdb-api/src/prepared.rs | 20 +++++++++++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 18de3351..2c18411d 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **Breaking:** `PreparedStatement::query` and `AsyncPreparedStatement::query` + now return a rowset borrowing the *statement* (`Rowset<'s>` / + `AsyncRowset<'s>`), not just the connection. Previously a rowset could + outlive its statement, and dropping the statement then deadlocked on the + connection lock the rowset still held. The compiler now rejects that code. - **BREAKING: `SqlType` is `#[non_exhaustive]`, and so are the struct variants of `SqlType` (`Numeric`, `Varchar`, `Char`) and of `Error` (`Connection`, `Server`, `Closed`, `Cancelled`, `Column`, `ColumnIndexOutOfBounds`, diff --git a/hyperdb-api/src/async_prepared.rs b/hyperdb-api/src/async_prepared.rs index 9366895f..628fa63d 100644 --- a/hyperdb-api/src/async_prepared.rs +++ b/hyperdb-api/src/async_prepared.rs @@ -84,7 +84,7 @@ impl<'conn> AsyncPreparedStatement<'conn> { /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`. /// - Returns [`Error::Connection`] on transport-level I/O failures. - pub async fn query(&self, params: &[&dyn ToSqlParam]) -> Result> { + pub async fn query<'s>(&'s self, params: &[&dyn ToSqlParam]) -> Result> { let (encoded, formats) = encode_params(params); let client = async_tcp_client(self.connection)?; let stream = client diff --git a/hyperdb-api/src/prepared.rs b/hyperdb-api/src/prepared.rs index fd06e941..02ef3599 100644 --- a/hyperdb-api/src/prepared.rs +++ b/hyperdb-api/src/prepared.rs @@ -94,6 +94,24 @@ impl<'conn> PreparedStatement<'conn> { /// the prepared-statement equivalent of /// [`Connection::execute_query`]. /// + /// The returned rowset borrows the statement, not just the connection: + /// it holds the connection lock while streaming, and dropping the + /// statement closes it on that same connection, which would deadlock. + /// The borrow checker therefore rejects a rowset that outlives its + /// statement: + /// + /// ```compile_fail + /// # use hyperdb_api::{Connection, Result}; + /// # fn demo(conn: &Connection) -> Result<()> { + /// let rows = { + /// let stmt = conn.prepare("SELECT 1")?; + /// stmt.query(&[])? + /// }; // error[E0597]: `stmt` does not live long enough + /// # drop(rows); + /// # Ok(()) + /// # } + /// ``` + /// /// # Errors /// /// - Returns [`Error::FeatureNotSupported`] if the underlying [`Connection`] is on @@ -101,7 +119,7 @@ impl<'conn> PreparedStatement<'conn> { /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute` (type mismatch, runtime error while streaming). /// - Returns [`Error::Connection`] on transport-level I/O failures. - pub fn query(&self, params: &[&dyn ToSqlParam]) -> Result> { + pub fn query<'s>(&'s self, params: &[&dyn ToSqlParam]) -> Result> { let (encoded, formats) = encode_params(params); let client = tcp_client(self.connection)?; let stream = client.execute_streaming_with_formats( From 1eb49d717ec8f453b1f4f47b51108b1358962858 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 00:55:42 -0700 Subject: [PATCH 19/61] feat(api)!: move to arrow/parquet 60 and re-export arrow, chrono, geo_types Bump arrow and parquet 59 -> 60 (MSRV 1.88 verified) before 1.0 freezes the public API, re-export the three crates whose types appear in it, and state the policy: a major bump of a re-exported crate is a hyperdb-api major. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 79 ++++++++++++++++++------------- docs/RUST_GUIDELINES.md | 14 +++--- hyperdb-api-core/Cargo.toml | 4 +- hyperdb-api-salesforce/Cargo.toml | 2 +- hyperdb-api/CHANGELOG.md | 7 +++ hyperdb-api/Cargo.toml | 6 ++- hyperdb-api/README.md | 12 +++++ hyperdb-api/src/lib.rs | 7 +++ hyperdb-compile-check/Cargo.lock | 67 ++++++++++++++++---------- hyperdb-mcp/Cargo.toml | 4 +- 10 files changed, 132 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8928b8e2..629d2cff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "arrow" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" +checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14" dependencies = [ "arrow-arith", "arrow-array", @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" +checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720" dependencies = [ "arrow-array", "arrow-buffer", @@ -142,9 +142,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" dependencies = [ "ahash", "arrow-buffer", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" dependencies = [ "bytes", "half", @@ -172,9 +172,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" +checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -191,11 +191,22 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cmp" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-csv" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" +checksum = "2ad83176cc677f996096d694a3cccd7310069faeee4b3df9e2dabf92fdcf8b4b" dependencies = [ "arrow-array", "arrow-cast", @@ -208,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -221,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +246,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" +checksum = "342fc6600a93a48038d853431803c941436b7b51f756763e76cad9784b7e3718" dependencies = [ "arrow-array", "arrow-buffer", @@ -260,12 +271,13 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" +checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", @@ -273,9 +285,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" +checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a" dependencies = [ "arrow-array", "arrow-buffer", @@ -286,19 +298,20 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" [[package]] name = "arrow-select" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -306,9 +319,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" +checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1" dependencies = [ "arrow-array", "arrow-buffer", @@ -395,9 +408,9 @@ dependencies = [ [[package]] name = "atoi" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" dependencies = [ "num-traits", ] @@ -1792,9 +1805,11 @@ dependencies = [ "arrow", "async-stream", "bytes", + "chrono", "deadpool", "futures", "futures-core", + "geo-types", "hyperdb-api-core", "hyperdb-api-derive", "libc", @@ -2788,9 +2803,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" +checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba" dependencies = [ "ahash", "arrow-array", diff --git a/docs/RUST_GUIDELINES.md b/docs/RUST_GUIDELINES.md index 11660cdb..0c1501aa 100644 --- a/docs/RUST_GUIDELINES.md +++ b/docs/RUST_GUIDELINES.md @@ -108,16 +108,18 @@ review; point to them when requesting changes. parameters, look for a helper type. - **M-DONT-LEAK-TYPES.** Prefer `std` types in public APIs. Third-party types (`bytes::Bytes`, `arrow::RecordBatch`, …) are only exposed when they - materially improve the API over an equivalent in `std`. Note upstream also - sanctions leaking "behind a relevant feature flag" — the option this crate - cannot currently take, since `hyperdb-api` has no features. Revisit the - unconditional `arrow` / `chrono` / `geo-types` leaks when feature flags - land post-1.0.0. + materially improve the API over an equivalent in `std`. The 1.0 policy keeps + the `arrow` / `chrono` / `geo-types` leaks, re-exports those crates from the + `hyperdb-api` root so callers use the exact version we build against, and + treats a major bump of any of them as a `hyperdb-api` major (README, "Semver + and re-exported crates"). A default-off feature cannot decouple the cadence + and moving existing capability behind one would itself be breaking, so none + is planned. - **M-FEATURES-ADDITIVE.** Any feature added must be purely additive: it must not disable or modify a public item, must not require another feature to be manually enabled, and every combination must compile. Prefer a `std` feature over a `no-std` one. Currently near-vacuous here (`hyperdb-api` has - no features), but load-bearing for the planned post-1.0.0 feature work — + no features), but load-bearing for any feature added later — and note that moving *existing* always-on capability behind a default-off feature is a breaking change, while default-on is not. diff --git a/hyperdb-api-core/Cargo.toml b/hyperdb-api-core/Cargo.toml index 32fa9252..c4b2382c 100644 --- a/hyperdb-api-core/Cargo.toml +++ b/hyperdb-api-core/Cargo.toml @@ -70,7 +70,7 @@ hyperdb-api-salesforce = { path = "../hyperdb-api-salesforce", version = "=1.0.0 # x-release-please-end # Arrow parsing for catalog operations (optional, used by authenticated_client) -arrow = { version = "59.0", optional = true } +arrow = { version = "60", optional = true } [lints] workspace = true @@ -82,7 +82,7 @@ tonic-prost-build = { workspace = true } [dev-dependencies] tracing-subscriber = { version = "0.3", features = ["env-filter"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util"] } -arrow = "59.0" +arrow = "60" # For integration tests - we need HyperProcess to start a server hyperdb-api = { path = "../hyperdb-api" } tempfile = { workspace = true } diff --git a/hyperdb-api-salesforce/Cargo.toml b/hyperdb-api-salesforce/Cargo.toml index 4d484172..4635deff 100644 --- a/hyperdb-api-salesforce/Cargo.toml +++ b/hyperdb-api-salesforce/Cargo.toml @@ -33,7 +33,7 @@ workspace = true [dev-dependencies] rand = { workspace = true } hyperdb-api-core = { path = "../hyperdb-api-core", features = ["salesforce-auth"] } -arrow = { version = "59.0", default-features = false, features = ["ipc"] } +arrow = { version = "60", default-features = false, features = ["ipc"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } tokio = { version = "1", features = ["full"] } diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 2c18411d..7c92f6a7 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **BREAKING:** the `arrow` dependency moved from **59** to **60**. Arrow types + appear in this crate's public API, so consumers must move to `arrow` 60 in + lockstep. No source change was needed on our side. `parquet` (used by + `hyperdb-mcp`) moved to 60 with it. +- `hyperdb-api` now re-exports `arrow`, `chrono` and `geo_types` from the crate + root, and the README states the policy: a major bump of a re-exported crate + is a `hyperdb-api` major. - **Breaking:** `PreparedStatement::query` and `AsyncPreparedStatement::query` now return a rowset borrowing the *statement* (`Rowset<'s>` / `AsyncRowset<'s>`), not just the connection. Previously a rowset could diff --git a/hyperdb-api/Cargo.toml b/hyperdb-api/Cargo.toml index 24093ad7..a62cbc12 100644 --- a/hyperdb-api/Cargo.toml +++ b/hyperdb-api/Cargo.toml @@ -25,7 +25,11 @@ bytes = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } # Arrow for result parsing (used by both TCP and gRPC transports) -arrow = { version = "59", default-features = false, features = ["ipc"] } +arrow = { version = "60", default-features = false, features = ["ipc"] } +# chrono / geo-types appear in the public API and are re-exported from the +# crate root (see lib.rs); versions track hyperdb-api-core's. +chrono = { workspace = true } +geo-types = { workspace = true } # SmallVec for memory-efficient collections that are typically small smallvec = "1.15" # Deadpool - async connection pool. diff --git a/hyperdb-api/README.md b/hyperdb-api/README.md index 84835d4e..f234a816 100644 --- a/hyperdb-api/README.md +++ b/hyperdb-api/README.md @@ -21,6 +21,18 @@ Hyper database files (`.hyper`) without any C library dependencies. hyperdb-api = "1.0" ``` +### Semver and re-exported crates + +`arrow`, `chrono` and `geo_types` types appear in this crate's public API +(`ArrowInserter`, `ArrowRowset`, `Timestamp` conversions, `Geography`), so +`hyperdb-api` re-exports all three crates from its root. Use +`hyperdb_api::arrow` (and friends), or depend on exactly the same major +yourself: two majors in one binary give two incompatible `RecordBatch` types. + +A **major** version bump of any re-exported crate is a **major** version bump +of `hyperdb-api`. This release builds against `arrow` 60, `chrono` 0.4 and +`geo-types` 0.7. + ## Runtime Requirements The `hyperd` executable (Hyper database server) must be available. Set its path via: diff --git a/hyperdb-api/src/lib.rs b/hyperdb-api/src/lib.rs index 2805759e..79caf2e9 100644 --- a/hyperdb-api/src/lib.rs +++ b/hyperdb-api/src/lib.rs @@ -201,6 +201,13 @@ mod grpc_connection; #[cfg(kani)] mod proofs; +// Third-party crates whose types appear in this crate's public API. Re-exported +// so callers use exactly the version we are built against; a major bump of any +// of them is a `hyperdb-api` major (see README, "Semver and re-exported crates"). +pub use arrow; +pub use chrono; +pub use geo_types; + pub use arrow_inserter::ArrowInserter; pub use arrow_reader::ArrowReader; pub use arrow_result::{ diff --git a/hyperdb-compile-check/Cargo.lock b/hyperdb-compile-check/Cargo.lock index 012c40bd..da24a569 100644 --- a/hyperdb-compile-check/Cargo.lock +++ b/hyperdb-compile-check/Cargo.lock @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "arrow" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" +checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14" dependencies = [ "arrow-arith", "arrow-array", @@ -70,9 +70,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" +checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720" dependencies = [ "arrow-array", "arrow-buffer", @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" dependencies = [ "ahash", "arrow-buffer", @@ -102,9 +102,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" dependencies = [ "bytes", "half", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" +checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -133,11 +133,22 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cmp" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-data" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -148,9 +159,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2" dependencies = [ "arrow-array", "arrow-buffer", @@ -162,12 +173,13 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" +checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", @@ -175,9 +187,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" +checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a" dependencies = [ "arrow-array", "arrow-buffer", @@ -188,19 +200,20 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" [[package]] name = "arrow-select" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -208,9 +221,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.3.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" +checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1" dependencies = [ "arrow-array", "arrow-buffer", @@ -258,9 +271,9 @@ dependencies = [ [[package]] name = "atoi" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" dependencies = [ "num-traits", ] @@ -868,8 +881,10 @@ dependencies = [ "arrow", "async-stream", "bytes", + "chrono", "deadpool", "futures-core", + "geo-types", "hyperdb-api-core", "serde", "serde_json", diff --git a/hyperdb-mcp/Cargo.toml b/hyperdb-mcp/Cargo.toml index fced895e..4adda1ca 100644 --- a/hyperdb-mcp/Cargo.toml +++ b/hyperdb-mcp/Cargo.toml @@ -33,8 +33,8 @@ tracing = { workspace = true } tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-appender = "0.2" csv = "1.4" -arrow = { version = "59", default-features = false, features = ["ipc", "json", "csv"] } -parquet = { version = "59", default-features = false, features = ["arrow"] } +arrow = { version = "60", default-features = false, features = ["ipc", "json", "csv"] } +parquet = { version = "60", default-features = false, features = ["arrow"] } schemars = "1.2" thiserror = { workspace = true } chrono = { workspace = true } From 5be7ae2fa30807037230d96bfdd253ca37e71053 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 01:01:46 -0700 Subject: [PATCH 20/61] refactor(api): name the prepared-statement borrow lifetime 'stmt Co-Authored-By: Claude Sonnet 5.5 --- hyperdb-api/src/async_prepared.rs | 5 ++++- hyperdb-api/src/prepared.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hyperdb-api/src/async_prepared.rs b/hyperdb-api/src/async_prepared.rs index 628fa63d..947d27de 100644 --- a/hyperdb-api/src/async_prepared.rs +++ b/hyperdb-api/src/async_prepared.rs @@ -84,7 +84,10 @@ impl<'conn> AsyncPreparedStatement<'conn> { /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute`. /// - Returns [`Error::Connection`] on transport-level I/O failures. - pub async fn query<'s>(&'s self, params: &[&dyn ToSqlParam]) -> Result> { + pub async fn query<'stmt>( + &'stmt self, + params: &[&dyn ToSqlParam], + ) -> Result> { let (encoded, formats) = encode_params(params); let client = async_tcp_client(self.connection)?; let stream = client diff --git a/hyperdb-api/src/prepared.rs b/hyperdb-api/src/prepared.rs index 02ef3599..5a7061a0 100644 --- a/hyperdb-api/src/prepared.rs +++ b/hyperdb-api/src/prepared.rs @@ -119,7 +119,7 @@ impl<'conn> PreparedStatement<'conn> { /// - Returns [`Error::Server`] if the server rejects `Bind` or /// `Execute` (type mismatch, runtime error while streaming). /// - Returns [`Error::Connection`] on transport-level I/O failures. - pub fn query<'s>(&'s self, params: &[&dyn ToSqlParam]) -> Result> { + pub fn query<'stmt>(&'stmt self, params: &[&dyn ToSqlParam]) -> Result> { let (encoded, formats) = encode_params(params); let client = tcp_client(self.connection)?; let stream = client.execute_streaming_with_formats( From 90f2fac0d486662cf51cf70bf32b008216457406 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 02:40:48 -0700 Subject: [PATCH 21/61] feat(core)!: TLS negotiation module with sslmode verifiers and TLS cancel sender Replace the unused `client::tls` module (pub-field TlsConfig, async-only rustls_impl, never called) with the building blocks for PG-wire TLS on TCP: - TlsMode with the libpq sslmode names (Disable default, Prefer, Require, VerifyCa, VerifyFull), FromStr/Display, and ParseTlsModeError. - TlsConfig with private fields and a consuming builder (root_cert, client_cert, server_name); From. - Crate-private TlsConnector: chain-only and signature-only verifiers, webpki fallback roots for VerifyFull, mTLS, IPv6 host handling. - negotiate_sync/negotiate_async: SSLRequest, one-byte reply read off the raw socket, Prefer falls back only on 'N'. The sync handshake is driven read_tls/write_tls at a time so the timeout is a deadline for the whole negotiation (complete_io keeps reading while a peer drips bytes). - send_cancel_sync/send_cancel_async: cancel over TLS, never plaintext, close_notify and a bounded drain. - frontend::ssl_request; client::Error::Tls, with from_io classifying wrapped rustls errors as Tls; mapped to hyperdb_api::Error::Tls. - Config::with_tls/tls; Config::from_str now returns client::Error and rejects the libpq ssl* keys. - hyperdb_api re-exports TlsConfig, TlsMode, ParseTlsModeError. Fake-server unit tests cover every mode, fallback, mTLS, timeouts and both cancel senders. The old tests/tls_tests.rs is removed. Clients and builders are wired up in the next change. BREAKING CHANGE: client::tls API replaced (TlsMode::VerifyCA renamed to VerifyCa, TlsConfig fields private, rustls_impl removed); client::Error gains a Tls variant; Config's FromStr error type is now client::Error. --- .../plans/2026-10-10-pg-wire-tls.md | 332 ++++ .../specs/2026-10-10-pg-wire-tls-design.md | 298 +++ hyperdb-api-core/src/client/config.rs | 76 +- hyperdb-api-core/src/client/error.rs | 52 +- hyperdb-api-core/src/client/tls.rs | 1742 ++++++++++++++--- .../src/protocol/message/frontend.rs | 36 + hyperdb-api-core/tests/tls_tests.rs | 412 ---- hyperdb-api/src/error.rs | 19 +- hyperdb-api/src/lib.rs | 2 + 9 files changed, 2316 insertions(+), 653 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-10-pg-wire-tls.md create mode 100644 docs/superpowers/specs/2026-10-10-pg-wire-tls-design.md delete mode 100644 hyperdb-api-core/tests/tls_tests.rs diff --git a/docs/superpowers/plans/2026-10-10-pg-wire-tls.md b/docs/superpowers/plans/2026-10-10-pg-wire-tls.md new file mode 100644 index 00000000..49346614 --- /dev/null +++ b/docs/superpowers/plans/2026-10-10-pg-wire-tls.md @@ -0,0 +1,332 @@ +# PG-wire TLS Implementation Plan + +**Goal:** Implement TLS for PG-wire TCP connections (libpq `sslmode` names, sqlx/tokio-postgres behaviour where libpq differs) in the core sync and async clients, cancel requests, the `hyperdb-api` builders and pools, and the Node bindings, and make the docs match. + +**Spec:** [`docs/superpowers/specs/2026-10-10-pg-wire-tls-design.md`](../specs/2026-10-10-pg-wire-tls-design.md). It holds the API, the mode table, the divergences from libpq, the negotiation rules, and the measured hyperd facts. Read it first and do not re-derive any of it. + +**Revision:** this is the plan after review. Two reviewers checked it against the code and against the rustls 0.23.45 and tokio-rustls 0.26.6 sources. Each verdict was "with fixes", with no Critical findings, and every fix is folded in below. + +**Tech stack:** rustls 0.23 (`ring`, `std`, `tls12`; already a core dependency), tokio-rustls 0.26, webpki-roots 1.0, rcgen 0.14 (core dev-dependency; to be added to `hyperdb-api` dev-dependencies), napi-rs. + +## Global constraints + +- **Engine and settings.** + - Use the pinned hyperd only, with `HYPERD_PATH="$PWD/.hyperd/current"` as an absolute path. + - Start TLS servers through `Parameters::set` with `"ssl_key"`, `"ssl_certificate"` and `"ssl_force"`. These hidden settings are defined in `hyper-db/hyper/tools/hyperd/hyperd.cpp`. Use no others. +- **Code rules.** + - No narrowing integer `as` casts (AGENTS.md reminder 7). + - Library code returns errors; it does not panic. + - Lifetimes get descriptive names, never `'s`. +- **Commits.** + - Stage files with an explicit `git add `. Never stage `.agents/` or `.codex/`. + - Use Conventional Commits, with `!` on the breaking iterations. +- **MSRV 1.88.** `cargo +1.88 check --workspace --all-targets` must stay clean. +- **Gate for every Rust iteration:** + - `cargo fmt --all --check` + - `cargo clippy --workspace --all-targets --all-features -- -D warnings` + - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace` + - `cargo test -p hyperdb-api-core --doc` (the rewritten module example must compile) + - the iteration's tests, with real output captured. +- **Dead-code lint.** It is on and enforced through `-D warnings`. Crate-private items that have no caller until a later iteration get `#[expect(dead_code, reason = "wired in iteration N")]`. The iteration that adds the caller removes the `expect`, because an unfulfilled `expect` is itself an error. + +## Iteration 1: core TLS module, fake-server proven (no client wiring yet) + +**Files:** + +- `hyperdb-api-core/src/client/tls.rs` (rewrite) +- `hyperdb-api-core/src/protocol/message/frontend.rs` (`ssl_request`) +- `hyperdb-api-core/src/client/error.rs` (the `Tls` variant, `from_io` classification, and the message-accessor match at about line 251) +- `hyperdb-api/src/error.rs` (the mapping, and the `Tls` doc wording) +- `hyperdb-api-core/src/client/config.rs` (`with_tls` and `tls`, plus rejecting libpq TLS keys) +- `hyperdb-api-core/tests/tls_tests.rs` (deleted; its cases move into unit tests) + +**Steps:** + +1. **`ssl_request`.** Add `frontend::ssl_request(buf)`, which writes `00 00 00 08 04 D2 16 2F`. Unit-test those exact bytes next to `cancel_request`'s test. + +2. **Core `Error::Tls`.** + - Add `client::Error::Tls(String)` with a `tls()` constructor. Handle it in every exhaustive match in core (including the accessor near `error.rs:251`). + - Map it to `hyperdb_api::Error::Tls(chain)`. + - Reword the `hyperdb_api::Error::Tls` doc to "TLS negotiation, handshake or certificate-verification failure". + - In `Error::from_io`, check `err.get_ref().and_then(|e| e.downcast_ref::())`. When the inner error is a rustls error the result is `Error::Tls`; otherwise it stays `Io` as today. + - Add a unit test for each mapping. + +3. **`TlsMode`, `ParseTlsModeError` and `TlsConfig`** (per the spec). + - `TlsMode` is `#[non_exhaustive]` with `Default` = `Disable`, and implements `FromStr` and `Display` with the libpq strings. + - `FromStr` has `type Err = ParseTlsModeError`. That is a public struct implementing `Display` and `std::error::Error`; its message names the five valid values. + - The `TlsMode` rustdoc includes the short divergences table. + - `TlsConfig` has private fields and the setters `new`, `root_cert`, `client_cert` and `server_name`. The path setters take `impl AsRef` and store a `PathBuf`. + - `mode()` is public. The other getters are `pub(crate)`. + - Implement `From for TlsConfig`. + +4. **`TlsConnector`** (crate-private). + + ```rust + pub(crate) struct TlsConnector { config: Arc, server_name: ServerName<'static> } + impl TlsConnector { + /// `Ok(None)` for `Disable`. + pub(crate) fn build(tls: &TlsConfig, host: &str) -> Result>>; + } + ``` + + - **PEM loading.** Use `rustls::pki_types::pem::PemObject`: `CertificateDer::pem_file_iter` and `PrivateKeyDer::from_pem_file`. A load error is `Error::Config` naming the file. A file that loads but contains zero certificates is `Error::Config` too. + - **Verifiers, per the spec table:** + - `NoVerify` delegates `verify_tls12_signature`, `verify_tls13_signature` and `supported_verify_schemes` to `provider.signature_verification_algorithms`. + - `ChainOnly { roots, provider }` implements `verify_server_cert` by calling `rustls::client::verify_server_cert_signed_by_trust_anchor(&ParsedCertificate::try_from(end_entity)?, &roots, intermediates, now, algs.all)`. It does not match on name errors. Signature methods delegate as above. + - `VerifyFull` uses the stock `WebPkiServerVerifier::builder_with_provider(roots, provider)`. + - Roots are `root_cert` if it is set, and otherwise (`VerifyFull` only) `webpki_roots::TLS_SERVER_ROOTS`. + - `VerifyCa` without `root_cert` is `Error::Config`. + - Client auth uses `with_client_auth_cert` when `client_cert` is set, and `with_no_client_auth` otherwise. + - **Server name.** Take the `server_name` override, or else `host` with `[`/`]` stripped, and pass it to `ServerName::try_from(..).to_owned()`. Invalid input is `Error::Config`. + - `#[expect(dead_code)]` on anything not yet called outside tests. + +5. **Negotiation and cancel senders** (crate-private, in `tls.rs`). + + ```rust + pub(crate) enum Fallback { AllowPlaintext, Deny } + pub(crate) enum Negotiated { Plain(Plain), Tls(Tls) } + pub(crate) fn negotiate_sync(tcp: TcpStream, c: &TlsConnector, fallback: Fallback, timeout: Duration) + -> Result>>; + pub(crate) async fn negotiate_async(tcp: tokio::net::TcpStream, c: &TlsConnector, fallback: Fallback, timeout: Duration) + -> Result>>; + pub(crate) fn send_cancel_sync(addr: &str, process_id: i32, secret_key: i32, tls: Option<&TlsConnector>) -> Result<()>; + pub(crate) async fn send_cancel_async(addr: &str, process_id: i32, secret_key: i32, tls: Option<&TlsConnector>) -> Result<()>; + ``` + + - **Negotiation.** + - Sync: `set_read_timeout` and `set_write_timeout` to `timeout`, write the SSLRequest, then `read_exact(&mut [0u8; 1])` on the bare socket. + - On `'S'`, loop `while conn.is_handshaking() { conn.complete_io(&mut tcp)?; }`, then clear the timeouts and return `StreamOwned::new`. + - On `'N'`, use `Fallback`: plaintext when it is allowed, `Error::Tls("server does not support TLS")` otherwise. + - Any other byte is `Error::Protocol`. + - Async: the whole negotiation runs under `tokio::time::timeout(timeout, ..)`, and an elapsed timeout is `Error::Timeout`. + - Error classification follows `from_io` (rustls errors become `Tls`; I/O errors and timeouts stay `Io`/`Timeout`). + - **Cancel senders.** + - Match the existing cancel code exactly for socket setup: connect, then `set_nodelay`. + - With `tls` set, negotiate with `Fallback::Deny` and a timeout of `CANCEL_TLS_TIMEOUT` (10 s). Write the CancelRequest (`frontend::cancel_request`) and flush. + - Then send `close_notify`. Sync: `conn.send_close_notify()`, `complete_io`, then `shutdown(Write)`. Async: `shutdown().await`. + - Then drain to EOF with a 1 s timeout and ignore the outcome. + - Without `tls`, the plaintext behaviour is the same as today. The async plaintext path gains a `flush().await`. + - The sync sender must not need a tokio runtime, because it runs from `Drop`. + +6. **`Config` changes.** + - Add `Config::with_tls(impl Into)` and `Config::tls() -> &TlsConfig`, defaulting to `Disable`. The docs say TCP only, `Prefer` is plaintext over IPC, and the other modes are rejected over IPC. + - `Config::from_str` rejects the keys `sslmode`, `sslrootcert`, `sslcert`, `sslkey`, `sslpassword`, `sslcrl`, `sslsni` and `sslnegotiation` with `Error::Config("TLS options are not accepted in the connection string; use ConnectionBuilder::tls")`. Unit-test it. + +7. **Remove the old API.** Delete the public `rustls_impl` and the old helpers. The new module doc example compiles (it is not `ignore`) and has no `TODO`. + +8. **Fake-server unit tests** (`#[cfg(test)] mod tests` in `tls.rs`; rcgen is already a core dev-dependency). + - The harness is a `std::net::TcpListener` thread. It reads 8 bytes and asserts they are an SSLRequest, replies `S`, `N` or `E`, and on `S` runs a rustls `ServerConnection` with an optional client verifier. It reports the first 16 decrypted bytes (or the plaintext bytes after `N`) back over a channel. + - Cases: + - **Require:** + - with an unknown CA and no root cert: OK; + - with the wrong root cert: `Tls` (chain-only applies under Require too). + - **VerifyCa:** + - right CA: OK; + - wrong CA: `Tls`; + - wrong hostname: OK; + - no root cert: `Config`. + - **VerifyFull:** + - right host: OK; + - wrong hostname: `Tls`; + - matching `server_name` override: OK; + - IP host `127.0.0.1` with an IP SAN: OK. + - **Prefer:** + - against `S` with a wrong CA: `Tls`, not plaintext (no downgrade); + - against `N`: plaintext. + - **Server replies `N` or `E`:** + - Require against `N`: `Tls`; + - `E`: `Protocol`. + - **Disable:** `TlsConnector::build` returns `None`. + - **mTLS:** + - server requires a client cert and one from the CA is offered: OK; + - none offered: `Tls` on the first read after the handshake (TLS 1.3 delivers the alert late). Drive one read. + - **Timeout:** a server that accepts the connection and never answers makes `negotiate_sync` fail within the timeout. Use 500 ms in the test. + - **`send_cancel_sync` against `S`:** + - the server observes the SSLRequest, then a handshake, then a decrypted CancelRequest with code 80877102 and the right pid and secret; + - against `N`, the result is `Tls` and the server receives nothing after the SSLRequest. + - **`send_cancel_async`:** the same pair of cases. + - **`negotiate_async`:** one Require case and one VerifyFull case. + - **Parsing:** `TlsMode` parse and display round-trip; an unknown string gives `ParseTlsModeError`, whose message names all five valid modes. + - **PEM files:** a missing file or an empty one is `Config`. + - **`Config::from_str("h:1/db?sslmode=require")`** is `Config`. + - Delete `hyperdb-api-core/tests/tls_tests.rs` once its cases exist here. + +**Acceptance:** + +- The gate passes. +- `cargo test -p hyperdb-api-core --lib` passes with its output captured. +- No client behaviour changes yet, except two that are intended: `sslmode=` in a connection string is now rejected, and rustls I/O errors are classified as TLS. + +**Commit:** `feat(core)!: TLS negotiation module with sslmode verifiers and TLS cancel sender` + +## Iteration 2: core clients and builders, sync and async + +**Files:** + +- core: `sync_stream.rs`, `async_stream.rs`, `client.rs`, `async_client.rs` +- `hyperdb-api`: `connection_builder.rs`, `async_connection_builder.rs`, `connection.rs`, `async_connection.rs`, `transport.rs` (as needed for `is_tls`), `lib.rs`, `Cargo.toml` (rcgen dev-dependency) +- new: `tests/tls_tests.rs` + +**Steps:** + +1. **Stream types.** + - Add `SyncStream::Tls(Box>)`, delegating `Read` and `Write`. + - The Tls `Read` arm maps `io::ErrorKind::UnexpectedEof` to `Ok(0)`; see the spec for why. + - Socket-option methods go through `.sock`. + - Delete the unused `try_clone`. + - Add `AsyncStream::Tls(Box>)`, delegating `poll_read` (with the same `UnexpectedEof` → EOF mapping, i.e. return `Ready(Ok(()))` with nothing filled), `poll_write`, `poll_write_vectored`, `is_write_vectored`, `poll_flush` and `poll_shutdown`. + - Add `is_tls()` on both. +2. **TCP connect** (`Client::connect`, `AsyncClient::connect`). + - Apply the socket options as today. + - If `TlsConnector::build(config.tls(), config.host())?` is `Some(c)`, call `negotiate_*` with `Fallback::AllowPlaintext` only when the mode is `Prefer`, and with `config.connect_timeout()`. + - Wrap the result in the matching stream variant. + - Store `tls: Option>` only when TLS was negotiated. + - Add `is_tls()` accessors. + - Remove iteration 1's `expect(dead_code)` on items that now have callers. +3. **IPC.** In `connect_unix` and `connect_named_pipe` (sync and async; this also covers `connect_endpoint`'s IPC branches), return `FeatureNotSupported("TLS is only available over TCP; use a host:port endpoint")` when the mode is Require, VerifyCa or VerifyFull. Prefer and Disable connect in plaintext. +4. **Cancel.** The TCP branches of `Client::cancel` and `AsyncClient::cancel_sync` call `send_cancel_sync(addr, pid, secret, self.tls.as_deref())`. `AsyncClient::cancel` calls `send_cancel_async`. The IPC branches are unchanged. +5. **`hyperdb-api` wiring.** + - Re-export `TlsConfig`, `TlsMode` and `ParseTlsModeError` at the crate root. + - Both builders get a `tls: TlsConfig` field and `pub fn tls(mut self, tls: impl Into) -> Self`. Its docs cover: + - TCP only; + - IPC behaviour; + - gRPC is rejected; + - a short form of the modes table; + - the static `connect*` shortcuts and `Connection::new(&hyper, ..)` stay plaintext. + - `build_tcp`, `build_unix` and `build_named_pipe` set `config.with_tls(self.tls.clone())`. + - `build_grpc` checks TLS **first**, before connecting and next to the `query_timeout` check: any mode other than `Disable` returns `FeatureNotSupported("TLS for gRPC is selected by the https:// scheme")`. + - `Connection::is_tls()` and `AsyncConnection::is_tls()` return `true` when the TCP client negotiated TLS or the gRPC config `is_tls()`. +6. **Test fixture** (a module inside `tls_tests.rs`). + - rcgen generates a CA, a server leaf (SANs `localhost` and `127.0.0.1`), a client leaf from the same CA, and a second CA with its own client leaf, all in a `TempDir`. + - The server certificate file is `leaf PEM + CA PEM` (that file is also hyperd's client trust store). + - On Unix only, `chmod 0600` the key. + - `fn tls_hyper(name: &str, force: bool) -> (HyperProcess, Fixture)` starts from `common::test_hyper_params(name)` (check its exact shape), adds the ssl parameters, and sets `TransportMode::Tcp` explicitly. +7. **Tests** in `hyperdb-api/tests/tls_tests.rs`: + - **Enforcement control.** A plaintext builder against a `force` server fails, and the error mentions SSL. + - **Require, sync and async.** + - `is_tls()` is true. + - Create a table, insert 100k rows (`Inserter`; async `AsyncInserter` or `AsyncArrowInserter`), then `SELECT` all of them across more than one chunk and check the count. + - **VerifyCa:** right CA OK; wrong CA `Tls`. + - **VerifyFull:** + - `localhost` OK, and `127.0.0.1:PORT` OK; + - `server_name("wrong.example")` gives `Tls`, while VerifyCa with the same override is OK. + - **VerifyCa without a root cert** is `Config`. + - **Prefer:** + - against a TLS server, `is_tls()` is true; + - against a plain `HyperProcess`, the connection works and `!is_tls()`; + - with the wrong CA against a TLS server, `Tls`. + - **Require against a non-TLS server** is `Tls`. + - **mTLS:** a client cert from the CA is OK; one from the other CA gives `Tls`. The rejection may arrive during startup; assert on the variant. + - **Cancel under TLS + force, sync and async.** + - Follow `connection_tests.rs:280-370`. + - First, in this iteration, measure the query: `SELECT count(*) FROM generate_series(1,200000) a(i), generate_series(1,200000) b(i) WHERE a.i + b.i > 3`. Keep at least a 5× margin over the cancel delay, and grow the series if it is fast. + - Cancel after about 500 ms and assert SQLSTATE 57014 strictly. + - A comment explains that hyperd also accepts a plaintext cancel, so the proof that the cancel goes over TLS is the fake-server test from iteration 1; this test proves the TLS cancel path works end to end. + - **gRPC:** an `http://localhost:1` endpoint with `.tls(TlsMode::Require)` gives `FeatureNotSupported`, with no server needed. + - **Unix only:** a UDS endpoint (`HyperProcess` with `TransportMode::Ipc`) with Require gives `FeatureNotSupported`; with Prefer it connects and `!is_tls()`. + - **Red-before-green.** Stash the `with_tls` line in `build_tcp`. Require against the force server must then fail (plaintext rejected). Capture both runs. + +**Acceptance:** + +- The gate passes. +- `cargo test -p hyperdb-api --test tls_tests`, `-p hyperdb-api-core`, and `-p hyperdb-api --test connection_tests --test async_connection_tests` (check the file names) pass, with output captured. + +**Commit:** `feat(api,core)!: TLS for TCP connections via the builders' tls(), cancel over TLS` + +## Iteration 3: pools + +**Files:** `hyperdb-api/src/pool.rs` and `hyperdb-api/tests/tls_tests.rs`. + +**Steps:** + +1. Add `pub tls: TlsConfig` (default `Disable`) to `PoolConfig` and `SyncPoolConfig`, with a `pub fn tls(mut self, impl Into) -> Self`. Include it in their `Debug`. +2. Route both `open`s through the builders **unconditionally**: + - set endpoint, database and mode; + - set user and password only when **both** are `Some`, as today; + - pass `.tls(self.config.tls.clone())`. + This gives one code path. The async pool with auth and a non-TCP endpoint used to fail through `connect_with_auth`; this fixes it as a side effect, so add a test if it is cheap. +3. Tests: + - A sync pool and an async pool with Require against a force server: check out two connections, run a query on each, and check `is_tls()` on both. + - The existing `pool_tests` must stay green. + +**Acceptance:** the gate passes, and `pool_tests` and `tls_tests` pass. + +**Commit:** `feat(api)!: TLS option on PoolConfig and SyncPoolConfig; pools connect through the builders` + +## Iteration 4: Node bindings + +**Files:** `hyperdb-api-node/src/types.rs`, `src/connection.rs`, `src/process.rs`, `index.d.ts`, `__test__/smoke.mjs`, `README.md`, and `docs/NODEJS_API_SUMMARY.md`. + +**Steps:** + +1. Add a `#[napi(string_enum)] pub enum TlsMode { Disable, Prefer, Require, VerifyCa, VerifyFull }` and implement `From for hyperdb_api::TlsMode`. +2. Builder: + - Fields: `tls_mode: Option`, `tls_root_cert`, `tls_client_cert: Option<(String, String)>` and `tls_server_name`. + - Setters: `tlsMode`, `tlsRootCert`, `tlsClientCert(certPath, keyPath)` and `tlsServerName`, following the existing `&mut self -> &Self` pattern. + - `build()`: when a sub-option is set without `tlsMode`, return an error. When a mode is set, assemble a `TlsConfig`. +3. Add a `#[napi(getter)] is_tls`, which is `isTls` in JS. +4. `HyperProcess`: change the constructor to `new(hyper_path: Option, options: Option)`. `#[napi(object)] HyperProcessOptions { parameters: Option> }` is applied with `Parameters::set`. Existing one-argument calls keep working. +5. Update `index.d.ts` (hand-written): the `TlsMode` const enum in CreateMode style, the builder methods, the `isTls` getter, `HyperProcessOptions`, and the constructor. +6. Smoke test section: + - Locate `openssl` and run `openssl version`. If it is missing, skip with a printed note, but **fail** when `process.env.CI` is set. + - Generate a CA, then a leaf signed by it with `basicConstraints=critical,CA:FALSE`, `extendedKeyUsage=serverAuth` and `subjectAltName=DNS:localhost,IP:127.0.0.1`, using an extfile (portable across OpenSSL and LibreSSL, so no `-addext`). + - Write the cert file as leaf plus CA, `chmod 0600` the key, and start `new HyperProcess(undefined, { parameters: { ssl_key, ssl_certificate, ssl_force: 'true' } })`. + - Plaintext `Connection.connect` fails. + - Require: `conn.isTls === true`, and a query works. + - VerifyFull with `tlsRootCert(ca)`: OK. + - `tlsRootCert` without `tlsMode`: `build()` rejects. +7. Run `npm run build:debug && npm test` in `hyperdb-api-node/` with an absolute `HYPERD_PATH` and capture the output. This audit effort has never run the Node suite, so record any pre-existing failure unrelated to TLS separately instead of fixing it here. + +**Acceptance:** the Node build and `npm test` pass with output captured, and `index.d.ts` agrees with the napi exports (spot-check names). + +**Commit:** `feat(node): TLS options on ConnectionBuilder, isTls, HyperProcess options` + +## Iteration 5: docs, changelogs, MCP note, full run + +**Steps:** + +1. Fix the doc over-claims at the spec's listed lines. Then run `rg -in '\btls\b|\bssl' --glob '*.md' --glob '*.rs' --glob '*.d.ts'` and fix any remaining claim. + - **README.md:** the TCP/TLS line. + - **AGENTS.md:** lines 62 and 123. + - **Core:** `lib.rs:14` and `client/mod.rs:24,256-259`. + - **DEVELOPMENT.md:** lines 42, 92, 271-281, 800 and 890. + - **`hyperdb-api-core/docs/DEVELOPMENT-client.md`:** + - rewrite `## TLS Internals` (179-189) for the new module; + - rewrite `### TLS Tests` (233-255) for the new unit and integration tests; + - fix 300-301. + - **`docs/NODEJS_API_SUMMARY.md`:** lines 28 and 67. + - **Node README:** the builder section. +2. Per-crate CHANGELOG `## [Unreleased]`, merged into existing headings (MD024): + - **`hyperdb-api-core`:** + - Added: negotiation, `Config::with_tls`, `ParseTlsModeError`, `Error::Tls`. The new variant breaks exhaustive matches, because core's error type is deliberately not `#[non_exhaustive]`. + - Changed: `TlsMode::VerifyCA` → `VerifyCa`; `TlsConfig::new(mode)` and private fields; `Config::from_str` rejects `ssl*` keys; rustls I/O errors are classified as `Tls`. + - Removed: `rustls_impl`, `danger_accept_invalid_certs`, `has_client_cert`, `verify_server`, the `TlsMode` helper methods, `SyncStream::try_clone`. + - **`hyperdb-api`:** + - Added: `tls()` on the builders and pools, `is_tls`, the re-exports. + - Changed: the new `tls` field on `PoolConfig` and `SyncPoolConfig` breaks struct-literal construction; pools connect through the builders; Require, VerifyCa and VerifyFull over IPC are `FeatureNotSupported`. + - **`hyperdb-api-node`:** Added. The `HyperProcess` options expose hyperd parameters to JS, mirroring the public Rust `Parameters::set`. +3. MCP deferral comment at the `AttachSource` "Future: Tcp" note in `hyperdb-mcp/src/attach.rs`: a future `kind:"tcp"` must accept `tls_mode`, `tls_root_cert` (plus the client cert and server name) and pass them to `ConnectionBuilder::tls`. This is a comment only; the tool surface does not change. +4. Run `npx markdownlint-cli2` and judge its findings against the pre-existing backlog. +5. Full gate: + - fmt, clippy, doc, doctests; + - `cargo +1.88 check --workspace --all-targets`; + - `HYPERD_PATH="$PWD/.hyperd/current" cargo test --workspace --no-fail-fast`, with the `test result` lines summed. +6. Update ssteiner-ai `STATUS.md` and **do not commit it**. Record: + - decision 8 done, with the commits; + - the MCP deferral; + - the pre-existing finding that `Config::connect_timeout` and `login_timeout` are not applied to the TCP connect. + +**Commit:** `docs: describe the real TLS support; changelogs; MCP TLS note` + +## Review checkpoints + +- **After each iteration:** a `feature-dev:code-reviewer` pass on the diff (pasted inline), briefed with the spec and that iteration's acceptance criteria. +- **Final:** `feature-dev:code-reviewer` and `code-review` in parallel on the whole range. Focus: + - the single-byte read; + - `Fallback` use: Prefer only on connect, never on cancel; + - the verifier per mode; + - the cancel close sequence and timeouts, including `cancel_sync` from `Drop`; + - error classification; + - the IPC rejection; + - no narrowing casts; + - doc accuracy. diff --git a/docs/superpowers/specs/2026-10-10-pg-wire-tls-design.md b/docs/superpowers/specs/2026-10-10-pg-wire-tls-design.md new file mode 100644 index 00000000..697d8302 --- /dev/null +++ b/docs/superpowers/specs/2026-10-10-pg-wire-tls-design.md @@ -0,0 +1,298 @@ +# PG-wire TLS for TCP connections — design + +Status: approved direction (pre-1.0 audit decision 8, finding `A1:X-sec-a-02`). +Revised after plan review by two reviewers. Breaking changes are acceptable: +this lands before 1.0.0. + +## Problem + +The README, AGENTS.md, the core crate docs and the Node summary all advertise +TLS for the TCP transport. None of it exists. `hyperdb-api-core/src/client/tls.rs` +defines `TlsConfig`, `TlsMode` and an async-only `rustls_impl::create_connector`, +but nothing calls them: no client sends an `SSLRequest`, `Config` has no TLS +field, and no builder exposes one. Every TCP connection is plaintext, including +the password and the cancel secret. + +## Goals + +- Real TLS on PG-wire **TCP** connections, sync and async. The modes use the + libpq `sslmode` names; where libpq and sqlx/tokio-postgres differ, behaviour + follows sqlx/tokio-postgres (see [Divergences from libpq](#divergences-from-libpq)). +- Reachable from `ConnectionBuilder`, `AsyncConnectionBuilder`, both pools, and + the Node `ConnectionBuilder`. +- Cancel requests travel over TLS when the connection does, and never fall back + to plaintext. +- Tests against the real pinned `hyperd` (0.0.26479) with a TLS-enabled server, + plus fake-server unit tests that observe what goes on the wire. +- Remove every over-claim from the docs. + +## Non-goals + +- **MCP.** `hyperdb-mcp` has no remote-endpoint tool (`attach_database` accepts + only `kind:"local_file"`). A comment at the `AttachSource` "Future: Tcp" note in + `hyperdb-mcp/src/attach.rs` records that a future `kind:"tcp"` must accept the + TLS options. No MCP tool-surface change. +- **gRPC.** gRPC TLS is chosen by the `https://` scheme (tonic) and is unchanged. + Setting a non-`Disable` mode on a gRPC endpoint is `Error::FeatureNotSupported`. +- **TLS over Unix sockets / named pipes.** Not offered. `Prefer` is satisfied by + plaintext there; `Require`, `VerifyCa` and `VerifyFull` are + `Error::FeatureNotSupported` instead of silently connecting in plaintext (a + named pipe can be remote over SMB). hyperd's named-pipe listener also hangs + after answering `'S'`. +- **Static `connect*` shortcuts.** `Connection::new(&hyper, ..)`, + `Connection::connect*`, `AsyncConnection::connect*` and `transport.rs`'s + `connect_tcp*` stay plaintext. TLS goes through the builders and pools only; + their docs say so, and say that a server started with `ssl_force` is only + reachable through them. +- `sslmode=` and friends in connection strings. `Config::from_str` **rejects** + the libpq TLS keys with an error pointing at `.tls(..)` rather than passing + them to the server as startup options. +- In-memory PEM, CRLs, OS trust store, `sslnegotiation=direct`. `TlsConfig` has + private fields, so any of these can be added without a break. +- `Config::connect_timeout` is not applied to the TCP `connect()` today + (pre-existing; recorded separately). This change only applies it to the TLS + negotiation. + +## Engine facts (measured on hyperd 0.0.26479, and read from hyper-db source) + +- Enabled by `--ssl-key=PATH --ssl-certificate=PATH`, optionally + `--ssl-force=true`. Through `Parameters::set` these are `ssl_key`, + `ssl_certificate`, `ssl_force` (`_` maps to `-`). They are not listed in + `hyperd --help`; they are defined in `hyper/tools/hyperd/hyperd.cpp`. +- On POSIX the key file must not be group/other accessible (mode 0600). The + check is POSIX-only (`hyperd.cpp:667`), so Windows has no permission + requirement. Cert and key must match and be currently valid. +- `SSLRequest` (code 80877103) is answered with one byte: `'S'` and a TLS + handshake, or `'N'` when no certificate is configured. +- TLS 1.3 only; a TLS 1.2-only client gets alert 70. +- With `ssl_force`, a plaintext TCP startup is rejected ("only SSL connections + are allowed"), loopback included. This is how tests prove enforcement. +- A `CancelRequest` is accepted both plaintext and after `SSLRequest`+TLS, with + or without `ssl_force` (both return 57014). So hyperd cannot tell the tests + which one we sent; the fake-server unit tests do. +- Client certificates: hyperd loads the server certificate file as its client + trust store with `verify_peer` (`hyperd.cpp:684-688`). A client certificate, + if offered, must chain to a CA in that file, or the connection fails. No + certificate is fine. So mTLS is testable against hyperd when the certificate + file holds `leaf + CA`. +- `HyperProcess` reports `localhost:PORT` and does not itself open a client + connection, so `ssl_force` does not break startup. A certificate with SANs + `localhost` and `127.0.0.1` satisfies `VerifyFull` against both host forms. + +## Public API + +### Core (`hyperdb_api_core::client::tls`), re-exported from `hyperdb_api` + +```rust +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum TlsMode { + #[default] + Disable, + Prefer, + Require, + VerifyCa, // renamed from VerifyCA + VerifyFull, +} +// FromStr (Err = ParseTlsModeError) / Display: +// "disable" | "prefer" | "require" | "verify-ca" | "verify-full" + +/// Public error type for `TlsMode::from_str` (core's `client::Error` must not +/// leak through `hyperdb_api`). Display names the five valid values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseTlsModeError { /* private */ } + +#[derive(Debug, Clone, Default)] +pub struct TlsConfig { /* private: mode, root_cert, client_cert, server_name */ } + +impl TlsConfig { + pub fn new(mode: TlsMode) -> Self; + pub fn root_cert(self, pem_path: impl AsRef) -> Self; // sslrootcert + pub fn client_cert(self, cert: impl AsRef, key: impl AsRef) -> Self; // sslcert/sslkey + pub fn server_name(self, name: impl Into) -> Self; // override host for SNI + VerifyFull + pub fn mode(&self) -> TlsMode; + // other getters pub(crate): no external consumer; making them public later is additive +} +impl From for TlsConfig { .. } +``` + +Removed: `TlsConfig`'s pub fields, `verify_server`, `danger_accept_invalid_certs`, +`ca_cert` (now `root_cert`), `has_client_cert`, the `TlsMode` helper methods +encoding the old meaning, and the public `rustls_impl` module. + +### Builders and pools (`hyperdb_api`) + +- `ConnectionBuilder::tls(impl Into)`, + `AsyncConnectionBuilder::tls(..)`, `PoolConfig::tls(..)`, + `SyncPoolConfig::tls(..)`. So `.tls(TlsMode::Require)` works. +- `Connection::is_tls()` / `AsyncConnection::is_tls()` — `true` when the + session is encrypted: PG-wire TLS was negotiated, or the gRPC endpoint is + `https://`. With `Prefer`, this is how a caller learns the outcome. +- Re-exports: `hyperdb_api::{TlsConfig, TlsMode, ParseTlsModeError}`. +- Core `Config::with_tls(impl Into)` / `Config::tls()`. +- Core `client::Error::Tls(String)`, mapped to the existing + `hyperdb_api::Error::Tls(String)`, whose doc becomes "TLS negotiation, + handshake or certificate-verification failure". Unreadable PEM files and + invalid option combinations are `Error::Config`. + +### Node (`hyperdb-api-node`) + +- `#[napi(string_enum)] TlsMode { Disable, Prefer, Require, VerifyCa, VerifyFull }`. +- `ConnectionBuilder.tlsMode(mode)`, `.tlsRootCert(path)`, + `.tlsClientCert(certPath, keyPath)`, `.tlsServerName(name)`. Setting a + sub-option without `tlsMode` is an error at `build()` (otherwise it would be + silently dropped as `Disable`). +- `Connection.isTls` getter (like `isAlive`). +- `new HyperProcess(hyperPath?, options?)` with + `options.parameters?: Record`, applied through + `Parameters::set`, so a test (or user) can start a TLS-enabled server. +- `index.d.ts` (hand-written) updated to match. + +## Semantics + +| Mode | SSLRequest | `'N'` reply | Server certificate check | +|---|---|---|---| +| `Disable` (default) | not sent | n/a | n/a | +| `Prefer` | sent | plaintext on the same socket | none, or chain-only if `root_cert` is set | +| `Require` | sent | `Error::Tls` | none, or chain-only if `root_cert` is set | +| `VerifyCa` | sent | `Error::Tls` | chain against `root_cert` (**required**: `Error::Config` without it) | +| `VerifyFull` | sent | `Error::Tls` | chain + hostname, against `root_cert` or else the bundled webpki (Mozilla) roots | + +- When `root_cert` is given it is the **only** trust anchor. +- "None" verification still checks handshake signatures with the provider's + algorithms; it only skips identity and chain. +- Chain-only verification calls + `rustls::client::verify_server_cert_signed_by_trust_anchor` directly (the same + chain step `WebPkiServerVerifier` runs before its name check), rather than + pattern-matching name errors. +- `Prefer` falls back only on `'N'`. A handshake or verification failure is an + error, not a downgrade. +- Hostname for SNI and `VerifyFull` = `server_name` override, else the `Config` + host with IPv6 brackets stripped. IP hosts become `ServerName::IpAddress` (no + SNI, IP SAN check). +- A client certificate (mTLS) is offered in any enabled mode when configured. +- Crypto provider: rustls `ring` (the workspace's provider), safe default + protocol versions (hyperd negotiates 1.3). + +### Divergences from libpq + +| Area | libpq | Here | Why | +|---|---|---|---| +| Default mode | `prefer` | `Disable` | hyperd is usually a local engine with TLS off; an SSLRequest round trip on every connect buys nothing there. | +| `Prefer` on handshake/verification failure | retries without TLS | error | Same as sqlx/tokio-postgres; a downgrade after a failed handshake is exactly what an attacker wants. | +| `VerifyFull` without a root cert | error unless `sslrootcert=system` (PG16+) | bundled webpki roots | Matches the PG16 `system` rule's intent; OS-installed corporate CAs are not included, use `root_cert`. | +| Default cert files (`~/.postgresql/root.crt`, ...) | read | not read | Explicit configuration only. | +| TLS keys in connection strings | accepted | rejected with a pointer to `.tls()` | Keeps one configuration path. | +| `Require`/`Verify*` over a local socket | ignored | `FeatureNotSupported` | A Windows named pipe can be remote; silent plaintext is worse than an error. | + +The `TlsMode` rustdoc carries a short form of this table. + +## Negotiation + +On a freshly connected TCP socket (nodelay, 4 MiB buffers and keepalive +applied first, as today), before `StartupMessage`: + +1. Set read/write timeouts on the socket: `Config::connect_timeout` for a + connection, a fixed `CANCEL_TLS_TIMEOUT` (10 s) for a cancel. Cleared after a + successful connection handshake; the cancel socket keeps them. +2. Write the 8-byte SSLRequest (`frontend::ssl_request`). +3. Read **exactly one byte** from the raw socket, with no buffered reader in + between, so nothing the server sends after the reply can be taken as + plaintext (CVE-2021-23214 class). `RawConnection` is only built afterwards. +4. `'S'`: run the handshake. Sync: `rustls::ClientConnection`, driving + `complete_io` until `is_handshaking()` is false, then `StreamOwned`. Async: + `tokio_rustls::TlsConnector::connect`. +5. `'N'`: plaintext on the same socket only if the caller allows it + (`Fallback::AllowPlaintext`, used by `Prefer` connections); otherwise + `Error::Tls`. Cancel always passes `Fallback::Deny`. +6. Any other byte (e.g. `'E'`) is `Error::Protocol`. + +Error classification: `io::Error`s that wrap a `rustls::Error` (rustls reports +handshake and alert failures that way) become `Error::Tls`, everywhere I/O +errors are converted — not only during negotiation. That matters because under +TLS 1.3 the server's rejection of a *client* certificate arrives after the +client considers the handshake complete, on the first read during startup. +Server-certificate errors surface at connect; client-certificate rejection +surfaces during startup, still as `Error::Tls`. Plain I/O failures (reset, +timeout) stay connection/I-O errors. + +An EOF without `close_notify` (`UnexpectedEof` from rustls) is reported as a +clean EOF (`Ok(0)`) by the TLS read arms, so a server dying mid-session maps to +the same `Closed`/connection error as plaintext. PG-wire message framing +already detects truncation. + +## Client changes + +- `SyncStream::Tls(Box>)` and + `AsyncStream::Tls(Box>)`. New + `is_tls()`. The unused `SyncStream::try_clone` is deleted. +- `Client` / `AsyncClient` keep `tls: Option>` (built + `ClientConfig` + `ServerName<'static>`), set **only** when TLS was negotiated + (a `Prefer` fallback stores `None`), so cancel reuses it without reloading PEM + files. +- One shared cancel sender per I/O flavour: + `send_cancel_sync(addr, pid, secret, Option<&TlsConnector>)` and + `send_cancel_async(..)`. `Client::cancel` and `AsyncClient::cancel_sync` + (called from `Drop`, must stay blocking and runtime-free) use the sync one; + `AsyncClient::cancel` uses the async one. Over TLS they write the + CancelRequest, flush, send `close_notify`, shut down the write half and drain + to EOF with a short timeout (1 s), so the OS does not RST away the cancel + because unread session tickets were left in the receive buffer. +- `Client::connect_unix` / `connect_named_pipe` (and `connect_endpoint`'s IPC + branches) reject `Require`/`VerifyCa`/`VerifyFull` with + `FeatureNotSupported`; `Prefer`/`Disable` connect in plaintext. + +## Testing + +Two layers. + +**Fake-server unit tests** (`tls.rs` `#[cfg(test)]`, all platforms): a +`std::net::TcpListener` thread that reads 8 bytes and asserts they are an +SSLRequest, answers `S`/`N`/`E`, runs a rustls `ServerConnection`, and records +what it decrypts. They prove what goes on the wire, which hyperd cannot (it +accepts plaintext cancels). Cases: each mode × right/wrong CA, wrong hostname, +`server_name` override, IP host, `'N'` with each fallback, `'E'`, `Prefer` + +wrong CA must error (no downgrade), `Disable` sends no SSLRequest, mTLS +offered/required, handshake timeout against a server that accepts and never +answers, and the cancel senders: SSLRequest, then ClientHello, then a decrypted +16-byte CancelRequest with the right pid/secret; `'N'` on a cancel is +`Error::Tls` and nothing plaintext is sent. + +**Real hyperd** (`hyperdb-api/tests/tls_tests.rs`, all platforms; key chmod 0600 +on Unix only): rcgen CA + server cert (SANs `localhost`, `127.0.0.1`) with the +certificate file holding `leaf + CA`. + +- Enforcement control: plaintext connect to an `ssl_force` server fails. This + is what makes every other TLS success meaningful. +- `Require` sync and async: `is_tls()`, `Inserter` COPY, a multi-chunk query. +- `VerifyCa` right/wrong CA; `VerifyFull` `localhost` and `127.0.0.1`; + `server_name("wrong.example")` fails under `VerifyFull` but passes under + `VerifyCa`; `VerifyCa` without root cert is `Config`. +- `Prefer` against TLS and non-TLS servers; `Prefer` + wrong CA errors; + `Require` against non-TLS errors. +- mTLS: client cert from the CA accepted; client cert from another CA rejected + with `Error::Tls`. +- Cancel under TLS + `ssl_force`, sync and async: SQLSTATE 57014. +- Async builder, async pool, sync pool. +- gRPC + `.tls(Require)` is `FeatureNotSupported` (checked before connecting); + UDS endpoint + `Require` is `FeatureNotSupported`, + `Prefer` connects. +- `Config::from_str` rejects `sslmode=`. + +Node smoke test: CA + leaf generated with `openssl` (two invocations; leaf has +`basicConstraints=CA:FALSE`, SAN `localhost`/`127.0.0.1`), started via +`new HyperProcess(undefined, { parameters })` with `ssl_force`; plaintext +`Connection.connect` fails, `Require` and `VerifyFull` + `tlsRootCert(ca)` +succeed with `isTls`. Skipped with a note when `openssl` is missing, except +under `CI`, where it fails. + +## Docs + +Fix the over-claims to describe what now exists (TCP only, builders and pools +only, the modes table): README.md, AGENTS.md (62, 123), core `lib.rs` and +`client/mod.rs`, DEVELOPMENT.md (42, 92, 271-281, 800, 890), +`hyperdb-api-core/docs/DEVELOPMENT-client.md` (179-189 TLS internals, 233-255 +TLS tests, 300-301), `docs/NODEJS_API_SUMMARY.md` (28, 67), the Node README. +Per-crate CHANGELOGs under the right Keep-a-Changelog headings (core: Added / +Changed / Removed, including the new `client::Error::Tls` variant breaking +exhaustive matches; api: Added, plus Changed for the new `tls` field on the pool +configs; node: Added). diff --git a/hyperdb-api-core/src/client/config.rs b/hyperdb-api-core/src/client/config.rs index 47fb1fd1..e8664d40 100644 --- a/hyperdb-api-core/src/client/config.rs +++ b/hyperdb-api-core/src/client/config.rs @@ -5,6 +5,9 @@ use std::time::Duration; +use super::error::Error; +use super::tls::TlsConfig; + /// Configuration for a Hyper database connection. /// /// Use the builder pattern to construct a configuration: @@ -36,8 +39,23 @@ pub struct Config { connect_timeout: Option, application_name: Option, options: Vec<(String, String)>, + tls: TlsConfig, } +/// libpq connection-string keys that configure TLS. [`Config`]'s `FromStr` +/// rejects them instead of forwarding them to the server as startup options, +/// where they would be silently ignored. +const LIBPQ_TLS_KEYS: &[&str] = &[ + "sslmode", + "sslrootcert", + "sslcert", + "sslkey", + "sslpassword", + "sslcrl", + "sslsni", + "sslnegotiation", +]; + impl Config { /// Creates a new configuration with default settings. /// @@ -54,6 +72,7 @@ impl Config { application_name: None, // Set HyperBinary format by default for optimal performance options: vec![("result_format_code".to_string(), "HyperBinary".to_string())], + tls: TlsConfig::default(), } } @@ -105,6 +124,17 @@ impl Config { self } + /// Sets the TLS configuration; the default is + /// [`TlsMode::Disable`](super::tls::TlsMode::Disable). + /// + /// Applies to TCP connections only. Over a Unix domain socket or a named + /// pipe, [`TlsMode::Prefer`](super::tls::TlsMode::Prefer) connects in + /// plaintext and every mode that requires TLS is rejected. + pub fn with_tls(mut self, tls: impl Into) -> Self { + self.tls = tls.into(); + self + } + /// Returns the host. #[must_use] pub fn host(&self) -> &str { @@ -141,6 +171,12 @@ impl Config { self.connect_timeout } + /// Returns the TLS configuration. + #[must_use] + pub fn tls(&self) -> &TlsConfig { + &self.tls + } + /// Returns the application name. #[must_use] pub fn application_name(&self) -> Option<&str> { @@ -180,11 +216,15 @@ impl Config { } impl std::str::FromStr for Config { - type Err = String; + type Err = Error; /// Parses a connection string into a Config. /// /// Format: `host:port/database?user=xxx&password=xxx` + /// + /// The libpq TLS keys (`sslmode`, `sslrootcert`, `sslcert`, `sslkey`, ...) + /// are rejected with [`Error::Config`]; configure TLS with + /// [`Config::with_tls`] instead. fn from_str(s: &str) -> Result { let mut config = Config::new(); @@ -197,7 +237,9 @@ impl std::str::FromStr for Config { if let Some(idx) = addr.rfind(':') { config.host = addr[..idx].to_string(); - config.port = addr[idx + 1..].parse().map_err(|_| "invalid port number")?; + config.port = addr[idx + 1..] + .parse() + .map_err(|_| Error::config("invalid port number"))?; } else { config.host = addr.to_string(); } @@ -218,8 +260,14 @@ impl std::str::FromStr for Config { if param.is_empty() { continue; } + let name = param.split_once('=').map_or(param, |(name, _)| name); + if LIBPQ_TLS_KEYS.contains(&name) { + return Err(Error::config(format!( + "TLS options are not accepted in the connection string (found `{name}`); \ + use ConnectionBuilder::tls" + ))); + } if let Some(idx) = param.find('=') { - let name = ¶m[..idx]; let value = ¶m[idx + 1..]; match name { "user" => config.user = Some(value.to_string()), @@ -253,6 +301,28 @@ mod tests { assert_eq!(config.user, Some("test".to_string())); } + #[test] + fn test_config_from_str_rejects_libpq_tls_keys() { + for key in LIBPQ_TLS_KEYS { + let err = format!("h:1/db?{key}=x") + .parse::() + .expect_err("TLS key must be rejected"); + assert!(matches!(err, Error::Config(_)), "{key}: got {err:?}"); + assert!( + err.to_string().contains("ConnectionBuilder::tls"), + "got {err}" + ); + } + let err = "h:1/db?sslmode=require".parse::().unwrap_err(); + assert!(err.to_string().contains("sslmode"), "got {err}"); + // A bare key with no `=` is rejected too, not silently dropped. + let err = "h:1/db?user=u&sslmode".parse::().unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + + let err = "h:notaport/db".parse::().unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + } + /// Mirrors the `///` example on `Config` so the builder chain is still /// exercised at runtime even though the doctest itself is `no_run` (see /// the doc comment above for why). diff --git a/hyperdb-api-core/src/client/error.rs b/hyperdb-api-core/src/client/error.rs index d51d7954..85987c55 100644 --- a/hyperdb-api-core/src/client/error.rs +++ b/hyperdb-api-core/src/client/error.rs @@ -80,6 +80,14 @@ pub enum Error { #[error("{0}")] Config(String), + /// TLS negotiation, handshake or certificate-verification failure. + /// + /// Includes a server that refuses TLS when the configured mode requires + /// it. Unreadable certificate files and invalid option combinations are + /// [`Error::Config`] instead. + #[error("{0}")] + Tls(String), + /// Operation timed out. #[error("{0}")] Timeout(String), @@ -173,6 +181,11 @@ impl Error { Error::Config(message.into()) } + /// Creates a TLS error. + pub fn tls(message: impl Into) -> Self { + Error::Tls(message.into()) + } + /// Creates a timeout error. pub fn timeout(message: impl Into) -> Self { Error::Timeout(message.into()) @@ -214,7 +227,14 @@ impl Error { // Convenience constructors for the common shapes. - /// Creates an I/O error from an [`io::Error`]. + /// Creates an error from an [`io::Error`]. + /// + /// rustls reports handshake failures and TLS alerts as an `io::Error` + /// wrapping a `rustls::Error`; those become [`Error::Tls`]. Under TLS 1.3 + /// a server's rejection of the client certificate arrives only on the + /// first read after the handshake, so the classification has to happen + /// wherever I/O errors are converted, not only during negotiation. + /// Everything else is [`Error::Io`]. /// /// Takes the error by value so it can be used point-free as /// `.map_err(Error::from_io)`. @@ -224,7 +244,13 @@ impl Error { )] #[must_use] pub fn from_io(err: io::Error) -> Self { - Error::Io(err.to_string()) + match err + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + { + Some(tls_err) => Error::Tls(tls_err.to_string()), + None => Error::Io(err.to_string()), + } } /// Creates an error from a database error response. @@ -251,6 +277,7 @@ impl Error { | Error::Protocol(message) | Error::Io(message) | Error::Config(message) + | Error::Tls(message) | Error::Timeout(message) | Error::Conversion(message) | Error::FeatureNotSupported(message) @@ -464,4 +491,25 @@ mod tests { // Empty parentheses assert_eq!(extract_sqlstate("ERROR: message ()"), None); } + + /// rustls surfaces handshake failures and alerts as an `io::Error` + /// wrapping a `rustls::Error`; those must classify as `Tls`, not `Io`. + #[test] + fn from_io_classifies_rustls_errors_as_tls() { + let err = Error::from_io(io::Error::new( + io::ErrorKind::InvalidData, + rustls::Error::AlertReceived(rustls::AlertDescription::CertificateRequired), + )); + assert!(matches!(err, Error::Tls(_)), "got {err:?}"); + assert_eq!(err.message(), err.to_string()); + + let err = Error::from_io(io::Error::new(io::ErrorKind::ConnectionReset, "reset")); + assert!( + matches!(err, Error::Io(ref msg) if msg == "reset"), + "got {err:?}" + ); + + let err = Error::from(io::Error::new(io::ErrorKind::InvalidData, "not rustls")); + assert!(matches!(err, Error::Io(_)), "got {err:?}"); + } } diff --git a/hyperdb-api-core/src/client/tls.rs b/hyperdb-api-core/src/client/tls.rs index 356d7118..45df473c 100644 --- a/hyperdb-api-core/src/client/tls.rs +++ b/hyperdb-api-core/src/client/tls.rs @@ -1,315 +1,1587 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! TLS configuration and connection handling. +//! TLS for PG-wire TCP connections. //! -//! This module provides TLS support for secure connections to Hyper servers -//! using a pure Rust TLS implementation (rustls). +//! A [`TlsConfig`] picks a [`TlsMode`] (the libpq `sslmode` names) and, +//! optionally, a root certificate, a client certificate for mutual TLS, and a +//! server-name override. It is set with +//! [`Config::with_tls`](super::Config::with_tls). //! -//! # Certificate Formats +//! When a mode other than [`TlsMode::Disable`] is set, the client sends an +//! `SSLRequest` on the fresh TCP connection before the startup message, reads +//! the server's one-byte answer straight off the socket, and runs a rustls +//! handshake on `'S'`. A cancel request for a TLS session is sent over TLS +//! too, and never falls back to plaintext. TLS applies to TCP only; gRPC +//! picks TLS through its `https://` endpoint scheme. +//! +//! # Certificate files //! //! | Item | Format | //! |------|--------| -//! | CA Cert | PEM | -//! | Client Cert | PEM | -//! | Client Key | PEM (PKCS#8 or PKCS#1) | -//! -//! - **PEM format**: Base64-encoded with `-----BEGIN/END CERTIFICATE-----` headers -//! - **PKCS#8**: Key format with `-----BEGIN PRIVATE KEY-----` header -//! - **PKCS#1**: RSA-specific format with `-----BEGIN RSA PRIVATE KEY-----` header +//! | Root certificate(s) | PEM, one or more `-----BEGIN CERTIFICATE-----` blocks | +//! | Client certificate | PEM, leaf first, optionally followed by intermediates | +//! | Client key | PEM: PKCS#8 (`BEGIN PRIVATE KEY`), PKCS#1 (`BEGIN RSA PRIVATE KEY`) or SEC1 (`BEGIN EC PRIVATE KEY`) | //! //! # Example //! -//! ```ignore -//! use hyperdb_api_core::client::{Client, Config}; -//! use hyperdb_api_core::client::tls::TlsConfig; +//! ``` +//! use hyperdb_api_core::client::Config; +//! use hyperdb_api_core::client::tls::{TlsConfig, TlsMode}; //! -//! # fn example() -> hyperdb_api_core::client::Result<()> { +//! // Verify the server's chain against our CA and its name against the host. //! let config = Config::new() -//! .with_host("secure-hyper.example.com") -//! .with_port(7484) -//! .with_tls(TlsConfig::default()); // TODO: with_tls not yet on Config +//! .with_host("hyper.example.com") +//! .with_tls(TlsConfig::new(TlsMode::VerifyFull).root_cert("ca.pem")); +//! assert_eq!(config.tls().mode(), TlsMode::VerifyFull); //! -//! let client = Client::connect(&config)?; -//! # Ok(()) -//! # } +//! // A bare mode converts into a `TlsConfig`. +//! let config = Config::new().with_tls(TlsMode::Require); +//! assert_eq!(config.tls().mode(), TlsMode::Require); +//! +//! // The libpq names parse. +//! assert_eq!("verify-ca".parse::(), Ok(TlsMode::VerifyCa)); //! ``` -use std::path::PathBuf; +#![cfg_attr( + not(test), + expect(dead_code, reason = "wired into the clients in the next change") +)] -/// TLS configuration options. -#[derive(Debug, Clone)] -pub struct TlsConfig { - /// Whether to verify the server certificate. - pub verify_server: bool, - /// Path to CA certificate file (PEM format). - pub ca_cert_path: Option, - /// Path to client certificate file (PEM format). - pub client_cert_path: Option, - /// Path to client key file (PEM format). - pub client_key_path: Option, - /// Server name for SNI (Server Name Indication). - pub server_name: Option, -} - -impl Default for TlsConfig { - fn default() -> Self { - TlsConfig { - verify_server: true, - ca_cert_path: None, - client_cert_path: None, - client_key_path: None, - server_name: None, +use std::fmt; +use std::io::{self, Read, Write}; +use std::net::{Shutdown, TcpStream}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bytes::BytesMut; +use rustls::client::WebPkiServerVerifier; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::{WebPkiSupportedAlgorithms, verify_tls12_signature, verify_tls13_signature}; +use rustls::pki_types::pem::PemObject; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime}; +use rustls::server::ParsedCertificate; +use rustls::{ + ClientConfig, ClientConnection, DigitallySignedStruct, RootCertStore, SignatureScheme, + StreamOwned, +}; +use tracing::warn; + +use super::error::{Error, Result}; +use crate::protocol::message::frontend; + +/// How a TCP connection uses TLS, named after libpq's `sslmode`. +/// +/// | Mode | Server without TLS | Server certificate check | +/// |---|---|---| +/// | `Disable` (default) | plaintext; no `SSLRequest` is sent | — | +/// | `Prefer` | plaintext | none, or chain-only if a root certificate is set | +/// | `Require` | error | none, or chain-only if a root certificate is set | +/// | `VerifyCa` | error | chain against the root certificate (required) | +/// | `VerifyFull` | error | chain and host name, against the root certificate or else the bundled Mozilla roots | +/// +/// A configured root certificate is the **only** trust anchor. "None" still +/// checks the handshake signatures; it skips only the chain and the name. +/// +/// Where libpq and sqlx/tokio-postgres differ, this follows sqlx: +/// +/// - The default is `Disable`, not libpq's `prefer`: `hyperd` is usually a +/// local engine with TLS off. +/// - `Prefer` falls back to plaintext only when the server answers that it +/// has no TLS. A failed handshake or certificate check is an error, never a +/// downgrade. +/// - `VerifyFull` without a root certificate trusts the bundled +/// `webpki-roots` (Mozilla) set, not the OS store. +/// - Default certificate files such as `~/.postgresql/root.crt` are never +/// read. +/// - Over a Unix domain socket or a named pipe, `Prefer` connects in +/// plaintext and the other enabled modes are an error rather than ignored. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum TlsMode { + /// Never use TLS (the default). + #[default] + Disable, + /// Use TLS when the server offers it, plaintext otherwise. + Prefer, + /// Require TLS; check the certificate chain only if a root certificate + /// is set. + Require, + /// Require TLS and a certificate chain to the configured root + /// certificate. The host name is not checked. + VerifyCa, + /// Require TLS, a trusted certificate chain, and a certificate that + /// names the host. + VerifyFull, +} + +impl TlsMode { + /// The libpq `sslmode` spelling: `"disable"`, `"prefer"`, `"require"`, + /// `"verify-ca"` or `"verify-full"`. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + TlsMode::Disable => "disable", + TlsMode::Prefer => "prefer", + TlsMode::Require => "require", + TlsMode::VerifyCa => "verify-ca", + TlsMode::VerifyFull => "verify-full", } } } -impl TlsConfig { - /// Creates a new TLS configuration with default settings. - #[must_use] - pub fn new() -> Self { - Self::default() +impl fmt::Display for TlsMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) } +} - #[must_use] - /// Disables server certificate verification. - /// - /// # Security Warning - /// - /// **This is a serious security risk in production environments.** - /// Disabling certificate verification allows man-in-the-middle attacks. - /// - /// Only use this for: - /// - Local development with self-signed certificates - /// - Testing environments - /// - Debugging certificate issues (temporarily) - /// - /// A warning will be logged at runtime when this is enabled. - pub fn danger_accept_invalid_certs(mut self) -> Self { - tracing::warn!( - "TLS certificate verification disabled - this should only be used for testing. \ - Man-in-the-middle attacks are possible." - ); +impl FromStr for TlsMode { + type Err = ParseTlsModeError; - self.verify_server = false; - self + /// Parses the libpq `sslmode` spelling (case-sensitive, as in libpq). + fn from_str(s: &str) -> std::result::Result { + match s { + "disable" => Ok(TlsMode::Disable), + "prefer" => Ok(TlsMode::Prefer), + "require" => Ok(TlsMode::Require), + "verify-ca" => Ok(TlsMode::VerifyCa), + "verify-full" => Ok(TlsMode::VerifyFull), + _ => Err(ParseTlsModeError { + input: s.to_string(), + }), + } } +} - #[must_use] - /// Sets the CA certificate file for verifying the server. - pub fn ca_cert(mut self, path: impl Into) -> Self { - self.ca_cert_path = Some(path.into()); +/// The error returned when a string is not a [`TlsMode`] name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseTlsModeError { + input: String, +} + +impl fmt::Display for ParseTlsModeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid TLS mode `{}`; expected one of: disable, prefer, require, verify-ca, verify-full", + self.input + ) + } +} + +impl std::error::Error for ParseTlsModeError {} + +/// TLS settings for a TCP connection. +/// +/// Built from a [`TlsMode`] with optional files; see the mode table on +/// [`TlsMode`] for what each mode checks. The files are read when the +/// connection is opened, and an unreadable or empty file is a configuration +/// error then. +/// +/// ``` +/// use hyperdb_api_core::client::tls::{TlsConfig, TlsMode}; +/// +/// let tls = TlsConfig::new(TlsMode::VerifyFull) +/// .root_cert("ca.pem") +/// .client_cert("client.pem", "client.key") +/// .server_name("hyper.internal"); +/// assert_eq!(tls.mode(), TlsMode::VerifyFull); +/// ``` +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[must_use = "TlsConfig uses a consuming builder pattern; use the returned value"] +pub struct TlsConfig { + mode: TlsMode, + root_cert: Option, + client_cert: Option<(PathBuf, PathBuf)>, + server_name: Option, +} + +impl TlsConfig { + /// A configuration for `mode` with no files and no server-name override. + pub fn new(mode: TlsMode) -> Self { + TlsConfig { + mode, + ..TlsConfig::default() + } + } + + /// Trust only the certificates in this PEM file (libpq `sslrootcert`). + /// + /// Required for [`TlsMode::VerifyCa`]. With [`TlsMode::Prefer`] or + /// [`TlsMode::Require`] it turns on chain verification. + pub fn root_cert(mut self, pem_path: impl AsRef) -> Self { + self.root_cert = Some(pem_path.as_ref().to_path_buf()); self } - #[must_use] - /// Sets the client certificate for mutual TLS. - pub fn client_cert( - mut self, - cert_path: impl Into, - key_path: impl Into, - ) -> Self { - self.client_cert_path = Some(cert_path.into()); - self.client_key_path = Some(key_path.into()); + /// Offer this client certificate and key for mutual TLS (libpq `sslcert` + /// and `sslkey`). Both are PEM files. + pub fn client_cert(mut self, cert_path: impl AsRef, key_path: impl AsRef) -> Self { + self.client_cert = Some(( + cert_path.as_ref().to_path_buf(), + key_path.as_ref().to_path_buf(), + )); self } - #[must_use] - /// Sets the server name for SNI. + /// Use this name instead of the connection host for SNI and for the + /// [`TlsMode::VerifyFull`] host-name check. pub fn server_name(mut self, name: impl Into) -> Self { self.server_name = Some(name.into()); self } - /// Returns true if client certificates are configured. + /// The configured mode. #[must_use] - pub fn has_client_cert(&self) -> bool { - self.client_cert_path.is_some() && self.client_key_path.is_some() + pub fn mode(&self) -> TlsMode { + self.mode } } -/// TLS mode for the connection. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum TlsMode { - /// No TLS (plain TCP). - #[default] - Disable, - /// Prefer TLS if available, fall back to plain TCP. - Prefer, - /// Require TLS, fail if not available. - Require, - /// Require TLS and verify the server certificate. - VerifyCA, - /// Require TLS, verify server certificate, and verify hostname. - VerifyFull, +impl From for TlsConfig { + fn from(mode: TlsMode) -> Self { + TlsConfig::new(mode) + } } -impl TlsMode { - /// Returns true if TLS is enabled. - #[must_use] - pub fn is_enabled(&self) -> bool { - !matches!(self, TlsMode::Disable) +/// A ready-to-use client TLS setup: the built rustls configuration and the +/// name to present and verify. Built once per connection and kept for the +/// connection's cancel requests, so the PEM files are not re-read. +#[derive(Debug)] +pub(crate) struct TlsConnector { + config: Arc, + server_name: ServerName<'static>, +} + +impl TlsConnector { + /// Builds the connector for `host`, or `Ok(None)` for + /// [`TlsMode::Disable`]. + /// + /// # Errors + /// + /// [`Error::Config`] when a file cannot be read or holds no usable PEM + /// item, when [`TlsMode::VerifyCa`] has no root certificate, when the + /// client certificate and key do not form a pair, or when the server + /// name is not a valid DNS name or IP address. + pub(crate) fn build(tls: &TlsConfig, host: &str) -> Result>> { + if tls.mode == TlsMode::Disable { + return Ok(None); + } + + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let algorithms = provider.signature_verification_algorithms; + let roots = tls.root_cert.as_deref().map(load_roots).transpose()?; + + let verifier: Arc = match (tls.mode, roots) { + (TlsMode::Disable, _) => return Ok(None), + (TlsMode::Prefer | TlsMode::Require, None) => Arc::new(NoVerify { algorithms }), + (TlsMode::Prefer | TlsMode::Require | TlsMode::VerifyCa, Some(roots)) => { + Arc::new(ChainOnly { + roots: Arc::new(roots), + algorithms, + }) + } + (TlsMode::VerifyCa, None) => { + return Err(Error::config( + "TlsMode::VerifyCa needs a root certificate; set TlsConfig::root_cert", + )); + } + (TlsMode::VerifyFull, roots) => { + let roots = roots.unwrap_or_else(|| { + let mut store = RootCertStore::empty(); + store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + store + }); + WebPkiServerVerifier::builder_with_provider(Arc::new(roots), Arc::clone(&provider)) + .build() + .map_err(|e| Error::config(format!("cannot build TLS verifier: {e}")))? + } + }; + + let builder = ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|e| Error::config(format!("TLS protocol configuration: {e}")))? + .dangerous() + .with_custom_certificate_verifier(verifier); + let config = match &tls.client_cert { + Some((cert_path, key_path)) => builder + .with_client_auth_cert( + load_certs(cert_path, "client certificate")?, + load_key(key_path)?, + ) + .map_err(|e| Error::config(format!("invalid client certificate or key: {e}")))?, + None => builder.with_no_client_auth(), + }; + + let name = tls.server_name.as_deref().unwrap_or_else(|| { + // An IPv6 host arrives as `[::1]`; the brackets are URL syntax. + host.strip_prefix('[') + .and_then(|inner| inner.strip_suffix(']')) + .unwrap_or(host) + }); + let server_name = ServerName::try_from(name) + .map_err(|e| Error::config(format!("invalid TLS server name `{name}`: {e}")))? + .to_owned(); + + Ok(Some(Arc::new(TlsConnector { + config: Arc::new(config), + server_name, + }))) } +} - /// Returns true if TLS is required (not just preferred). - #[must_use] - pub fn is_required(&self) -> bool { - matches!( - self, - TlsMode::Require | TlsMode::VerifyCA | TlsMode::VerifyFull - ) +fn load_certs(path: &Path, what: &str) -> Result>> { + let certs = CertificateDer::pem_file_iter(path) + .and_then(Iterator::collect::, _>>) + .map_err(|e| Error::config(format!("cannot read {what} `{}`: {e}", path.display())))?; + if certs.is_empty() { + return Err(Error::config(format!( + "{what} `{}` contains no PEM certificate", + path.display() + ))); } + Ok(certs) +} - /// Returns true if server certificate verification is required. - #[must_use] - pub fn verify_server(&self) -> bool { - matches!(self, TlsMode::VerifyCA | TlsMode::VerifyFull) +fn load_roots(path: &Path) -> Result { + let mut roots = RootCertStore::empty(); + for cert in load_certs(path, "root certificate")? { + roots.add(cert).map_err(|e| { + Error::config(format!( + "invalid root certificate in `{}`: {e}", + path.display() + )) + })?; } + Ok(roots) +} - /// Returns true if hostname verification is required. - #[must_use] - pub fn verify_hostname(&self) -> bool { - matches!(self, TlsMode::VerifyFull) +fn load_key(path: &Path) -> Result> { + // `from_pem_file` reports a file with no key section as `NoItemsFound`. + PrivateKeyDer::from_pem_file(path) + .map_err(|e| Error::config(format!("cannot read client key `{}`: {e}", path.display()))) +} + +/// [`TlsMode::Prefer`] / [`TlsMode::Require`] without a root certificate: +/// accept any certificate, but still verify the handshake signatures, so the +/// peer has to hold the key for the certificate it presents. +#[derive(Debug)] +struct NoVerify { + algorithms: WebPkiSupportedAlgorithms, +} + +impl ServerCertVerifier for NoVerify { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> std::result::Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls12_signature(message, cert, dss, &self.algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls13_signature(message, cert, dss, &self.algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.algorithms.supported_schemes() } } -/// TLS implementation using the `rustls` crate. -pub mod rustls_impl { - use super::TlsConfig; - use std::sync::Arc; +/// [`TlsMode::VerifyCa`], and `Prefer` / `Require` with a root certificate: +/// the chain must lead to `roots`; the name is not checked. This is the chain +/// step `WebPkiServerVerifier` runs before its name check, called directly. +#[derive(Debug)] +struct ChainOnly { + roots: Arc, + algorithms: WebPkiSupportedAlgorithms, +} - use rustls::pki_types::pem::PemObject; - use rustls::pki_types::{CertificateDer, PrivateKeyDer}; - use tokio::net::TcpStream; - use tokio_rustls::TlsConnector; - use tokio_rustls::rustls::{ClientConfig, RootCertStore}; +impl ServerCertVerifier for ChainOnly { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + now: UnixTime, + ) -> std::result::Result { + let cert = ParsedCertificate::try_from(end_entity)?; + rustls::client::verify_server_cert_signed_by_trust_anchor( + &cert, + &self.roots, + intermediates, + now, + self.algorithms.all, + )?; + Ok(ServerCertVerified::assertion()) + } - use crate::client::error::{Error, Result}; + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls12_signature(message, cert, dss, &self.algorithms) + } - /// Creates a TLS connector from the configuration. - /// - /// # Errors - /// - /// Returns [`Error::Config`] when: - /// - The CA cert path is set but cannot be opened or the PEM bytes - /// cannot be parsed / added to the root store. - /// - The client cert / key path is set but cannot be opened, the - /// PEM payload is invalid, the key section is missing, or - /// `rustls` rejects the cert/key pair. - /// - Rustls cannot build a `ClientConfig` with the configured - /// protocol versions. - /// - /// # Panics - /// - /// Does not panic in practice. The `client_cert_path` and - /// `client_key_path` `.unwrap()` calls are guarded by the - /// preceding [`TlsConfig::has_client_cert`] check, which only - /// returns `true` when both paths are `Some`. - pub fn create_connector(config: &TlsConfig, _host: &str) -> Result { - let mut root_store = RootCertStore::empty(); - - // Add system root certificates - root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - - // Add custom CA certificate if provided - if let Some(ref ca_path) = config.ca_cert_path { - let certs = CertificateDer::pem_file_iter(ca_path) - .map_err(|e| Error::config(format!("failed to read CA cert: {e}")))? - .collect::, _>>() - .map_err(|e| Error::config(format!("invalid CA cert: {e}")))?; - for cert in certs { - root_store - .add(cert) - .map_err(|e| Error::config(format!("failed to add CA cert: {e}")))?; + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls13_signature(message, cert, dss, &self.algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.algorithms.supported_schemes() + } +} + +/// What to do when the server answers the `SSLRequest` with `'N'`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Fallback { + /// Continue in plaintext on the same socket ([`TlsMode::Prefer`]). + AllowPlaintext, + /// Fail with [`Error::Tls`]. Every enabled mode but `Prefer`, and every + /// cancel request. + Deny, +} + +impl Fallback { + /// The fallback a connection in `mode` uses. + pub(crate) fn for_mode(mode: TlsMode) -> Self { + if mode == TlsMode::Prefer { + Fallback::AllowPlaintext + } else { + Fallback::Deny + } + } +} + +/// The outcome of a negotiation: the socket unchanged (after `'N'` with +/// [`Fallback::AllowPlaintext`]), or the TLS stream. +pub(crate) enum Negotiated { + Plain(Plain), + Tls(Tls), +} + +/// How long a cancel request may spend on TLS negotiation and on sending. +/// A cancel runs from `Drop` too, so it must not hang on a silent server. +pub(crate) const CANCEL_TLS_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long a TLS cancel waits for the server to close after `close_notify`. +const CANCEL_DRAIN_TIMEOUT: Duration = Duration::from_secs(1); + +const NO_TLS_MESSAGE: &str = "server does not support TLS (answered 'N' to SSLRequest)"; + +const EARLY_CLOSE_MESSAGE: &str = "server closed the connection during TLS negotiation"; + +fn ssl_request_bytes() -> BytesMut { + let mut buf = BytesMut::with_capacity(8); + frontend::ssl_request(&mut buf); + buf +} + +fn unexpected_reply(byte: u8) -> Error { + Error::protocol(format!( + "unexpected response to SSLRequest: {:?} (expected 'S' or 'N')", + char::from(byte) + )) +} + +fn timed_out(timeout: Option) -> Error { + match timeout { + Some(limit) => Error::timeout(format!("TLS negotiation timed out after {limit:?}")), + None => Error::timeout("TLS negotiation timed out"), + } +} + +/// Classifies an I/O error raised while negotiating: a socket timeout is +/// [`Error::Timeout`], an early close is reported as such, and everything +/// else goes through [`Error::from_io`] (so rustls failures are `Tls`). +fn negotiation_error(err: io::Error, timeout: Option) -> Error { + match err.kind() { + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut => timed_out(timeout), + io::ErrorKind::UnexpectedEof + if err + .get_ref() + .is_none_or(|inner| !inner.is::()) => + { + Error::io(EARLY_CLOSE_MESSAGE) + } + _ => Error::from_io(err), + } +} + +/// Sets the socket timeouts to what is left until `deadline`, so the limit +/// holds for the negotiation as a whole: a peer that answers each read just +/// in time cannot stretch it. No deadline leaves the socket blocking. +fn arm(tcp: &TcpStream, deadline: Option, timeout: Option) -> Result<()> { + let Some(deadline) = deadline else { + return Ok(()); + }; + let remaining = deadline.saturating_duration_since(Instant::now()); + // A zero timeout is `InvalidInput` to the socket; it means time is up. + if remaining.is_zero() { + return Err(timed_out(timeout)); + } + tcp.set_read_timeout(Some(remaining)) + .and_then(|()| tcp.set_write_timeout(Some(remaining))) + .map_err(Error::from_io) +} + +/// Negotiates TLS on a freshly connected socket, before the startup message. +/// +/// `timeout` bounds the whole negotiation; the socket timeouts it uses are +/// cleared again before returning. The answer byte is read straight off the +/// socket, so nothing the server sends after it can be mistaken for +/// plaintext. +pub(crate) fn negotiate_sync( + mut tcp: TcpStream, + connector: &TlsConnector, + fallback: Fallback, + timeout: Option, +) -> Result>> { + // An unrepresentable deadline (a timeout of centuries) is no deadline. + let deadline = timeout.and_then(|limit| Instant::now().checked_add(limit)); + + let mut reply = [0u8; 1]; + arm(&tcp, deadline, timeout)?; + tcp.write_all(&ssl_request_bytes()) + .map_err(|e| negotiation_error(e, timeout))?; + arm(&tcp, deadline, timeout)?; + tcp.read_exact(&mut reply) + .map_err(|e| negotiation_error(e, timeout))?; + + match reply[0] { + b'S' => {} + b'N' => { + return match fallback { + Fallback::AllowPlaintext => { + clear_timeouts(&tcp)?; + Ok(Negotiated::Plain(tcp)) + } + Fallback::Deny => Err(Error::tls(NO_TLS_MESSAGE)), + }; + } + other => return Err(unexpected_reply(other)), + } + + let mut conn = + ClientConnection::new(Arc::clone(&connector.config), connector.server_name.clone()) + .map_err(|e| Error::tls(e.to_string()))?; + // Driven one `read_tls` / `write_tls` at a time rather than with + // `complete_io`, which keeps reading for as long as bytes arrive: the + // deadline has to be re-armed before every socket operation. Looping + // until nothing is left to write also sends the client's Finished (and + // certificate, for mutual TLS) now rather than with the first + // application write. + while conn.is_handshaking() || conn.wants_write() { + arm(&tcp, deadline, timeout)?; + if conn.wants_write() { + match conn.write_tls(&mut tcp) { + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(negotiation_error(e, timeout)), } + continue; + } + match conn.read_tls(&mut tcp) { + Ok(0) => return Err(Error::io(EARLY_CLOSE_MESSAGE)), + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(negotiation_error(e, timeout)), } + if let Err(e) = conn.process_new_packets() { + // Last-gasp attempt to send the alert describing the failure. + let _ = conn.write_tls(&mut tcp); + return Err(Error::tls(e.to_string())); + } + } - let provider = Arc::new(rustls::crypto::ring::default_provider()); - let builder = ClientConfig::builder_with_provider(provider) - .with_safe_default_protocol_versions() - .map_err(|e| Error::config(format!("TLS protocol config error: {e}")))? - .with_root_certificates(root_store); - - let client_config = if config.has_client_cert() { - // Load client certificate and key - let cert_path = config.client_cert_path.as_ref().unwrap(); - let key_path = config.client_key_path.as_ref().unwrap(); - - let certs = CertificateDer::pem_file_iter(cert_path) - .map_err(|e| Error::config(format!("failed to read client cert: {e}")))? - .collect::, _>>() - .map_err(|e| Error::config(format!("invalid client cert: {e}")))?; - - // `from_pem_file` returns `Error::NoItemsFound` when the file is - // syntactically valid PEM but contains no private-key section, so - // we don't need a separate "no private key found" branch. - let key = PrivateKeyDer::from_pem_file(key_path) - .map_err(|e| Error::config(format!("invalid client key: {e}")))?; - - builder - .with_client_auth_cert(certs, key) - .map_err(|e| Error::config(format!("invalid client auth: {e}")))? - } else { - builder.with_no_client_auth() + clear_timeouts(&tcp)?; + Ok(Negotiated::Tls(StreamOwned::new(conn, tcp))) +} + +fn clear_timeouts(tcp: &TcpStream) -> Result<()> { + tcp.set_read_timeout(None).map_err(Error::from_io)?; + tcp.set_write_timeout(None).map_err(Error::from_io) +} + +/// Async [`negotiate_sync`]: the whole negotiation runs under `timeout`. +pub(crate) async fn negotiate_async( + mut tcp: tokio::net::TcpStream, + connector: &TlsConnector, + fallback: Fallback, + timeout: Option, +) -> Result>> +{ + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let negotiation = async move { + let mut reply = [0u8; 1]; + let request = async { + tcp.write_all(&ssl_request_bytes()).await?; + tcp.flush().await?; + tcp.read_exact(&mut reply).await }; + request.await.map_err(|e| negotiation_error(e, timeout))?; - Ok(TlsConnector::from(Arc::new(client_config))) - } + match reply[0] { + b'S' => {} + b'N' => { + return match fallback { + Fallback::AllowPlaintext => Ok(Negotiated::Plain(tcp)), + Fallback::Deny => Err(Error::tls(NO_TLS_MESSAGE)), + }; + } + other => return Err(unexpected_reply(other)), + } - /// Type alias for a TLS-wrapped TCP stream. - pub type TlsStream = tokio_rustls::client::TlsStream; + let stream = tokio_rustls::TlsConnector::from(Arc::clone(&connector.config)) + .connect(connector.server_name.clone(), tcp) + .await + .map_err(|e| negotiation_error(e, timeout))?; + Ok(Negotiated::Tls(stream)) + }; - /// Wraps a TCP stream with TLS. - /// - /// # Errors - /// - /// - Returns [`Error::Config`] if `server_name` is not a - /// valid DNS name or IP literal accepted by `rustls`. - /// - Returns [`Error::Connection`] if the TLS handshake with - /// the peer fails (certificate rejected, protocol error, I/O - /// failure). - pub async fn wrap_stream( - stream: TcpStream, - connector: &TlsConnector, - server_name: &str, - ) -> Result { - let domain = rustls::pki_types::ServerName::try_from(server_name.to_string()) - .map_err(|_| Error::config("invalid server name"))?; - - connector - .connect(domain, stream) + match timeout { + Some(limit) => tokio::time::timeout(limit, negotiation) .await - .map_err(|e| Error::connection(format!("TLS handshake failed: {e}"))) + .map_err(|_| timed_out(timeout))?, + None => negotiation.await, } } +fn cancel_connect_error(addr: &str, err: &io::Error) -> Error { + warn!( + target: "hyperdb_api", + addr = %addr, + error = %err, + "query-cancel-connect-failed" + ); + Error::connection(format!( + "failed to connect for cancel request to {addr}: {err}" + )) +} + +fn cancel_send_error(err: io::Error) -> Error { + warn!( + target: "hyperdb_api", + error = %err, + "query-cancel-send-failed" + ); + Error::from_io(err) +} + +/// Sends a `CancelRequest` for (`process_id`, `secret_key`) to the TCP +/// address `addr` (`host:port`), over TLS when `tls` is set. +/// +/// Blocking and runtime-free, because it also runs from `Drop`. Over TLS the +/// negotiation never falls back to plaintext and is bounded as a whole by +/// [`CANCEL_TLS_TIMEOUT`], as is each write of the request. After the +/// request the client sends `close_notify`, shuts down its write half and +/// drains the server's remaining bytes (session tickets) for up to a second, +/// so closing the socket with unread data cannot turn into a reset that +/// drops the request. Like the plaintext cancel, the TCP connect itself has +/// no timeout. +pub(crate) fn send_cancel_sync( + addr: &str, + process_id: i32, + secret_key: i32, + tls: Option<&TlsConnector>, +) -> Result<()> { + let mut tcp = TcpStream::connect(addr).map_err(|e| cancel_connect_error(addr, &e))?; + // Cancel is a 16-byte fire-and-forget — disable Nagle so the request + // hits the wire without waiting on a coalesce timer. + tcp.set_nodelay(true).ok(); + + let mut buf = BytesMut::with_capacity(16); + frontend::cancel_request(process_id, secret_key, &mut buf); + + let Some(connector) = tls else { + tcp.write_all(&buf).map_err(cancel_send_error)?; + return tcp.flush().map_err(Error::from_io); + }; + + let mut stream = match negotiate_sync(tcp, connector, Fallback::Deny, Some(CANCEL_TLS_TIMEOUT))? + { + Negotiated::Tls(stream) => stream, + Negotiated::Plain(_) => return Err(Error::tls(NO_TLS_MESSAGE)), + }; + stream + .sock + .set_write_timeout(Some(CANCEL_TLS_TIMEOUT)) + .map_err(Error::from_io)?; + stream.write_all(&buf).map_err(cancel_send_error)?; + stream.flush().map_err(cancel_send_error)?; + + // The request is out; the rest is a best-effort orderly close. + stream.conn.send_close_notify(); + while stream.conn.wants_write() { + if stream.conn.write_tls(&mut stream.sock).is_err() { + break; + } + } + stream.sock.shutdown(Shutdown::Write).ok(); + drain_sync(&mut stream.sock); + Ok(()) +} + +fn drain_sync(sock: &mut TcpStream) { + let deadline = Instant::now() + CANCEL_DRAIN_TIMEOUT; + let mut sink = [0u8; 4096]; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() || sock.set_read_timeout(Some(remaining)).is_err() { + return; + } + match sock.read(&mut sink) { + Ok(0) | Err(_) => return, + Ok(_) => {} + } + } +} + +/// Async [`send_cancel_sync`], with the same TLS behaviour. +pub(crate) async fn send_cancel_async( + addr: &str, + process_id: i32, + secret_key: i32, + tls: Option<&TlsConnector>, +) -> Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut tcp = tokio::net::TcpStream::connect(addr) + .await + .map_err(|e| cancel_connect_error(addr, &e))?; + // Cancel is a 16-byte fire-and-forget — disable Nagle so the request + // hits the wire without waiting on a coalesce timer. + tcp.set_nodelay(true).ok(); + + let mut buf = BytesMut::with_capacity(16); + frontend::cancel_request(process_id, secret_key, &mut buf); + + let Some(connector) = tls else { + tcp.write_all(&buf).await.map_err(cancel_send_error)?; + return tcp.flush().await.map_err(Error::from_io); + }; + + let mut stream = + match negotiate_async(tcp, connector, Fallback::Deny, Some(CANCEL_TLS_TIMEOUT)).await? { + Negotiated::Tls(stream) => stream, + Negotiated::Plain(_) => return Err(Error::tls(NO_TLS_MESSAGE)), + }; + let send = async { + stream.write_all(&buf).await?; + stream.flush().await + }; + tokio::time::timeout(CANCEL_TLS_TIMEOUT, send) + .await + .map_err(|_| Error::timeout("timed out sending the cancel request"))? + .map_err(cancel_send_error)?; + + // The request is out; the rest is a best-effort orderly close. + // `shutdown` sends `close_notify`, then shuts down the write half. + let close = async { + stream.shutdown().await?; + let mut sink = [0u8; 4096]; + while stream.read(&mut sink).await? > 0 {} + Ok::<(), io::Error>(()) + }; + let _ = tokio::time::timeout(CANCEL_DRAIN_TIMEOUT, close).await; + Ok(()) +} + #[cfg(test)] mod tests { + //! A fake server on a `std::net::TcpListener` thread checks that the + //! client sends an `SSLRequest`, answers `'S'`, `'N'` or `'E'`, runs a + //! rustls server on `'S'`, and reports what it received. This proves + //! what goes on the wire, which `hyperd` cannot: it accepts a plaintext + //! cancel even from a TLS session. + use super::*; + use std::net::{SocketAddr, TcpListener}; + use std::sync::mpsc; + use std::thread; + + use rustls::ServerConnection; + use rustls::server::{ServerConfig, WebPkiClientVerifier}; + use tempfile::NamedTempFile; + + struct TestCa { + issuer: rcgen::Issuer<'static, rcgen::KeyPair>, + cert_pem: String, + } + + struct TestCert { + cert_pem: String, + key_pem: String, + } + + fn generate_ca(name: &str) -> TestCa { + let mut params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); + params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params + .distinguished_name + .push(rcgen::DnType::CommonName, name); + let key_pair = rcgen::KeyPair::generate().unwrap(); + let cert = params.self_signed(&key_pair).unwrap(); + TestCa { + cert_pem: cert.pem(), + issuer: rcgen::Issuer::new(params, key_pair), + } + } + + /// A leaf signed by `ca`. `sans` may mix DNS names and IP literals. + fn generate_leaf(ca: &TestCa, sans: &[&str]) -> TestCert { + let sans = sans.iter().map(|s| (*s).to_string()).collect::>(); + let params = rcgen::CertificateParams::new(sans).unwrap(); + let key_pair = rcgen::KeyPair::generate().unwrap(); + let cert = params.signed_by(&key_pair, &ca.issuer).unwrap(); + TestCert { + cert_pem: cert.pem(), + key_pem: key_pair.serialize_pem(), + } + } + + fn pem_file(pem: &str) -> NamedTempFile { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(pem.as_bytes()).unwrap(); + file.flush().unwrap(); + file + } + + /// Server certificates for the common cases: a CA and a leaf for + /// `localhost` and `127.0.0.1`. + struct Pki { + ca: TestCa, + server: TestCert, + ca_file: NamedTempFile, + } + + fn pki() -> Pki { + let ca = generate_ca("Test CA"); + let server = generate_leaf(&ca, &["localhost", "127.0.0.1"]); + let ca_file = pem_file(&ca.cert_pem); + Pki { + ca, + server, + ca_file, + } + } + + fn server_config(server: &TestCert, client_ca_pem: Option<&str>) -> Arc { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let certs = CertificateDer::pem_slice_iter(server.cert_pem.as_bytes()) + .collect::, _>>() + .unwrap(); + let key = PrivateKeyDer::from_pem_slice(server.key_pem.as_bytes()).unwrap(); + let builder = ServerConfig::builder_with_provider(Arc::clone(&provider)) + .with_safe_default_protocol_versions() + .unwrap(); + let config = match client_ca_pem { + Some(ca_pem) => { + let mut roots = RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(ca_pem.as_bytes()) { + roots.add(cert.unwrap()).unwrap(); + } + let verifier = + WebPkiClientVerifier::builder_with_provider(Arc::new(roots), provider) + .build() + .unwrap(); + builder + .with_client_cert_verifier(verifier) + .with_single_cert(certs, key) + } + None => builder.with_no_client_auth().with_single_cert(certs, key), + }; + Arc::new(config.unwrap()) + } + + /// What the fake server saw after the `SSLRequest`. + #[derive(Debug)] + #[expect( + dead_code, + reason = "the diagnostic payloads are read only through Debug in assertion messages" + )] + enum Observed { + /// The first 8 bytes were not an `SSLRequest`. + NotSslRequest(Vec), + /// After `'N'`: every plaintext byte until EOF. + Plain(Vec), + /// After `'S'`: up to 16 decrypted bytes, and whether the client + /// then closed with `close_notify` (rather than a bare EOF). + Tls { data: Vec, close_notify: bool }, + /// After `'S'`: the server-side handshake failed. + HandshakeFailed(String), + /// After `'E'`: nothing further is read. + Refused, + } + + const IO_TIMEOUT: Duration = Duration::from_secs(5); + + /// Accepts one connection, answers its `SSLRequest` with `reply`, and + /// reports what followed. `tls` is required when `reply` is `b'S'`. + fn spawn_server( + reply: u8, + tls: Option>, + ) -> (SocketAddr, mpsc::Receiver) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = mpsc::channel(); + thread::spawn(move || { + let (mut sock, _) = listener.accept().unwrap(); + sock.set_read_timeout(Some(IO_TIMEOUT)).unwrap(); + sock.set_write_timeout(Some(IO_TIMEOUT)).unwrap(); + let mut request = [0u8; 8]; + if sock.read_exact(&mut request).is_err() || request != ssl_request_bytes()[..] { + tx.send(Observed::NotSslRequest(request.to_vec())).ok(); + return; + } + sock.write_all(&[reply]).unwrap(); + let observed = match reply { + b'S' => serve_tls(sock, tls.expect("TLS config for 'S'")), + b'N' => { + let mut data = Vec::new(); + sock.read_to_end(&mut data).ok(); + Observed::Plain(data) + } + _ => Observed::Refused, + }; + tx.send(observed).ok(); + }); + (addr, rx) + } + + fn serve_tls(mut sock: TcpStream, config: Arc) -> Observed { + let mut conn = ServerConnection::new(config).unwrap(); + while conn.is_handshaking() { + if let Err(e) = conn.complete_io(&mut sock) { + return Observed::HandshakeFailed(e.to_string()); + } + } + let mut stream = StreamOwned::new(conn, sock); + let mut data = vec![0u8; 16]; + let mut filled = 0; + while filled < data.len() { + match stream.read(&mut data[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(e) => return Observed::HandshakeFailed(e.to_string()), + } + } + data.truncate(filled); + // rustls reports a `close_notify` as a clean EOF and a bare TCP close + // as `UnexpectedEof`. + let mut rest = Vec::new(); + let close_notify = stream.read_to_end(&mut rest).is_ok(); + Observed::Tls { data, close_notify } + } + + fn observed(rx: &mpsc::Receiver) -> Observed { + rx.recv_timeout(Duration::from_secs(10)) + .expect("fake server reported nothing") + } + + const PAYLOAD: &[u8; 16] = b"sixteen-byte-msg"; + + fn connector(tls: &TlsConfig, host: &str) -> Arc { + TlsConnector::build(tls, host) + .expect("connector builds") + .expect("mode is enabled") + } + + /// Negotiates against `addr`, writes [`PAYLOAD`] over whatever was + /// negotiated, and closes. Returns whether TLS was negotiated. + fn negotiate_and_send(addr: SocketAddr, tls: &TlsConfig, host: &str) -> Result { + let tcp = TcpStream::connect(addr).unwrap(); + let negotiated = negotiate_sync( + tcp, + &connector(tls, host), + Fallback::for_mode(tls.mode()), + Some(IO_TIMEOUT), + )?; + match negotiated { + Negotiated::Plain(mut tcp) => { + tcp.write_all(PAYLOAD).unwrap(); + Ok(false) + } + Negotiated::Tls(mut stream) => { + stream.write_all(PAYLOAD).map_err(Error::from_io)?; + stream.flush().map_err(Error::from_io)?; + Ok(true) + } + } + } + + /// Runs one handshake against a fresh fake TLS server and returns the + /// client outcome together with what the server saw. + fn handshake(server: &TestCert, tls: &TlsConfig, host: &str) -> (Result, Observed) { + let (addr, rx) = spawn_server(b'S', Some(server_config(server, None))); + let outcome = negotiate_and_send(addr, tls, host); + (outcome, observed(&rx)) + } + + fn assert_payload_received(observed: &Observed) { + assert!( + matches!(observed, Observed::Tls { data, .. } if data == PAYLOAD), + "server saw {observed:?}" + ); + } + + // ---- Modes and verification ------------------------------------------ + + #[test] + fn require_accepts_unknown_ca_without_root_cert() { + let pki = pki(); + let (outcome, seen) = handshake(&pki.server, &TlsMode::Require.into(), "localhost"); + assert!(outcome.expect("Require without root cert accepts any cert")); + assert_payload_received(&seen); + } + + #[test] + fn require_with_wrong_root_cert_checks_the_chain() { + let pki = pki(); + let other = pem_file(&generate_ca("Other CA").cert_pem); + let tls = TlsConfig::new(TlsMode::Require).root_cert(other.path()); + let (outcome, _) = handshake(&pki.server, &tls, "localhost"); + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + } + + #[test] + fn verify_ca_accepts_right_ca() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyCa).root_cert(pki.ca_file.path()); + let (outcome, seen) = handshake(&pki.server, &tls, "localhost"); + assert!(outcome.unwrap()); + assert_payload_received(&seen); + } + + #[test] + fn verify_ca_rejects_wrong_ca() { + let pki = pki(); + let other = pem_file(&generate_ca("Other CA").cert_pem); + let tls = TlsConfig::new(TlsMode::VerifyCa).root_cert(other.path()); + let (outcome, _) = handshake(&pki.server, &tls, "localhost"); + let err = outcome.unwrap_err(); + assert!(matches!(err, Error::Tls(_)), "got {err:?}"); + assert!(err.to_string().contains("UnknownIssuer"), "got {err}"); + } + + #[test] + fn verify_ca_ignores_the_host_name() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyCa).root_cert(pki.ca_file.path()); + let (outcome, seen) = handshake(&pki.server, &tls, "wrong.example"); + assert!(outcome.unwrap()); + assert_payload_received(&seen); + } + + #[test] + fn verify_ca_without_root_cert_is_config_error() { + let err = TlsConnector::build(&TlsMode::VerifyCa.into(), "localhost").unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + } + + #[test] + fn verify_full_accepts_matching_host() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyFull).root_cert(pki.ca_file.path()); + let (outcome, seen) = handshake(&pki.server, &tls, "localhost"); + assert!(outcome.unwrap()); + assert_payload_received(&seen); + } #[test] - fn test_tls_config_default() { - let config = TlsConfig::default(); - assert!(config.verify_server); - assert!(config.ca_cert_path.is_none()); - assert!(!config.has_client_cert()); + fn verify_full_rejects_wrong_host() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyFull).root_cert(pki.ca_file.path()); + let (outcome, _) = handshake(&pki.server, &tls, "wrong.example"); + let err = outcome.unwrap_err(); + assert!(matches!(err, Error::Tls(_)), "got {err:?}"); + assert!(err.to_string().contains("not valid for name"), "got {err}"); } #[test] - fn test_tls_config_builder() { - let config = TlsConfig::new() - .ca_cert("/path/to/ca.pem") - .client_cert("/path/to/cert.pem", "/path/to/key.pem") - .server_name("example.com"); + fn verify_full_server_name_override_replaces_host() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyFull) + .root_cert(pki.ca_file.path()) + .server_name("localhost"); + let (outcome, seen) = handshake(&pki.server, &tls, "wrong.example"); + assert!(outcome.unwrap()); + assert_payload_received(&seen); - assert!(config.has_client_cert()); - assert_eq!(config.server_name, Some("example.com".to_string())); + // And the override is what gets checked. + let tls = TlsConfig::new(TlsMode::VerifyFull) + .root_cert(pki.ca_file.path()) + .server_name("wrong.example"); + let (outcome, _) = handshake(&pki.server, &tls, "localhost"); + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + } + + #[test] + fn verify_full_checks_ip_host_against_ip_san() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyFull).root_cert(pki.ca_file.path()); + let (outcome, seen) = handshake(&pki.server, &tls, "127.0.0.1"); + assert!(outcome.unwrap()); + assert_payload_received(&seen); + + // An IP the certificate does not name fails. + let (outcome, _) = handshake(&pki.server, &tls, "127.0.0.2"); + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + } + + #[test] + fn verify_full_without_root_cert_uses_bundled_roots() { + // The test CA is not a Mozilla root, so the chain fails; this proves + // the bundled store is the one in use rather than "no verification". + let pki = pki(); + let (outcome, _) = handshake(&pki.server, &TlsMode::VerifyFull.into(), "localhost"); + let err = outcome.unwrap_err(); + assert!(err.to_string().contains("UnknownIssuer"), "got {err}"); + } + + #[test] + fn ipv6_host_brackets_are_stripped() { + let tls = TlsConfig::from(TlsMode::Require); + let built = connector(&tls, "[::1]"); + assert!( + matches!(built.server_name, ServerName::IpAddress(_)), + "got {:?}", + built.server_name + ); + let built = connector(&tls, "localhost"); + assert!(matches!(built.server_name, ServerName::DnsName(_))); + let err = TlsConnector::build(&tls, "not a host").unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + } + + #[test] + fn disable_builds_no_connector() { + assert!( + TlsConnector::build(&TlsConfig::default(), "localhost") + .unwrap() + .is_none() + ); + } + + // ---- Server answers and fallback -------------------------------------- + + #[test] + fn prefer_does_not_downgrade_after_failed_verification() { + let pki = pki(); + let other = pem_file(&generate_ca("Other CA").cert_pem); + let tls = TlsConfig::new(TlsMode::Prefer).root_cert(other.path()); + let (outcome, seen) = handshake(&pki.server, &tls, "localhost"); + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + assert!( + matches!(seen, Observed::HandshakeFailed(_)), + "server saw {seen:?}" + ); + } + + #[test] + fn prefer_falls_back_to_plaintext_on_n() { + let (addr, rx) = spawn_server(b'N', None); + let negotiated_tls = negotiate_and_send(addr, &TlsMode::Prefer.into(), "localhost"); + assert!(!negotiated_tls.unwrap(), "Prefer + 'N' must be plaintext"); + assert!( + matches!(observed(&rx), Observed::Plain(ref data) if data == PAYLOAD), + "the payload follows in plaintext on the same socket" + ); } #[test] - fn test_tls_mode() { - assert!(!TlsMode::Disable.is_enabled()); - assert!(TlsMode::Require.is_required()); - assert!(TlsMode::VerifyFull.verify_hostname()); + fn require_fails_on_n() { + let (addr, rx) = spawn_server(b'N', None); + let outcome = negotiate_and_send(addr, &TlsMode::Require.into(), "localhost"); + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + assert!( + matches!(observed(&rx), Observed::Plain(ref data) if data.is_empty()), + "nothing may be sent in plaintext" + ); + } + + #[test] + fn unexpected_reply_is_protocol_error() { + let (addr, rx) = spawn_server(b'E', None); + let outcome = negotiate_and_send(addr, &TlsMode::Require.into(), "localhost"); + assert!( + matches!(outcome, Err(Error::Protocol(_))), + "got {outcome:?}" + ); + assert!(matches!(observed(&rx), Observed::Refused)); + } + + #[test] + fn fallback_for_mode() { + assert_eq!( + Fallback::for_mode(TlsMode::Prefer), + Fallback::AllowPlaintext + ); + for mode in [TlsMode::Require, TlsMode::VerifyCa, TlsMode::VerifyFull] { + assert_eq!(Fallback::for_mode(mode), Fallback::Deny, "{mode}"); + } + } + + #[test] + fn negotiation_times_out_on_a_silent_server() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let holder = thread::spawn(move || { + let (sock, _) = listener.accept().unwrap(); + thread::sleep(Duration::from_secs(3)); + drop(sock); + }); + let started = Instant::now(); + let outcome = negotiate_sync( + TcpStream::connect(addr).unwrap(), + &connector(&TlsMode::Require.into(), "localhost"), + Fallback::Deny, + Some(Duration::from_millis(500)), + ); + let elapsed = started.elapsed(); + assert!( + matches!(outcome, Err(Error::Timeout(_))), + "got {:?}", + outcome.err() + ); + assert!(elapsed < Duration::from_secs(2), "took {elapsed:?}"); + holder.join().unwrap(); + } + + /// The timeout bounds the negotiation as a whole: a server that keeps + /// every single read inside the limit still cannot stretch it. + #[test] + fn negotiation_deadline_holds_against_a_dripping_server() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let dripper = thread::spawn(move || { + let (mut sock, _) = listener.accept().unwrap(); + let mut request = [0u8; 8]; + sock.read_exact(&mut request).unwrap(); + // 'S', then the header of a handshake record announcing 16 KiB, + // whose body arrives one byte at a time: rustls keeps waiting + // for the rest of the record. + sock.write_all(&[b'S', 0x16, 0x03, 0x03, 0x40, 0x00]) + .unwrap(); + for _ in 0..20 { + thread::sleep(Duration::from_millis(150)); + if sock.write_all(&[0]).is_err() { + break; + } + } + }); + let started = Instant::now(); + let outcome = negotiate_sync( + TcpStream::connect(addr).unwrap(), + &connector(&TlsMode::Require.into(), "localhost"), + Fallback::Deny, + Some(Duration::from_millis(600)), + ); + let elapsed = started.elapsed(); + let err = outcome.err().expect("negotiation must time out"); + assert!(matches!(err, Error::Timeout(_)), "got {err:?}"); + assert_eq!(err.to_string(), "TLS negotiation timed out after 600ms"); + assert!(elapsed < Duration::from_millis(1500), "took {elapsed:?}"); + dripper.join().unwrap(); + } + + #[test] + fn zero_timeout_is_a_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let outcome = negotiate_sync( + TcpStream::connect(listener.local_addr().unwrap()).unwrap(), + &connector(&TlsMode::Require.into(), "localhost"), + Fallback::Deny, + Some(Duration::ZERO), + ); + assert!( + matches!(outcome, Err(Error::Timeout(_))), + "got {:?}", + outcome.err() + ); + } + + // ---- Mutual TLS ------------------------------------------------------- + + #[test] + fn mtls_client_cert_from_the_ca_is_accepted() { + let pki = pki(); + let client = generate_leaf(&pki.ca, &["client"]); + let (cert, key) = (pem_file(&client.cert_pem), pem_file(&client.key_pem)); + let (addr, rx) = spawn_server( + b'S', + Some(server_config(&pki.server, Some(&pki.ca.cert_pem))), + ); + let tls = TlsConfig::new(TlsMode::VerifyFull) + .root_cert(pki.ca_file.path()) + .client_cert(cert.path(), key.path()); + assert!(negotiate_and_send(addr, &tls, "localhost").unwrap()); + assert_payload_received(&observed(&rx)); + } + + #[test] + fn mtls_missing_client_cert_fails_on_first_read_as_tls() { + let pki = pki(); + let (addr, rx) = spawn_server( + b'S', + Some(server_config(&pki.server, Some(&pki.ca.cert_pem))), + ); + let tls = TlsConfig::new(TlsMode::VerifyFull).root_cert(pki.ca_file.path()); + // Under TLS 1.3 the client's handshake completes before the server + // has looked at the (absent) certificate; the rejection arrives as an + // alert on the first read. + let Negotiated::Tls(mut stream) = negotiate_sync( + TcpStream::connect(addr).unwrap(), + &connector(&tls, "localhost"), + Fallback::Deny, + Some(IO_TIMEOUT), + ) + .expect("the client side of the handshake completes") else { + panic!("expected TLS"); + }; + let err = stream + .read(&mut [0u8; 1]) + .map_err(Error::from_io) + .unwrap_err(); + assert!(matches!(err, Error::Tls(_)), "got {err:?}"); + assert!( + matches!(observed(&rx), Observed::HandshakeFailed(_)), + "server must reject the connection" + ); + } + + // ---- Cancel senders --------------------------------------------------- + + fn assert_cancel_received(observed: &Observed) { + let Observed::Tls { data, close_notify } = observed else { + panic!("server saw {observed:?}"); + }; + let mut expected = BytesMut::new(); + frontend::cancel_request(4242, -7, &mut expected); + assert_eq!(data[..], expected[..], "decrypted CancelRequest"); + assert!(*close_notify, "cancel must end with close_notify"); + } + + #[test] + fn cancel_sync_sends_the_request_over_tls() { + let pki = pki(); + let (addr, rx) = spawn_server(b'S', Some(server_config(&pki.server, None))); + let tls = connector(&TlsMode::Require.into(), "localhost"); + send_cancel_sync(&addr.to_string(), 4242, -7, Some(&tls)).unwrap(); + assert_cancel_received(&observed(&rx)); + } + + #[test] + fn cancel_sync_never_falls_back_to_plaintext() { + let (addr, rx) = spawn_server(b'N', None); + // Even a connector built for `Prefer` must not fall back. + let tls = connector(&TlsMode::Prefer.into(), "localhost"); + let err = send_cancel_sync(&addr.to_string(), 4242, -7, Some(&tls)).unwrap_err(); + assert!(matches!(err, Error::Tls(_)), "got {err:?}"); + assert!( + matches!(observed(&rx), Observed::Plain(ref data) if data.is_empty()), + "no plaintext CancelRequest may follow" + ); + } + + #[test] + fn cancel_sync_without_tls_sends_plaintext() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let reader = thread::spawn(move || { + let (mut sock, _) = listener.accept().unwrap(); + let mut data = Vec::new(); + sock.read_to_end(&mut data).unwrap(); + data + }); + send_cancel_sync(&addr.to_string(), 4242, -7, None).unwrap(); + let mut expected = BytesMut::new(); + frontend::cancel_request(4242, -7, &mut expected); + assert_eq!(reader.join().unwrap(), expected[..]); + } + + #[tokio::test] + async fn cancel_async_sends_the_request_over_tls() { + let pki = pki(); + let (addr, rx) = spawn_server(b'S', Some(server_config(&pki.server, None))); + let tls = connector(&TlsMode::Require.into(), "localhost"); + send_cancel_async(&addr.to_string(), 4242, -7, Some(&tls)) + .await + .unwrap(); + assert_cancel_received(&observed(&rx)); + } + + #[tokio::test] + async fn cancel_async_never_falls_back_to_plaintext() { + let (addr, rx) = spawn_server(b'N', None); + let tls = connector(&TlsMode::Prefer.into(), "localhost"); + let err = send_cancel_async(&addr.to_string(), 4242, -7, Some(&tls)) + .await + .unwrap_err(); + assert!(matches!(err, Error::Tls(_)), "got {err:?}"); + assert!( + matches!(observed(&rx), Observed::Plain(ref data) if data.is_empty()), + "no plaintext CancelRequest may follow" + ); + } + + // ---- Async negotiation ------------------------------------------------ + + async fn negotiate_async_and_send( + addr: SocketAddr, + tls: &TlsConfig, + host: &str, + ) -> Result { + use tokio::io::AsyncWriteExt; + + let tcp = tokio::net::TcpStream::connect(addr).await.unwrap(); + let negotiated = negotiate_async( + tcp, + &connector(tls, host), + Fallback::for_mode(tls.mode()), + Some(IO_TIMEOUT), + ) + .await?; + match negotiated { + Negotiated::Plain(mut tcp) => { + tcp.write_all(PAYLOAD).await.unwrap(); + Ok(false) + } + Negotiated::Tls(mut stream) => { + stream.write_all(PAYLOAD).await.map_err(Error::from_io)?; + stream.flush().await.map_err(Error::from_io)?; + Ok(true) + } + } + } + + #[tokio::test] + async fn negotiate_async_require() { + let pki = pki(); + let (addr, rx) = spawn_server(b'S', Some(server_config(&pki.server, None))); + let tls = TlsConfig::new(TlsMode::Require); + assert!( + negotiate_async_and_send(addr, &tls, "localhost") + .await + .unwrap() + ); + assert_payload_received(&observed(&rx)); + } + + #[tokio::test] + async fn negotiate_async_verify_full() { + let pki = pki(); + let tls = TlsConfig::new(TlsMode::VerifyFull).root_cert(pki.ca_file.path()); + + let (addr, rx) = spawn_server(b'S', Some(server_config(&pki.server, None))); + assert!( + negotiate_async_and_send(addr, &tls, "localhost") + .await + .unwrap() + ); + assert_payload_received(&observed(&rx)); + + let (addr, _rx) = spawn_server(b'S', Some(server_config(&pki.server, None))); + let outcome = negotiate_async_and_send(addr, &tls, "wrong.example").await; + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + } + + // ---- Configuration ---------------------------------------------------- + + #[test] + fn tls_mode_parses_and_displays_libpq_names() { + for mode in [ + TlsMode::Disable, + TlsMode::Prefer, + TlsMode::Require, + TlsMode::VerifyCa, + TlsMode::VerifyFull, + ] { + assert_eq!(mode.to_string().parse::(), Ok(mode)); + } + assert_eq!(TlsMode::VerifyCa.to_string(), "verify-ca"); + assert_eq!(TlsMode::default(), TlsMode::Disable); + + let err = "verify_full".parse::().unwrap_err(); + let message = err.to_string(); + assert!(message.contains("`verify_full`"), "got {message}"); + for name in ["disable", "prefer", "require", "verify-ca", "verify-full"] { + assert!(message.contains(name), "{name} missing from {message}"); + } + } + + #[test] + fn tls_config_builder_and_from_mode() { + let tls = TlsConfig::new(TlsMode::VerifyFull) + .root_cert("/ca.pem") + .client_cert("/c.pem", "/c.key") + .server_name("db.internal"); + assert_eq!(tls.mode(), TlsMode::VerifyFull); + assert_eq!(tls.root_cert.as_deref(), Some(Path::new("/ca.pem"))); + assert_eq!( + tls.client_cert, + Some((PathBuf::from("/c.pem"), PathBuf::from("/c.key"))) + ); + assert_eq!(tls.server_name.as_deref(), Some("db.internal")); + assert_eq!( + TlsConfig::from(TlsMode::Require), + TlsConfig::new(TlsMode::Require) + ); + assert_eq!(TlsConfig::default().mode(), TlsMode::Disable); + } + + #[test] + fn missing_or_empty_pem_files_are_config_errors() { + let missing = TlsConfig::new(TlsMode::VerifyCa).root_cert("/nonexistent/ca.pem"); + let err = TlsConnector::build(&missing, "localhost").unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + assert!(err.to_string().contains("/nonexistent/ca.pem"), "got {err}"); + + let empty = pem_file(""); + let err = TlsConnector::build( + &TlsConfig::new(TlsMode::VerifyCa).root_cert(empty.path()), + "localhost", + ) + .unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + assert!(err.to_string().contains("no PEM certificate"), "got {err}"); + + let pki = pki(); + let cert = pem_file(&pki.server.cert_pem); + let err = TlsConnector::build( + &TlsConfig::new(TlsMode::Require).client_cert(cert.path(), empty.path()), + "localhost", + ) + .unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); } } diff --git a/hyperdb-api-core/src/protocol/message/frontend.rs b/hyperdb-api-core/src/protocol/message/frontend.rs index a557a374..e3c99afb 100644 --- a/hyperdb-api-core/src/protocol/message/frontend.rs +++ b/hyperdb-api-core/src/protocol/message/frontend.rs @@ -486,6 +486,17 @@ pub fn cancel_request(process_id: i32, secret_key: i32, buf: &mut BytesMut) { buf.put_i32(secret_key); } +/// Writes an `SSLRequest`. +/// +/// Sent on a fresh TCP connection in place of the startup message. The +/// server answers with a single byte: `'S'` (proceed with a TLS handshake) +/// or `'N'` (TLS not available), after which the startup message follows, +/// encrypted or not. +pub fn ssl_request(buf: &mut BytesMut) { + buf.put_i32(8); // Length + buf.put_i32(80877103); // SSL request code +} + /// Writes a copy data message. /// /// Sends a chunk of COPY data to the server during COPY IN operation. @@ -525,3 +536,28 @@ pub fn copy_fail(message: &str, buf: &mut BytesMut) { buf.put_slice(message.as_bytes()); buf.put_u8(0); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ssl_request_bytes() { + let mut buf = BytesMut::new(); + ssl_request(&mut buf); + assert_eq!(&buf[..], &[0x00, 0x00, 0x00, 0x08, 0x04, 0xD2, 0x16, 0x2F]); + } + + #[test] + fn cancel_request_bytes() { + let mut buf = BytesMut::new(); + cancel_request(7, -2, &mut buf); + assert_eq!( + &buf[..], + &[ + 0x00, 0x00, 0x00, 0x10, 0x04, 0xD2, 0x16, 0x2E, 0x00, 0x00, 0x00, 0x07, 0xFF, 0xFF, + 0xFF, 0xFE, + ] + ); + } +} diff --git a/hyperdb-api-core/tests/tls_tests.rs b/hyperdb-api-core/tests/tls_tests.rs deleted file mode 100644 index 7ff55ccc..00000000 --- a/hyperdb-api-core/tests/tls_tests.rs +++ /dev/null @@ -1,412 +0,0 @@ -// Copyright (c) 2026, Salesforce, Inc. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! Integration tests for TLS support. -//! -//! These tests verify the rustls backend by: -//! 1. Generating self-signed certificates at test time using `rcgen` -//! 2. Starting a local TLS echo server -//! 3. Connecting through `hyperdb_api_core::client::tls` module functions -//! 4. Verifying data can be exchanged over the encrypted connection -//! -//! Run with: -//! cargo test --test `tls_tests` - -use std::io::Write; -use tempfile::NamedTempFile; - -use hyperdb_api_core::client::tls::TlsConfig; - -/// A CA: its params (needed as issuer reference), key pair, and self-signed cert PEM. -struct TestCa { - params: rcgen::CertificateParams, - key_pair: rcgen::KeyPair, - cert_pem: String, -} - -/// A leaf cert: its key pair and cert PEM. -struct TestCert { - key_pair: rcgen::KeyPair, - cert_pem: String, -} - -/// Generate a self-signed CA. -fn generate_ca() -> TestCa { - let mut params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); - params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); - params - .distinguished_name - .push(rcgen::DnType::CommonName, "Test CA"); - let key_pair = rcgen::KeyPair::generate().unwrap(); - let cert = params.self_signed(&key_pair).unwrap(); - let cert_pem = cert.pem(); - TestCa { - params, - key_pair, - cert_pem, - } -} - -/// Generate a server certificate signed by the CA. -fn generate_server_cert(ca: &TestCa, san: &str) -> TestCert { - let params = rcgen::CertificateParams::new(vec![san.to_string()]).unwrap(); - let key_pair = rcgen::KeyPair::generate().unwrap(); - let issuer = rcgen::Issuer::from_params(&ca.params, &ca.key_pair); - let cert = params.signed_by(&key_pair, &issuer).unwrap(); - TestCert { - key_pair, - cert_pem: cert.pem(), - } -} - -/// Generate a client certificate signed by the CA. -fn generate_client_cert(ca: &TestCa) -> TestCert { - let mut params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); - params - .distinguished_name - .push(rcgen::DnType::CommonName, "Test Client"); - let key_pair = rcgen::KeyPair::generate().unwrap(); - let issuer = rcgen::Issuer::from_params(&ca.params, &ca.key_pair); - let cert = params.signed_by(&key_pair, &issuer).unwrap(); - TestCert { - key_pair, - cert_pem: cert.pem(), - } -} - -/// Write PEM data to a temp file. The returned handle keeps the file alive. -fn write_pem_file(pem: &str) -> NamedTempFile { - let mut f = NamedTempFile::new().unwrap(); - f.write_all(pem.as_bytes()).unwrap(); - f.flush().unwrap(); - f -} - -// ============================================================================= -// rustls tests -// ============================================================================= - -mod rustls_tests { - use super::*; - use hyperdb_api_core::client::tls::rustls_impl; - use rustls::pki_types::pem::PemObject; - use rustls::pki_types::{CertificateDer, PrivateKeyDer}; - use std::sync::Arc; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::net::{TcpListener, TcpStream}; - use tokio_rustls::TlsAcceptor; - - /// Start a TLS echo server using rustls. Returns (addr, `join_handle`). - /// The server accepts one connection, reads data, and echoes it back prefixed with "echo:". - async fn start_echo_server( - server_cert_pem: &str, - server_key_pem: &str, - require_client_cert_ca_pem: Option<&str>, - ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { - let provider = Arc::new(rustls::crypto::ring::default_provider()); - - let certs: Vec> = - CertificateDer::pem_slice_iter(server_cert_pem.as_bytes()) - .map(|r| r.unwrap()) - .collect(); - let key = PrivateKeyDer::from_pem_slice(server_key_pem.as_bytes()).unwrap(); - - let server_config = if let Some(ca_pem) = require_client_cert_ca_pem { - let mut root_store = rustls::RootCertStore::empty(); - let ca_certs: Vec> = - CertificateDer::pem_slice_iter(ca_pem.as_bytes()) - .map(|r| r.unwrap()) - .collect(); - for cert in ca_certs { - root_store.add(cert).unwrap(); - } - let verifier = rustls::server::WebPkiClientVerifier::builder_with_provider( - Arc::new(root_store), - Arc::clone(&provider), - ) - .build() - .unwrap(); - rustls::ServerConfig::builder_with_provider(Arc::clone(&provider)) - .with_safe_default_protocol_versions() - .unwrap() - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .unwrap() - } else { - rustls::ServerConfig::builder_with_provider(Arc::clone(&provider)) - .with_safe_default_protocol_versions() - .unwrap() - .with_no_client_auth() - .with_single_cert(certs, key) - .unwrap() - }; - - let acceptor = TlsAcceptor::from(Arc::new(server_config)); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - - let handle = tokio::spawn(async move { - if let Ok((stream, _)) = listener.accept().await - && let Ok(mut tls_stream) = acceptor.accept(stream).await - { - let mut buf = [0u8; 1024]; - if let Ok(n) = tls_stream.read(&mut buf).await { - let msg = std::str::from_utf8(&buf[..n]).unwrap_or("?"); - let response = format!("echo:{msg}"); - let _ = tls_stream.write_all(response.as_bytes()).await; - let _ = tls_stream.shutdown().await; - } - } - }); - - (addr, handle) - } - - #[tokio::test] - async fn test_rustls_connector_creation_with_ca() { - let ca = generate_ca(); - let ca_pem_file = write_pem_file(&ca.cert_pem); - - let config = TlsConfig::new().ca_cert(ca_pem_file.path()); - let connector = rustls_impl::create_connector(&config, "localhost"); - assert!( - connector.is_ok(), - "Failed to create rustls connector: {:?}", - connector.err() - ); - } - - #[tokio::test] - async fn test_rustls_connector_creation_system_roots() { - let config = TlsConfig::new(); - let connector = rustls_impl::create_connector(&config, "localhost"); - assert!( - connector.is_ok(), - "Failed to create connector with system roots: {:?}", - connector.err() - ); - } - - #[tokio::test] - async fn test_rustls_tls_handshake_and_data_exchange() { - let ca = generate_ca(); - let server = generate_server_cert(&ca, "localhost"); - - let (addr, server_handle) = - start_echo_server(&server.cert_pem, &server.key_pair.serialize_pem(), None).await; - - let ca_pem_file = write_pem_file(&ca.cert_pem); - let config = TlsConfig::new().ca_cert(ca_pem_file.path()); - let connector = rustls_impl::create_connector(&config, "localhost").unwrap(); - - let tcp_stream = TcpStream::connect(addr).await.unwrap(); - let mut tls_stream = rustls_impl::wrap_stream(tcp_stream, &connector, "localhost") - .await - .expect("TLS handshake should succeed"); - - tls_stream.write_all(b"hello TLS").await.unwrap(); - tls_stream.shutdown().await.unwrap(); - - let mut response = String::new(); - tls_stream.read_to_string(&mut response).await.unwrap(); - assert_eq!(response, "echo:hello TLS"); - - server_handle.await.unwrap(); - } - - #[tokio::test] - async fn test_rustls_rejects_unknown_ca() { - let ca = generate_ca(); - let server = generate_server_cert(&ca, "localhost"); - - let (addr, _server_handle) = - start_echo_server(&server.cert_pem, &server.key_pair.serialize_pem(), None).await; - - // Use a different CA that didn't sign the server cert - let wrong_ca = generate_ca(); - let wrong_ca_file = write_pem_file(&wrong_ca.cert_pem); - let config = TlsConfig::new().ca_cert(wrong_ca_file.path()); - let connector = rustls_impl::create_connector(&config, "localhost").unwrap(); - - let tcp_stream = TcpStream::connect(addr).await.unwrap(); - let result = rustls_impl::wrap_stream(tcp_stream, &connector, "localhost").await; - assert!( - result.is_err(), - "Should reject connection signed by unknown CA" - ); - } - - #[tokio::test] - async fn test_rustls_rejects_wrong_hostname() { - let ca = generate_ca(); - let server = generate_server_cert(&ca, "localhost"); - - let (addr, _server_handle) = - start_echo_server(&server.cert_pem, &server.key_pair.serialize_pem(), None).await; - - let ca_pem_file = write_pem_file(&ca.cert_pem); - let config = TlsConfig::new().ca_cert(ca_pem_file.path()); - let connector = rustls_impl::create_connector(&config, "wrong.host.name").unwrap(); - - let tcp_stream = TcpStream::connect(addr).await.unwrap(); - let result = rustls_impl::wrap_stream(tcp_stream, &connector, "wrong.host.name").await; - assert!( - result.is_err(), - "Should reject connection when server name doesn't match cert SAN" - ); - } - - #[tokio::test] - async fn test_rustls_mutual_tls() { - let ca = generate_ca(); - let server = generate_server_cert(&ca, "localhost"); - let client = generate_client_cert(&ca); - - // Server requires client certificates (mTLS) - let (addr, server_handle) = start_echo_server( - &server.cert_pem, - &server.key_pair.serialize_pem(), - Some(&ca.cert_pem), - ) - .await; - - let ca_pem_file = write_pem_file(&ca.cert_pem); - let client_cert_file = write_pem_file(&client.cert_pem); - let client_key_file = write_pem_file(&client.key_pair.serialize_pem()); - - let config = TlsConfig::new() - .ca_cert(ca_pem_file.path()) - .client_cert(client_cert_file.path(), client_key_file.path()); - let connector = rustls_impl::create_connector(&config, "localhost").unwrap(); - - let tcp_stream = TcpStream::connect(addr).await.unwrap(); - let mut tls_stream = rustls_impl::wrap_stream(tcp_stream, &connector, "localhost") - .await - .expect("mTLS handshake should succeed"); - - tls_stream.write_all(b"mtls works").await.unwrap(); - tls_stream.shutdown().await.unwrap(); - - let mut response = String::new(); - tls_stream.read_to_string(&mut response).await.unwrap(); - assert_eq!(response, "echo:mtls works"); - - server_handle.await.unwrap(); - } - - #[tokio::test] - async fn test_rustls_mtls_rejected_without_client_cert() { - let ca = generate_ca(); - let server = generate_server_cert(&ca, "localhost"); - - // Server requires client certs (mTLS) - let (addr, _server_handle) = start_echo_server( - &server.cert_pem, - &server.key_pair.serialize_pem(), - Some(&ca.cert_pem), - ) - .await; - - // Client omits client certificate - let ca_pem_file = write_pem_file(&ca.cert_pem); - let config = TlsConfig::new().ca_cert(ca_pem_file.path()); - let connector = rustls_impl::create_connector(&config, "localhost").unwrap(); - - let tcp_stream = TcpStream::connect(addr).await.unwrap(); - let result = rustls_impl::wrap_stream(tcp_stream, &connector, "localhost").await; - - // The handshake may fail outright, or succeed but the server drops the connection - if let Ok(mut stream) = result { - let write_result = stream.write_all(b"test").await; - let mut buf = [0u8; 1024]; - let read_result = stream.read(&mut buf).await; - assert!( - write_result.is_err() || read_result.is_err() || read_result.unwrap_or(0) == 0, - "Server should reject connection without client cert" - ); - } - } -} - -// ============================================================================= -// TlsConfig unit tests (no feature gates needed) -// ============================================================================= - -mod config_tests { - use super::*; - use hyperdb_api_core::client::tls::TlsMode; - - #[test] - fn test_tls_config_default_values() { - let config = TlsConfig::default(); - assert!(config.verify_server); - assert!(config.ca_cert_path.is_none()); - assert!(config.client_cert_path.is_none()); - assert!(config.client_key_path.is_none()); - assert!(config.server_name.is_none()); - assert!(!config.has_client_cert()); - } - - #[test] - fn test_tls_config_builder_pattern() { - let ca = generate_ca(); - let client = generate_client_cert(&ca); - - let ca_file = write_pem_file(&ca.cert_pem); - let cert_file = write_pem_file(&client.cert_pem); - let key_file = write_pem_file(&client.key_pair.serialize_pem()); - - let config = TlsConfig::new() - .ca_cert(ca_file.path()) - .client_cert(cert_file.path(), key_file.path()) - .server_name("example.com"); - - assert!(config.verify_server); - assert!(config.ca_cert_path.is_some()); - assert!(config.has_client_cert()); - assert_eq!(config.server_name, Some("example.com".to_string())); - } - - #[test] - fn test_tls_config_danger_accept_invalid_certs() { - let config = TlsConfig::new().danger_accept_invalid_certs(); - assert!(!config.verify_server); - } - - #[test] - fn test_tls_mode_disable() { - assert!(!TlsMode::Disable.is_enabled()); - assert!(!TlsMode::Disable.is_required()); - assert!(!TlsMode::Disable.verify_server()); - assert!(!TlsMode::Disable.verify_hostname()); - } - - #[test] - fn test_tls_mode_prefer() { - assert!(TlsMode::Prefer.is_enabled()); - assert!(!TlsMode::Prefer.is_required()); - } - - #[test] - fn test_tls_mode_require() { - assert!(TlsMode::Require.is_enabled()); - assert!(TlsMode::Require.is_required()); - assert!(!TlsMode::Require.verify_server()); - } - - #[test] - fn test_tls_mode_verify_ca() { - assert!(TlsMode::VerifyCA.is_enabled()); - assert!(TlsMode::VerifyCA.is_required()); - assert!(TlsMode::VerifyCA.verify_server()); - assert!(!TlsMode::VerifyCA.verify_hostname()); - } - - #[test] - fn test_tls_mode_verify_full() { - assert!(TlsMode::VerifyFull.is_enabled()); - assert!(TlsMode::VerifyFull.is_required()); - assert!(TlsMode::VerifyFull.verify_server()); - assert!(TlsMode::VerifyFull.verify_hostname()); - } -} diff --git a/hyperdb-api/src/error.rs b/hyperdb-api/src/error.rs index 433f9b94..332da34d 100644 --- a/hyperdb-api/src/error.rs +++ b/hyperdb-api/src/error.rs @@ -66,7 +66,12 @@ pub enum Error { #[error("authentication failed: {0}")] Authentication(String), - /// TLS handshake or configuration failure. + /// TLS negotiation, handshake or certificate-verification failure. + /// + /// Includes a server that refuses TLS when the configured + /// [`TlsMode`](crate::TlsMode) requires it, and a server rejecting the + /// client certificate. Unreadable certificate files and invalid TLS + /// option combinations are [`Self::Config`] instead. #[error("TLS error: {0}")] Tls(String), @@ -540,6 +545,7 @@ impl From for Error { sqlstate: None, }, CoreError::Config(_) => Error::Config(chain), + CoreError::Tls(_) => Error::Tls(chain), CoreError::Timeout(_) => Error::Timeout(chain), CoreError::Cancelled { sqlstate, .. } => Error::Cancelled { message: chain, @@ -650,6 +656,7 @@ mod tests { CoreError::protocol("test message"), CoreError::io("test message"), CoreError::config("test message"), + CoreError::tls("test message"), CoreError::timeout("test message"), CoreError::cancelled("test message"), CoreError::closed("test message"), @@ -667,6 +674,16 @@ mod tests { } } + #[test] + fn from_client_tls_error_stays_tls() { + let public: Error = CoreError::tls("unknown issuer").into(); + assert!( + matches!(public, Error::Tls(ref m) if m == "unknown issuer"), + "got {public:?}" + ); + assert_eq!(public.to_string(), "TLS error: unknown issuer"); + } + #[test] fn sqlstate_returns_some_for_server_connection_closed_cancelled() { // Server still surfaces SQLSTATE. diff --git a/hyperdb-api/src/lib.rs b/hyperdb-api/src/lib.rs index 79caf2e9..3c5cacf4 100644 --- a/hyperdb-api/src/lib.rs +++ b/hyperdb-api/src/lib.rs @@ -227,6 +227,8 @@ pub use connection_builder::ConnectionBuilder; pub use error::{ColumnErrorKind, Error, Result}; pub use params::{ParamFormat, ToSqlParam}; pub use prepared::PreparedStatement; +// TLS settings for TCP connections (`ConnectionBuilder::tls` and friends). +pub use hyperdb_api_core::client::tls::{ParseTlsModeError, TlsConfig, TlsMode}; // Re-export Notice for callback registrants. `hyperdb-api-core`'s // `client::Error` is intentionally NOT re-exported — callers match // directly on the flat `Error` enum this crate defines. From 4c090cd03de9791406ce582ab7c4fbd1f74820e8 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 03:09:17 -0700 Subject: [PATCH 22/61] feat(api,core)!: TLS for TCP connections via the builders' tls(), cancel over TLS ConnectionBuilder::tls and AsyncConnectionBuilder::tls negotiate TLS with libpq sslmode semantics (Disable default, Prefer, Require, VerifyCa, VerifyFull; root cert, client cert, server-name override). Prefer falls back to plaintext only when the server declines TLS. A cancel for a TLS session goes over TLS. Over UDS / named pipes Prefer is plaintext and requiring modes fail with FeatureNotSupported; gRPC rejects tls(). Connection::is_tls and AsyncConnection::is_tls report the outcome. rustls session resumption is disabled: hyperd sets verify_peer without an OpenSSL session ID context and aborts every resumed handshake, which the cancel connection hit. BREAKING CHANGE: SyncStream::try_clone is removed and SyncStream / AsyncStream gain a Tls variant. Integration tests run against a real hyperd with ssl_force, using rcgen certificates: Require round trips (sync/async), VerifyCa/VerifyFull checks, Prefer fallback and no-downgrade, mTLS accept/reject, cancel, gRPC and IPC rejection. --- Cargo.lock | 1 + hyperdb-api-core/CHANGELOG.md | 19 + hyperdb-api-core/src/client/async_client.rs | 128 +++--- hyperdb-api-core/src/client/async_stream.rs | 49 +- hyperdb-api-core/src/client/client.rs | 104 +++-- hyperdb-api-core/src/client/sync_stream.rs | 63 ++- hyperdb-api-core/src/client/tls.rs | 29 +- hyperdb-api/CHANGELOG.md | 23 +- hyperdb-api/Cargo.toml | 2 + hyperdb-api/src/async_connection.rs | 15 + hyperdb-api/src/async_connection_builder.rs | 71 ++- hyperdb-api/src/connection.rs | 15 + hyperdb-api/src/connection_builder.rs | 68 ++- hyperdb-api/tests/tls_tests.rs | 477 ++++++++++++++++++++ 14 files changed, 932 insertions(+), 132 deletions(-) create mode 100644 hyperdb-api/tests/tls_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 629d2cff..9ede9d73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1814,6 +1814,7 @@ dependencies = [ "hyperdb-api-derive", "libc", "rand 0.10.3", + "rcgen", "serde", "serde_json", "smallvec", diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index 32c2c849..a54d1246 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -13,8 +13,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- **`client::tls`: TLS for TCP connections**, negotiated with PostgreSQL's + `SSLRequest` and libpq `sslmode` semantics. `TlsConfig` / `TlsMode` (with + `ParseTlsModeError`) are set through `Config::with_tls` and read back with + `Config::tls`. `Client::is_tls` and `AsyncClient::is_tls` report the + outcome, and `SyncStream::Tls` / `AsyncStream::Tls` carry the encrypted + stream. A cancel request for a TLS session is sent over TLS. Session + resumption is disabled, because `hyperd` aborts every resumed handshake. + +### Removed + +- **`SyncStream::try_clone`.** A rustls stream cannot be cloned, and nothing + called it. + ### Changed +- **`Config`'s `FromStr` error type is `client::Error`** (was `String`). The + libpq TLS keys (`sslmode`, `sslrootcert`, `sslcert`, `sslkey`, ...) are + rejected with `Error::Config` instead of being forwarded to the server as + startup options, where they were silently ignored. - **`SqlType` and its struct variants (`Numeric`, `Varchar`, `Char`) are `#[non_exhaustive]`.** `SqlType::try_numeric` is the non-panicking counterpart of `SqlType::numeric` for untrusted precision and scale. diff --git a/hyperdb-api-core/src/client/async_client.rs b/hyperdb-api-core/src/client/async_client.rs index e3aa0401..83625c10 100644 --- a/hyperdb-api-core/src/client/async_client.rs +++ b/hyperdb-api-core/src/client/async_client.rs @@ -25,6 +25,7 @@ use super::error::{Error, Result}; use super::notice::{Notice, NoticeReceiver}; use super::row::{Row, StreamRow}; use super::statement::ParamFormat; +use super::tls::{self, Fallback, Negotiated, TlsConnector}; use crate::protocol::message::Message; @@ -62,6 +63,9 @@ pub struct AsyncClient { endpoint: ConnectionEndpoint, /// Optional notice receiver callback for server notices/warnings. notice_receiver: Option>, + /// The TLS connector, kept only when TLS was negotiated, so that a cancel + /// request for this session goes over TLS too. + tls: Option>, } impl std::fmt::Debug for AsyncClient { @@ -74,6 +78,7 @@ impl std::fmt::Debug for AsyncClient { "notice_receiver", &self.notice_receiver.as_ref().map(|_| ""), ) + .field("tls", &self.tls.is_some()) .finish_non_exhaustive() } } @@ -81,6 +86,9 @@ impl std::fmt::Debug for AsyncClient { impl AsyncClient { /// Connects to a Hyper server using the given configuration (async). /// + /// Negotiates TLS when [`Config::tls`] asks for it, as + /// [`Client::connect`](super::client::Client::connect) does. + /// /// # Example /// /// ```no_run @@ -98,8 +106,13 @@ impl AsyncClient { /// /// # Errors /// + /// - Returns [`Error::Config`] if the TLS configuration is invalid or a + /// certificate file cannot be read. /// - Returns [`Error`] (connection) if the TCP connection cannot be /// established to `config.host():config.port()`. + /// - Returns [`Error::Tls`] if TLS negotiation or certificate + /// verification fails, and [`Error::Timeout`] if it does not finish + /// within [`Config::connect_timeout`]. /// - Propagates any [`Error`] from the startup handshake — /// [`Error`] (auth) for missing/wrong credentials, /// [`Error`] (server) for server-side startup errors, [`Error`] (protocol) @@ -115,6 +128,10 @@ impl AsyncClient { "connection-parameters" ); + // Built before connecting, so a bad certificate file or option + // combination fails without touching the network. + let connector = TlsConnector::build(config.tls(), config.host())?; + let endpoint = ConnectionEndpoint::tcp(config.host(), config.port()); let addr = format!("{}:{}", config.host(), config.port()); let tcp_stream = TcpStream::connect(&addr).await.map_err(|e| { @@ -144,7 +161,20 @@ impl AsyncClient { sock.set_tcp_keepalive(&keepalive).ok(); } - let stream = AsyncStream::tcp(tcp_stream); + let (stream, tls) = match connector { + None => (AsyncStream::tcp(tcp_stream), None), + Some(connector) => { + // A zero connect timeout means no limit, as in libpq. + let timeout = config.connect_timeout().filter(|t| !t.is_zero()); + let fallback = Fallback::for_mode(config.tls().mode()); + match tls::negotiate_async(tcp_stream, &connector, fallback, timeout).await? { + Negotiated::Plain(tcp_stream) => (AsyncStream::tcp(tcp_stream), None), + Negotiated::Tls(tls_stream) => { + (AsyncStream::Tls(Box::new(tls_stream)), Some(connector)) + } + } + } + }; let mut connection = AsyncRawConnection::new(stream); // Perform startup with authentication @@ -158,6 +188,7 @@ impl AsyncClient { debug!( target: "hyperdb_api", process_id, + tls = tls.is_some(), "connection-established" ); @@ -167,6 +198,7 @@ impl AsyncClient { secret_key, endpoint, notice_receiver: None, + tls, }) } @@ -187,6 +219,10 @@ impl AsyncClient { /// /// # Errors /// + /// - Returns [`Error::FeatureNotSupported`] if [`Config::tls`] requires + /// TLS, which a Unix domain socket does not carry; + /// [`TlsMode::Prefer`](super::tls::TlsMode::Prefer) connects in + /// plaintext. /// - Returns [`Error`] (connection) if the Unix domain socket cannot /// be connected. /// - Propagates any error from the startup handshake (see @@ -198,6 +234,7 @@ impl AsyncClient { ) -> Result { use std::path::Path; + tls::reject_on_ipc(config.tls())?; let path = socket_path.as_ref(); info!( target: "hyperdb_api", @@ -243,6 +280,7 @@ impl AsyncClient { secret_key, endpoint, notice_receiver: None, + tls: None, }) } @@ -255,14 +293,19 @@ impl AsyncClient { /// /// # Errors /// - /// Returns an error if the Named Pipe cannot be opened (e.g., pipe does not - /// exist, all instances are busy after the retry window, or permission is - /// denied) or if the authentication handshake fails. + /// Returns [`Error::FeatureNotSupported`] if [`Config::tls`] requires TLS, + /// which a named pipe does not carry + /// ([`TlsMode::Prefer`](super::tls::TlsMode::Prefer) connects in + /// plaintext). Returns an error if the Named Pipe cannot be opened (e.g., + /// pipe does not exist, all instances are busy after the retry window, or + /// permission is denied) or if the authentication handshake fails. #[cfg(windows)] pub async fn connect_named_pipe(pipe_path: &str, config: &Config) -> Result { use std::time::{Duration, Instant}; use tokio::net::windows::named_pipe::ClientOptions; + tls::reject_on_ipc(config.tls())?; + info!( target: "hyperdb_api", pipe_path = %pipe_path, @@ -337,6 +380,7 @@ impl AsyncClient { secret_key, endpoint, notice_receiver: None, + tls: None, }) } @@ -375,6 +419,13 @@ impl AsyncClient { &self.endpoint } + /// Returns true if this connection negotiated TLS; see + /// [`Client::is_tls`](super::client::Client::is_tls). + #[must_use] + pub fn is_tls(&self) -> bool { + self.tls.is_some() + } + /// Returns the server process ID for this connection. #[must_use] pub fn process_id(&self) -> i32 { @@ -390,7 +441,8 @@ impl AsyncClient { /// Cancels the currently executing query on this connection (async). /// /// This method opens a separate connection to send a cancel request. - /// For TCP endpoints, it opens a new TCP connection. + /// For TCP endpoints, it opens a new TCP connection, negotiating TLS on + /// it for a TLS session and never falling back to plaintext. /// For Unix domain sockets, it connects to the same socket path. /// /// # Errors @@ -398,6 +450,8 @@ impl AsyncClient { /// - Returns [`Error`] (connection) if a fresh cancel-side socket /// (TCP / UDS / named-pipe) cannot be opened to /// [`Self::endpoint`]. + /// - Returns [`Error::Tls`] or [`Error::Timeout`] if this is a TLS + /// session and TLS cannot be negotiated on the cancel-side socket. /// - Returns [`Error`] (I/O) if writing the cancel request fails. pub async fn cancel(&self) -> Result<()> { use crate::protocol::message::frontend; @@ -414,33 +468,13 @@ impl AsyncClient { match &self.endpoint { ConnectionEndpoint::Tcp { host, port } => { - let addr = format!("{host}:{port}"); - let mut stream = TcpStream::connect(&addr).await.map_err(|e| { - warn!( - target: "hyperdb_api", - addr = %endpoint_str, - error = %e, - "query-cancel-connect-failed" - ); - Error::connection(format!( - "failed to connect for cancel request to {endpoint_str}: {e}" - )) - })?; - // Cancel is a 16-byte fire-and-forget — disable Nagle so the - // request hits the wire without waiting on a coalesce timer. - stream.set_nodelay(true).ok(); - - let mut buf = BytesMut::new(); - frontend::cancel_request(self.process_id, self.secret_key, &mut buf); - - stream.write_all(&buf).await.map_err(|e| { - warn!( - target: "hyperdb_api", - error = %e, - "query-cancel-send-failed" - ); - Error::from_io(e) - })?; + tls::send_cancel_async( + &format!("{host}:{port}"), + self.process_id, + self.secret_key, + self.tls.as_deref(), + ) + .await?; } #[cfg(unix)] ConnectionEndpoint::DomainSocket { directory, name } => { @@ -570,7 +604,8 @@ impl AsyncClient { /// is usable from [`Drop`] impls (notably /// [`AsyncQueryStream::drop`](super::async_stream_query::AsyncQueryStream)). /// - /// Cancellation opens a short-lived TCP / UDS / Named-Pipe connection, + /// Cancellation opens a short-lived TCP / UDS / Named-Pipe connection + /// (TLS over TCP for a TLS session), /// writes the cancel packet, and drops it — the server recognizes the /// (`process_id`, `secret_key`) tuple and signals the long-running query /// to abort. No response is expected. @@ -589,27 +624,12 @@ impl AsyncClient { match &self.endpoint { ConnectionEndpoint::Tcp { host, port } => { - let addr = format!("{host}:{port}"); - let mut stream = std::net::TcpStream::connect(&addr).map_err(|e| { - warn!( - target: "hyperdb_api", - addr = %endpoint_str, - error = %e, - "query-cancel-connect-failed" - ); - Error::connection(format!( - "failed to connect for cancel request to {endpoint_str}: {e}" - )) - })?; - // Cancel is a 16-byte fire-and-forget — disable Nagle so the - // request hits the wire without waiting on a coalesce timer. - stream.set_nodelay(true).ok(); - - let mut buf = BytesMut::with_capacity(16); - frontend::cancel_request(self.process_id, self.secret_key, &mut buf); - - stream.write_all(&buf).map_err(Error::from_io)?; - stream.flush().map_err(Error::from_io)?; + tls::send_cancel_sync( + &format!("{host}:{port}"), + self.process_id, + self.secret_key, + self.tls.as_deref(), + )?; } #[cfg(unix)] ConnectionEndpoint::DomainSocket { directory, name } => { diff --git a/hyperdb-api-core/src/client/async_stream.rs b/hyperdb-api-core/src/client/async_stream.rs index 390f7578..8b84c200 100644 --- a/hyperdb-api-core/src/client/async_stream.rs +++ b/hyperdb-api-core/src/client/async_stream.rs @@ -4,8 +4,8 @@ //! Async stream abstraction for multiple transport types. //! //! This module provides [`AsyncStream`], an enum that can hold different -//! async stream types (TCP, Unix Domain Socket) while implementing the -//! necessary async I/O traits. +//! async stream types (TCP, TLS over TCP, Unix Domain Socket, named pipe) +//! while implementing the necessary async I/O traits. use std::io; use std::pin::Pin; @@ -13,6 +13,7 @@ use std::task::{Context, Poll}; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::net::TcpStream; +use tokio_rustls::client::TlsStream; #[cfg(unix)] use tokio::net::UnixStream; @@ -20,16 +21,19 @@ use tokio::net::UnixStream; #[cfg(windows)] use tokio::net::windows::named_pipe::NamedPipeClient; -/// An async stream that can be either TCP or Unix Domain Socket. +/// An async stream over TCP, TLS over TCP, a Unix Domain Socket or a named pipe. /// /// This enum provides a unified interface for different transport mechanisms, -/// allowing [`AsyncClient`](crate::client::AsyncClient) to work with both TCP and -/// Unix Domain Sockets transparently. +/// allowing [`AsyncClient`](crate::client::AsyncClient) to work with all of +/// them transparently. #[derive(Debug)] pub enum AsyncStream { /// TCP stream for network connections. Tcp(TcpStream), + /// TLS-encrypted TCP stream, after a successful `SSLRequest` handshake. + Tls(Box>), + /// Unix Domain Socket stream for local IPC (Unix only). #[cfg(unix)] Unix(UnixStream), @@ -51,9 +55,14 @@ impl AsyncStream { AsyncStream::Unix(stream) } - /// Returns true if this is a TCP stream. + /// Returns true if this is a TCP stream, encrypted or not. pub fn is_tcp(&self) -> bool { - matches!(self, AsyncStream::Tcp(_)) + matches!(self, AsyncStream::Tcp(_) | AsyncStream::Tls(_)) + } + + /// Returns true if this is a TLS-encrypted TCP stream. + pub fn is_tls(&self) -> bool { + matches!(self, AsyncStream::Tls(_)) } /// Returns true if this is a Unix Domain Socket stream. @@ -85,6 +94,7 @@ impl AsyncStream { pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { match self { AsyncStream::Tcp(stream) => stream.set_nodelay(nodelay), + AsyncStream::Tls(stream) => stream.get_ref().0.set_nodelay(nodelay), #[cfg(unix)] AsyncStream::Unix(_) => Ok(()), // No-op for Unix sockets #[cfg(windows)] @@ -98,6 +108,11 @@ impl AsyncStream { AsyncStream::Tcp(stream) => stream .local_addr() .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), + AsyncStream::Tls(stream) => stream + .get_ref() + .0 + .local_addr() + .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), #[cfg(unix)] AsyncStream::Unix(stream) => stream .local_addr() @@ -115,6 +130,11 @@ impl AsyncStream { AsyncStream::Tcp(stream) => stream .peer_addr() .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), + AsyncStream::Tls(stream) => stream + .get_ref() + .0 + .peer_addr() + .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), #[cfg(unix)] AsyncStream::Unix(stream) => stream .peer_addr() @@ -135,6 +155,16 @@ impl AsyncRead for AsyncStream { ) -> Poll> { match self.get_mut() { AsyncStream::Tcp(stream) => Pin::new(stream).poll_read(cx, buf), + // rustls reports a peer that closes the socket without a + // `close_notify` alert as `UnexpectedEof`. Report it as EOF, as + // the plaintext stream would: the wire protocol is length-framed, + // so a truncated message is still caught by the reader. + AsyncStream::Tls(stream) => match Pin::new(stream.as_mut()).poll_read(cx, buf) { + Poll::Ready(Err(err)) if err.kind() == io::ErrorKind::UnexpectedEof => { + Poll::Ready(Ok(())) + } + other => other, + }, #[cfg(unix)] AsyncStream::Unix(stream) => Pin::new(stream).poll_read(cx, buf), #[cfg(windows)] @@ -151,6 +181,7 @@ impl AsyncWrite for AsyncStream { ) -> Poll> { match self.get_mut() { AsyncStream::Tcp(stream) => Pin::new(stream).poll_write(cx, buf), + AsyncStream::Tls(stream) => Pin::new(stream.as_mut()).poll_write(cx, buf), #[cfg(unix)] AsyncStream::Unix(stream) => Pin::new(stream).poll_write(cx, buf), #[cfg(windows)] @@ -161,6 +192,7 @@ impl AsyncWrite for AsyncStream { fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.get_mut() { AsyncStream::Tcp(stream) => Pin::new(stream).poll_flush(cx), + AsyncStream::Tls(stream) => Pin::new(stream.as_mut()).poll_flush(cx), #[cfg(unix)] AsyncStream::Unix(stream) => Pin::new(stream).poll_flush(cx), #[cfg(windows)] @@ -171,6 +203,7 @@ impl AsyncWrite for AsyncStream { fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.get_mut() { AsyncStream::Tcp(stream) => Pin::new(stream).poll_shutdown(cx), + AsyncStream::Tls(stream) => Pin::new(stream.as_mut()).poll_shutdown(cx), #[cfg(unix)] AsyncStream::Unix(stream) => Pin::new(stream).poll_shutdown(cx), #[cfg(windows)] @@ -185,6 +218,7 @@ impl AsyncWrite for AsyncStream { ) -> Poll> { match self.get_mut() { AsyncStream::Tcp(stream) => Pin::new(stream).poll_write_vectored(cx, bufs), + AsyncStream::Tls(stream) => Pin::new(stream.as_mut()).poll_write_vectored(cx, bufs), #[cfg(unix)] AsyncStream::Unix(stream) => Pin::new(stream).poll_write_vectored(cx, bufs), #[cfg(windows)] @@ -195,6 +229,7 @@ impl AsyncWrite for AsyncStream { fn is_write_vectored(&self) -> bool { match self { AsyncStream::Tcp(stream) => stream.is_write_vectored(), + AsyncStream::Tls(stream) => stream.is_write_vectored(), #[cfg(unix)] AsyncStream::Unix(stream) => stream.is_write_vectored(), #[cfg(windows)] diff --git a/hyperdb-api-core/src/client/client.rs b/hyperdb-api-core/src/client/client.rs index 44b8f5f1..20237299 100644 --- a/hyperdb-api-core/src/client/client.rs +++ b/hyperdb-api-core/src/client/client.rs @@ -75,6 +75,7 @@ use super::prepare; use super::row::{Row, StreamRow}; use super::statement::ParamFormat; use super::sync_stream::SyncStream; +use super::tls::{self, Fallback, Negotiated, TlsConnector}; use crate::protocol::message::Message; use crate::types::Oid; @@ -119,6 +120,9 @@ pub struct Client { endpoint: ConnectionEndpoint, /// Optional notice receiver callback for server notices/warnings. notice_receiver: Option>, + /// The TLS connector, kept only when TLS was negotiated, so that a cancel + /// request for this session goes over TLS too. + tls: Option>, } // Manual Debug implementation because NoticeReceiver doesn't implement Debug @@ -132,6 +136,7 @@ impl std::fmt::Debug for Client { "notice_receiver", &self.notice_receiver.as_ref().map(|_| ""), ) + .field("tls", &self.tls.is_some()) .finish_non_exhaustive() } } @@ -139,7 +144,8 @@ impl std::fmt::Debug for Client { impl Client { /// Connects to a Hyper server using the given configuration. /// - /// Establishes a TCP connection, performs authentication, and initializes + /// Establishes a TCP connection, negotiates TLS when + /// [`Config::tls`] asks for it, performs authentication, and initializes /// the client. Returns an error if the connection fails or authentication /// is rejected. /// @@ -150,7 +156,12 @@ impl Client { /// # Errors /// /// Returns `Error` if: + /// - The TLS configuration is invalid or a certificate file cannot be + /// read ([`Error::Config`]) /// - Connection to the server fails + /// - TLS negotiation or certificate verification fails ([`Error::Tls`]), + /// or does not finish within [`Config::connect_timeout`] + /// ([`Error::Timeout`]) /// - Authentication fails /// - Protocol handshake fails /// @@ -181,6 +192,10 @@ impl Client { "connection-parameters" ); + // Built before connecting, so a bad certificate file or option + // combination fails without touching the network. + let connector = TlsConnector::build(config.tls(), config.host())?; + let endpoint = ConnectionEndpoint::tcp(config.host(), config.port()); let addr = format!("{}:{}", config.host(), config.port()); let tcp_stream = TcpStream::connect(&addr).map_err(|e| { @@ -219,7 +234,20 @@ impl Client { sock.set_send_buffer_size(4 * 1024 * 1024).ok(); apply_tcp_keepalive(&sock); - let stream = SyncStream::tcp(tcp_stream); + let (stream, tls) = match connector { + None => (SyncStream::tcp(tcp_stream), None), + Some(connector) => { + // A zero connect timeout means no limit, as in libpq. + let timeout = config.connect_timeout().filter(|t| !t.is_zero()); + let fallback = Fallback::for_mode(config.tls().mode()); + match tls::negotiate_sync(tcp_stream, &connector, fallback, timeout)? { + Negotiated::Plain(tcp_stream) => (SyncStream::tcp(tcp_stream), None), + Negotiated::Tls(tls_stream) => { + (SyncStream::Tls(Box::new(tls_stream)), Some(connector)) + } + } + } + }; let mut connection = RawConnection::new(stream); // Perform startup with authentication @@ -233,6 +261,7 @@ impl Client { debug!( target: "hyperdb_api", process_id, + tls = tls.is_some(), "connection-established" ); @@ -242,6 +271,7 @@ impl Client { secret_key, endpoint, notice_receiver: None, + tls, }) } @@ -262,6 +292,10 @@ impl Client { /// /// # Errors /// + /// - Returns [`Error::FeatureNotSupported`] if [`Config::tls`] requires + /// TLS, which a Unix domain socket does not carry; + /// [`TlsMode::Prefer`](super::tls::TlsMode::Prefer) connects in + /// plaintext. /// - Returns [`Error`] (connection) if the Unix domain socket cannot /// be connected. /// - Propagates any [`Error`] from the startup handshake @@ -270,6 +304,7 @@ impl Client { pub fn connect_unix(socket_path: impl AsRef, config: &Config) -> Result { use std::path::Path; + tls::reject_on_ipc(config.tls())?; let path = socket_path.as_ref(); info!( target: "hyperdb_api", @@ -315,6 +350,7 @@ impl Client { secret_key, endpoint, notice_receiver: None, + tls: None, }) } @@ -327,14 +363,19 @@ impl Client { /// /// # Errors /// - /// Returns an error if the Named Pipe cannot be opened (e.g., pipe does not - /// exist, all instances are busy after the retry window, or permission is - /// denied) or if the authentication handshake fails. + /// Returns [`Error::FeatureNotSupported`] if [`Config::tls`] requires TLS, + /// which a named pipe does not carry + /// ([`TlsMode::Prefer`](super::tls::TlsMode::Prefer) connects in + /// plaintext). Returns an error if the Named Pipe cannot be opened (e.g., + /// pipe does not exist, all instances are busy after the retry window, or + /// permission is denied) or if the authentication handshake fails. #[cfg(windows)] pub fn connect_named_pipe(pipe_path: &str, config: &Config) -> Result { use std::fs::OpenOptions; use std::time::Instant; + tls::reject_on_ipc(config.tls())?; + info!( target: "hyperdb_api", pipe_path = %pipe_path, @@ -413,6 +454,7 @@ impl Client { secret_key, endpoint, notice_receiver: None, + tls: None, }) } @@ -451,6 +493,17 @@ impl Client { &self.endpoint } + /// Returns true if this connection negotiated TLS. + /// + /// False for a plaintext TCP connection, including one where + /// [`TlsMode::Prefer`](super::tls::TlsMode::Prefer) fell back after the + /// server declined TLS, and for Unix-domain-socket and named-pipe + /// connections. + #[must_use] + pub fn is_tls(&self) -> bool { + self.tls.is_some() + } + /// Returns the server process ID for this connection. #[must_use] pub fn process_id(&self) -> i32 { @@ -471,7 +524,8 @@ impl Client { /// /// # How It Works /// - /// 1. Opens a new TCP connection to the same server + /// 1. Opens a new connection to the same server; for a TLS session it + /// negotiates TLS on it, never falling back to plaintext /// 2. Sends a cancel request containing the process ID and secret key /// 3. The server receives this and cancels the running query /// 4. The original query will fail with error code 57014 (`query_canceled`) @@ -528,6 +582,8 @@ impl Client { /// - Returns [`Error`] (connection) if the fresh cancel-side socket /// cannot be opened (TCP / UDS / named-pipe, depending on /// [`Self::endpoint`]). + /// - Returns [`Error::Tls`] or [`Error::Timeout`] if this is a TLS + /// session and TLS cannot be negotiated on the cancel-side socket. /// - Returns [`Error`] (I/O) if writing or flushing the cancel /// request fails. pub fn cancel(&self) -> Result<()> { @@ -546,36 +602,12 @@ impl Client { // Open a new connection specifically for the cancel request match &self.endpoint { ConnectionEndpoint::Tcp { host, port } => { - let addr = format!("{host}:{port}"); - let mut stream = TcpStream::connect(&addr).map_err(|e| { - warn!( - target: "hyperdb_api", - addr = %endpoint_str, - error = %e, - "query-cancel-connect-failed" - ); - Error::connection(format!( - "failed to connect for cancel request to {endpoint_str}: {e}" - )) - })?; - // Cancel is a 16-byte fire-and-forget — disable Nagle so the - // request hits the wire without waiting on a coalesce timer. - stream.set_nodelay(true).ok(); - - // Build and send the cancel request - let mut buf = BytesMut::with_capacity(16); - frontend::cancel_request(self.process_id, self.secret_key, &mut buf); - - stream.write_all(&buf).map_err(|e| { - warn!( - target: "hyperdb_api", - error = %e, - "query-cancel-send-failed" - ); - Error::from_io(e) - })?; - - stream.flush().map_err(Error::from_io)?; + tls::send_cancel_sync( + &format!("{host}:{port}"), + self.process_id, + self.secret_key, + self.tls.as_deref(), + )?; } #[cfg(unix)] ConnectionEndpoint::DomainSocket { directory, name } => { diff --git a/hyperdb-api-core/src/client/sync_stream.rs b/hyperdb-api-core/src/client/sync_stream.rs index 06d49442..6c48c0c7 100644 --- a/hyperdb-api-core/src/client/sync_stream.rs +++ b/hyperdb-api-core/src/client/sync_stream.rs @@ -4,28 +4,33 @@ //! Sync stream abstraction for multiple transport types. //! //! This module provides [`SyncStream`], an enum that can hold different -//! sync stream types (TCP, Unix Domain Socket) while implementing the -//! necessary I/O traits. +//! sync stream types (TCP, TLS over TCP, Unix Domain Socket, named pipe) +//! while implementing the necessary I/O traits. use std::io::{self, Read, Write}; use std::net::TcpStream; +use rustls::{ClientConnection, StreamOwned}; + #[cfg(unix)] use std::os::unix::net::UnixStream; #[cfg(windows)] use std::fs::File; -/// A sync stream that can be either TCP or Unix Domain Socket. +/// A sync stream over TCP, TLS over TCP, a Unix Domain Socket or a named pipe. /// /// This enum provides a unified interface for different transport mechanisms, -/// allowing [`Client`](crate::client::Client) to work with both TCP and -/// Unix Domain Sockets transparently. +/// allowing [`Client`](crate::client::Client) to work with all of them +/// transparently. #[derive(Debug)] pub enum SyncStream { /// TCP stream for network connections. Tcp(TcpStream), + /// TLS-encrypted TCP stream, after a successful `SSLRequest` handshake. + Tls(Box>), + /// Unix Domain Socket stream for local IPC (Unix only). #[cfg(unix)] Unix(UnixStream), @@ -49,10 +54,16 @@ impl SyncStream { SyncStream::Unix(stream) } - /// Returns true if this is a TCP stream. + /// Returns true if this is a TCP stream, encrypted or not. #[must_use] pub fn is_tcp(&self) -> bool { - matches!(self, SyncStream::Tcp(_)) + matches!(self, SyncStream::Tcp(_) | SyncStream::Tls(_)) + } + + /// Returns true if this is a TLS-encrypted TCP stream. + #[must_use] + pub fn is_tls(&self) -> bool { + matches!(self, SyncStream::Tls(_)) } /// Returns true if this is a Unix Domain Socket stream. @@ -85,6 +96,7 @@ impl SyncStream { pub fn set_nodelay(&self, nodelay: bool) -> io::Result<()> { match self { SyncStream::Tcp(stream) => stream.set_nodelay(nodelay), + SyncStream::Tls(stream) => stream.sock.set_nodelay(nodelay), #[cfg(unix)] SyncStream::Unix(_) => Ok(()), // No-op for Unix sockets #[cfg(windows)] @@ -102,6 +114,7 @@ impl SyncStream { pub fn set_read_timeout(&self, dur: Option) -> io::Result<()> { match self { SyncStream::Tcp(stream) => stream.set_read_timeout(dur), + SyncStream::Tls(stream) => stream.sock.set_read_timeout(dur), #[cfg(unix)] SyncStream::Unix(stream) => stream.set_read_timeout(dur), #[cfg(windows)] @@ -119,6 +132,7 @@ impl SyncStream { pub fn set_write_timeout(&self, dur: Option) -> io::Result<()> { match self { SyncStream::Tcp(stream) => stream.set_write_timeout(dur), + SyncStream::Tls(stream) => stream.sock.set_write_timeout(dur), #[cfg(unix)] SyncStream::Unix(stream) => stream.set_write_timeout(dur), #[cfg(windows)] @@ -133,6 +147,10 @@ impl SyncStream { SyncStream::Tcp(stream) => stream .local_addr() .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), + SyncStream::Tls(stream) => stream + .sock + .local_addr() + .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), #[cfg(unix)] SyncStream::Unix(stream) => stream .local_addr() @@ -151,6 +169,10 @@ impl SyncStream { SyncStream::Tcp(stream) => stream .peer_addr() .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), + SyncStream::Tls(stream) => stream + .sock + .peer_addr() + .map_or_else(|_| "unknown".to_string(), |a| a.to_string()), #[cfg(unix)] SyncStream::Unix(stream) => stream .peer_addr() @@ -161,29 +183,20 @@ impl SyncStream { SyncStream::NamedPipe(_) => "named-pipe".to_string(), } } - - /// Attempts to clone the stream. - /// - /// # Errors - /// - /// Returns an [`io::Error`] from the underlying transport's - /// `try_clone` call — typically because the OS refused to - /// duplicate the descriptor. - pub fn try_clone(&self) -> io::Result { - match self { - SyncStream::Tcp(stream) => Ok(SyncStream::Tcp(stream.try_clone()?)), - #[cfg(unix)] - SyncStream::Unix(stream) => Ok(SyncStream::Unix(stream.try_clone()?)), - #[cfg(windows)] - SyncStream::NamedPipe(file) => Ok(SyncStream::NamedPipe(file.try_clone()?)), - } - } } impl Read for SyncStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { match self { SyncStream::Tcp(stream) => stream.read(buf), + // rustls reports a peer that closes the socket without a + // `close_notify` alert as `UnexpectedEof`. Report it as EOF, as + // the plaintext stream would: the wire protocol is length-framed, + // so a truncated message is still caught by the reader. + SyncStream::Tls(stream) => match stream.read(buf) { + Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => Ok(0), + other => other, + }, #[cfg(unix)] SyncStream::Unix(stream) => stream.read(buf), #[cfg(windows)] @@ -196,6 +209,7 @@ impl Write for SyncStream { fn write(&mut self, buf: &[u8]) -> io::Result { match self { SyncStream::Tcp(stream) => stream.write(buf), + SyncStream::Tls(stream) => stream.write(buf), #[cfg(unix)] SyncStream::Unix(stream) => stream.write(buf), #[cfg(windows)] @@ -206,6 +220,7 @@ impl Write for SyncStream { fn flush(&mut self) -> io::Result<()> { match self { SyncStream::Tcp(stream) => stream.flush(), + SyncStream::Tls(stream) => stream.flush(), #[cfg(unix)] SyncStream::Unix(stream) => stream.flush(), #[cfg(windows)] diff --git a/hyperdb-api-core/src/client/tls.rs b/hyperdb-api-core/src/client/tls.rs index 45df473c..e7d30391 100644 --- a/hyperdb-api-core/src/client/tls.rs +++ b/hyperdb-api-core/src/client/tls.rs @@ -43,11 +43,6 @@ //! assert_eq!("verify-ca".parse::(), Ok(TlsMode::VerifyCa)); //! ``` -#![cfg_attr( - not(test), - expect(dead_code, reason = "wired into the clients in the next change") -)] - use std::fmt; use std::io::{self, Read, Write}; use std::net::{Shutdown, TcpStream}; @@ -306,7 +301,7 @@ impl TlsConnector { .map_err(|e| Error::config(format!("TLS protocol configuration: {e}")))? .dangerous() .with_custom_certificate_verifier(verifier); - let config = match &tls.client_cert { + let mut config = match &tls.client_cert { Some((cert_path, key_path)) => builder .with_client_auth_cert( load_certs(cert_path, "client certificate")?, @@ -315,6 +310,12 @@ impl TlsConnector { .map_err(|e| Error::config(format!("invalid client certificate or key: {e}")))?, None => builder.with_no_client_auth(), }; + // hyperd requests client certificates (`verify_peer`) without setting + // an OpenSSL session ID context, so it aborts every resumed handshake + // with an `internal_error` alert. The cancel request reuses this + // connector and would resume. libpq never resumes either: it builds a + // fresh `SSL_CTX` per connection. + config.resumption = rustls::client::Resumption::disabled(); let name = tls.server_name.as_deref().unwrap_or_else(|| { // An IPv6 host arrives as `[::1]`; the brackets are URL syntax. @@ -481,6 +482,22 @@ impl Fallback { } } +/// Checks `tls` for a Unix-domain-socket or named-pipe connection, which has +/// no TLS: [`TlsMode::Prefer`] connects in plaintext, as libpq does over a +/// Unix socket, and every mode that requires TLS is rejected rather than +/// silently downgraded. +pub(crate) fn reject_on_ipc(tls: &TlsConfig) -> Result<()> { + match tls.mode() { + TlsMode::Disable | TlsMode::Prefer => Ok(()), + TlsMode::Require | TlsMode::VerifyCa | TlsMode::VerifyFull => { + Err(Error::feature_not_supported(format!( + "TLS mode {} is only available over TCP; use a host:port endpoint", + tls.mode() + ))) + } + } +} + /// The outcome of a negotiation: the socket unchanged (after `'N'` with /// [`Fallback::AllowPlaintext`]), or the TLS stream. pub(crate) enum Negotiated { diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 7c92f6a7..88ffcc7f 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -17,8 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). root, and the README states the policy: a major bump of a re-exported crate is a `hyperdb-api` major. - **Breaking:** `PreparedStatement::query` and `AsyncPreparedStatement::query` - now return a rowset borrowing the *statement* (`Rowset<'s>` / - `AsyncRowset<'s>`), not just the connection. Previously a rowset could + now return a rowset borrowing the *statement* (`Rowset<'stmt>` / + `AsyncRowset<'stmt>`), not just the connection. Previously a rowset could outlive its statement, and dropping the statement then deadlocked on the connection lock the rowset still held. The compiler now rejects that code. - **BREAKING: `SqlType` is `#[non_exhaustive]`, and so are the struct variants @@ -91,6 +91,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/). reporting unhealthy, so a pool evicts it on the next checkout. - **`HyperProcess::connection_endpoint_string`**, and `Pool::close` / `Pool::is_closed` on the async pool. +- **TLS for TCP connections.** Until now every TCP connection was plaintext, + whatever the server offered. `ConnectionBuilder::tls` and + `AsyncConnectionBuilder::tls` take a `TlsMode` or a `TlsConfig` (both + re-exported from the crate root) with libpq `sslmode` semantics: `Disable` + (the default), `Prefer`, `Require`, `VerifyCa` and `VerifyFull`, plus a root + certificate, a client certificate for mutual TLS, and a server-name + override. `Prefer` falls back to plaintext only when the server declines + TLS, never after a failed handshake or verification. A query cancel for a + TLS session is sent over TLS. `Connection::is_tls` and + `AsyncConnection::is_tls` report whether the session is encrypted. + - Over a Unix domain socket or a named pipe, `Prefer` connects in plaintext + and the modes that require TLS fail with `Error::FeatureNotSupported`. A + gRPC connection picks TLS through its `https://` scheme and rejects + `tls()` the same way. + - The `Connection::connect*` shortcuts and `Connection::new` stay + plaintext. + - TLS session resumption is disabled: `hyperd` requests client certificates + without an OpenSSL session ID context, so it aborts every resumed + handshake (and a cancel would resume). ### Fixed diff --git a/hyperdb-api/Cargo.toml b/hyperdb-api/Cargo.toml index a62cbc12..345250c6 100644 --- a/hyperdb-api/Cargo.toml +++ b/hyperdb-api/Cargo.toml @@ -68,6 +68,8 @@ futures = "0.3" rand = { workspace = true } serde = { workspace = true } sysinfo = { workspace = true } +# Certificates for the TLS tests against a real hyperd +rcgen = "0.14" # B4 compile-time validation end-to-end integration tests [[test]] diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index 9590ace3..fe9746a7 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -1014,6 +1014,21 @@ impl AsyncConnection { } } + /// Returns true if the connection is encrypted with TLS. + /// + /// For TCP, true when TLS was negotiated — set with + /// [`AsyncConnectionBuilder::tls`](crate::AsyncConnectionBuilder::tls); a + /// [`TlsMode::Prefer`](crate::TlsMode::Prefer) connection to a server that + /// declined TLS reports false. For gRPC, true for an `https://` endpoint. + /// Always false over a Unix domain socket or a named pipe. + #[must_use] + pub fn is_tls(&self) -> bool { + match &self.transport { + AsyncTransport::Tcp(tcp) => tcp.client.is_tls(), + AsyncTransport::Grpc(grpc) => grpc.config.is_tls(), + } + } + /// Actively pings the server with `SELECT 1` (async). /// /// # Errors diff --git a/hyperdb-api/src/async_connection_builder.rs b/hyperdb-api/src/async_connection_builder.rs index 718ce5ea..45b31940 100644 --- a/hyperdb-api/src/async_connection_builder.rs +++ b/hyperdb-api/src/async_connection_builder.rs @@ -17,6 +17,7 @@ use crate::connection::CreateMode; use crate::connection_builder::query_timeout_statement; use crate::error::{Error, Result}; use crate::transport::{TransportType, detect_transport_type}; +use hyperdb_api_core::client::tls::{TlsConfig, TlsMode}; use hyperdb_api_core::client::{AsyncClient, Config}; /// An async builder for creating database connections. @@ -52,6 +53,8 @@ pub struct AsyncConnectionBuilder { query_timeout: Option, application_name: Option, transfer_mode: Option, + /// TLS settings for TCP connections. + tls: TlsConfig, } impl Default for AsyncConnectionBuilder { @@ -73,6 +76,7 @@ impl AsyncConnectionBuilder { query_timeout: None, application_name: None, transfer_mode: None, + tls: TlsConfig::default(), } } @@ -171,6 +175,55 @@ impl AsyncConnectionBuilder { self } + /// Sets the TLS configuration for a TCP connection. + /// + /// The default is [`TlsMode::Disable`]. A bare [`TlsMode`] converts into a + /// [`TlsConfig`]; use [`TlsConfig`] to add a root certificate, a client + /// certificate for mutual TLS, or a server-name override. The modes follow + /// libpq's `sslmode`: + /// + /// | Mode | Encrypted | Server certificate checked | + /// |------|-----------|----------------------------| + /// | [`Disable`](TlsMode::Disable) | never | — | + /// | [`Prefer`](TlsMode::Prefer) | if the server offers TLS, else plaintext | chain, only if a root certificate is set | + /// | [`Require`](TlsMode::Require) | always | chain, only if a root certificate is set | + /// | [`VerifyCa`](TlsMode::VerifyCa) | always | chain, against the root certificate | + /// | [`VerifyFull`](TlsMode::VerifyFull) | always | chain and host name | + /// + /// Over a Unix domain socket or a named pipe, [`TlsMode::Prefer`] + /// connects in plaintext and every mode that requires TLS fails + /// [`build`](Self::build) with [`Error::FeatureNotSupported`]. A gRPC + /// endpoint picks TLS through its `https://` scheme, so any mode other + /// than [`TlsMode::Disable`] fails there with + /// [`Error::FeatureNotSupported`] too. A query cancel for a TLS session + /// is sent over TLS. TLS negotiation counts against + /// [`login_timeout`](Self::login_timeout) (30 seconds by default; zero + /// means no limit). + /// + /// The `AsyncConnection::connect*` shortcuts and `AsyncConnection::new` always + /// connect in plaintext; use this builder for TLS. + /// + /// # Example + /// + /// ```no_run + /// use hyperdb_api::{AsyncConnectionBuilder, Result, TlsConfig, TlsMode}; + /// + /// async fn example() -> Result<()> { + /// let conn = AsyncConnectionBuilder::new("hyper.example.com:7483") + /// .database("example.hyper") + /// .tls(TlsConfig::new(TlsMode::VerifyFull).root_cert("ca.pem")) + /// .build() + /// .await?; + /// assert!(conn.is_tls()); + /// Ok(()) + /// } + /// ``` + #[must_use] + pub fn tls(mut self, tls: impl Into) -> Self { + self.tls = tls.into(); + self + } + /// Sets the transfer mode for gRPC connections (ignored for TCP). #[must_use] pub fn transfer_mode(mut self, mode: crate::grpc::TransferMode) -> Self { @@ -189,9 +242,17 @@ impl AsyncConnectionBuilder { /// /// # Errors /// + /// - Returns [`Error::Config`] if the endpoint cannot be parsed, or if the + /// TLS settings are invalid or a certificate file cannot be read. + /// - Returns [`Error::FeatureNotSupported`] for a gRPC endpoint with a + /// [`tls`](Self::tls) mode other than [`TlsMode::Disable`], and for a + /// Unix-socket or named-pipe endpoint whose mode requires TLS. /// - Returns [`Error::Connection`] if the transport - /// handshake fails (TCP refused, TLS rejected, named-pipe not + /// handshake fails (TCP refused, named-pipe not /// found, gRPC channel setup failure). + /// - Returns [`Error::Tls`] if TLS negotiation or certificate + /// verification fails, and [`Error::Timeout`] if it does not finish + /// within the [`login_timeout`](Self::login_timeout). /// - Returns [`Error::Authentication`] if authentication is rejected. /// - Returns [`Error::Server`] if the `CreateMode` SQL is rejected /// for a builder that configured a database path. @@ -226,6 +287,7 @@ impl AsyncConnectionBuilder { if let Some(timeout) = self.login_timeout { config = config.with_connect_timeout(timeout); } + config = config.with_tls(self.tls); let db_path_str = self .database @@ -273,6 +335,7 @@ impl AsyncConnectionBuilder { if let Some(password) = &self.password { config = config.with_password(password); } + config = config.with_tls(self.tls); let db_path_str = self .database @@ -320,6 +383,7 @@ impl AsyncConnectionBuilder { if let Some(password) = &self.password { config = config.with_password(password); } + config = config.with_tls(self.tls); let db_path_str = self .database @@ -349,6 +413,11 @@ impl AsyncConnectionBuilder { "query_timeout is not supported on gRPC connections", )); } + if self.tls.mode() != TlsMode::Disable { + return Err(Error::feature_not_supported( + "tls is not supported on gRPC connections; use an https:// endpoint for TLS", + )); + } if self.create_mode != CreateMode::DoNotCreate { return Err(Error::feature_not_supported( "gRPC transport is read-only. Use CreateMode::DoNotCreate for gRPC connections.", diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index 8b00f16f..d8a11bd0 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -1849,6 +1849,21 @@ impl Connection { } } + /// Returns true if the connection is encrypted with TLS. + /// + /// For TCP, true when TLS was negotiated — set with + /// [`ConnectionBuilder::tls`](crate::ConnectionBuilder::tls); a + /// [`TlsMode::Prefer`](crate::TlsMode::Prefer) connection to a server that + /// declined TLS reports false. For gRPC, true for an `https://` endpoint. + /// Always false over a Unix domain socket or a named pipe. + #[must_use] + pub fn is_tls(&self) -> bool { + match &self.transport { + Transport::Tcp(tcp) => tcp.client.is_tls(), + Transport::Grpc(grpc) => grpc.config.is_tls(), + } + } + /// Actively checks that the connection is healthy by executing a trivial query. /// /// Unlike [`is_alive`](Self::is_alive) which only checks local state, diff --git a/hyperdb-api/src/connection_builder.rs b/hyperdb-api/src/connection_builder.rs index 72c935d7..2a00c733 100644 --- a/hyperdb-api/src/connection_builder.rs +++ b/hyperdb-api/src/connection_builder.rs @@ -8,6 +8,7 @@ use crate::connection::{Connection, CreateMode}; use crate::error::{Error, Result}; use crate::transport::{Transport, TransportType, detect_transport_type}; use hyperdb_api_core::client::Config; +use hyperdb_api_core::client::tls::{TlsConfig, TlsMode}; /// A builder for creating database connections. /// @@ -63,6 +64,8 @@ pub struct ConnectionBuilder { application_name: Option, /// Transfer mode for gRPC connections (ignored for TCP) transfer_mode: Option, + /// TLS settings for TCP connections. + tls: TlsConfig, } impl Default for ConnectionBuilder { @@ -101,6 +104,7 @@ impl ConnectionBuilder { query_timeout: None, application_name: None, transfer_mode: None, // Use default (Adaptive) + tls: TlsConfig::default(), } } @@ -206,6 +210,54 @@ impl ConnectionBuilder { self } + /// Sets the TLS configuration for a TCP connection. + /// + /// The default is [`TlsMode::Disable`]. A bare [`TlsMode`] converts into a + /// [`TlsConfig`]; use [`TlsConfig`] to add a root certificate, a client + /// certificate for mutual TLS, or a server-name override. The modes follow + /// libpq's `sslmode`: + /// + /// | Mode | Encrypted | Server certificate checked | + /// |------|-----------|----------------------------| + /// | [`Disable`](TlsMode::Disable) | never | — | + /// | [`Prefer`](TlsMode::Prefer) | if the server offers TLS, else plaintext | chain, only if a root certificate is set | + /// | [`Require`](TlsMode::Require) | always | chain, only if a root certificate is set | + /// | [`VerifyCa`](TlsMode::VerifyCa) | always | chain, against the root certificate | + /// | [`VerifyFull`](TlsMode::VerifyFull) | always | chain and host name | + /// + /// Over a Unix domain socket or a named pipe, [`TlsMode::Prefer`] + /// connects in plaintext and every mode that requires TLS fails + /// [`build`](Self::build) with [`Error::FeatureNotSupported`]. A gRPC + /// endpoint picks TLS through its `https://` scheme, so any mode other + /// than [`TlsMode::Disable`] fails there with + /// [`Error::FeatureNotSupported`] too. A query cancel for a TLS session + /// is sent over TLS. TLS negotiation counts against + /// [`login_timeout`](Self::login_timeout) (30 seconds by default; zero + /// means no limit). + /// + /// The `Connection::connect*` shortcuts and `Connection::new` always + /// connect in plaintext; use this builder for TLS. + /// + /// # Example + /// + /// ```no_run + /// use hyperdb_api::{ConnectionBuilder, Result, TlsConfig, TlsMode}; + /// + /// fn main() -> Result<()> { + /// let conn = ConnectionBuilder::new("hyper.example.com:7483") + /// .database("example.hyper") + /// .tls(TlsConfig::new(TlsMode::VerifyFull).root_cert("ca.pem")) + /// .build()?; + /// assert!(conn.is_tls()); + /// Ok(()) + /// } + /// ``` + #[must_use] + pub fn tls(mut self, tls: impl Into) -> Self { + self.tls = tls.into(); + self + } + /// Sets the transfer mode for gRPC connections. /// /// This setting is ignored for TCP connections. @@ -246,9 +298,13 @@ impl ConnectionBuilder { /// /// # Errors /// - /// - Returns [`Error::Config`] if the endpoint cannot be parsed. - /// - Returns [`Error::FeatureNotSupported`] for a gRPC endpoint with a create mode other than [`CreateMode::DoNotCreate`]. + /// - Returns [`Error::Config`] if the endpoint cannot be parsed, or if the + /// TLS settings are invalid or a certificate file cannot be read. + /// - Returns [`Error::FeatureNotSupported`] for a gRPC endpoint with a create mode other than [`CreateMode::DoNotCreate`], + /// or with a [`tls`](Self::tls) mode other than [`TlsMode::Disable`], and for a Unix-socket or named-pipe endpoint + /// whose [`tls`](Self::tls) mode requires TLS. /// - Returns [`Error::Connection`] or [`Error::Timeout`] if the transport handshake fails. + /// - Returns [`Error::Tls`] if TLS negotiation or certificate verification fails. /// - Returns [`Error::Authentication`] if authentication is rejected. /// - Returns [`Error::Server`] or [`Error::Internal`] if the database creation or attach SQL fails. pub fn build(self) -> Result { @@ -286,6 +342,7 @@ impl ConnectionBuilder { if let Some(timeout) = self.login_timeout { config = config.with_connect_timeout(timeout); } + config = config.with_tls(self.tls); let db_path_str = self .database @@ -338,6 +395,7 @@ impl ConnectionBuilder { if let Some(password) = &self.password { config = config.with_password(password); } + config = config.with_tls(self.tls); let db_path_str = self .database @@ -391,6 +449,7 @@ impl ConnectionBuilder { if let Some(password) = &self.password { config = config.with_password(password); } + config = config.with_tls(self.tls); let db_path_str = self .database @@ -421,6 +480,11 @@ impl ConnectionBuilder { "query_timeout is not supported on gRPC connections", )); } + if self.tls.mode() != TlsMode::Disable { + return Err(Error::feature_not_supported( + "tls is not supported on gRPC connections; use an https:// endpoint for TLS", + )); + } // Validate create_mode - gRPC is read-only if self.create_mode != CreateMode::DoNotCreate { diff --git a/hyperdb-api/tests/tls_tests.rs b/hyperdb-api/tests/tls_tests.rs new file mode 100644 index 00000000..941af106 --- /dev/null +++ b/hyperdb-api/tests/tls_tests.rs @@ -0,0 +1,477 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! TLS against a real `hyperd` started with `ssl_key` / `ssl_certificate`. +//! +//! Most servers here run with `ssl_force`, which makes hyperd reject any +//! plaintext startup. [`plaintext_is_rejected_by_a_force_server`] proves that, +//! and it is what makes every other successful connection evidence of TLS +//! rather than of a silent downgrade. + +mod common; + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +use hyperdb_api::{ + AsyncConnectionBuilder, AsyncInserter, Connection, ConnectionBuilder, CreateMode, Error, + HyperProcess, Inserter, SqlType, TableDefinition, TlsConfig, TlsMode, TransportMode, +}; +use tempfile::TempDir; + +/// Rows inserted by the `Require` round-trip tests; enough for several +/// COPY buffers and several result chunks. +const ROWS: i64 = 100_000; + +/// Scans 4e10 rows: about 11 s on an Apple M3 Max, where hyperd counts 4e9 +/// rows in 1.15 s. That is far past the 500 ms at which the cancel tests fire. If the cancel is lost, the +/// query completes and the test fails instead of hanging. +const SLOW_QUERY: &str = "SELECT count(*) FROM generate_series(1, 200000) a(i), \ + generate_series(1, 200000) b(i) WHERE a.i + b.i > 3"; + +/// When the cancel tests fire, relative to the query's start. +const CANCEL_AFTER: Duration = Duration::from_millis(500); + +/// A CA, the server certificate it signs, and two client certificates: one +/// from the same CA (accepted by hyperd) and one from an unrelated CA +/// (rejected). +struct Fixture { + dir: TempDir, +} + +impl Fixture { + fn new() -> Self { + let dir = tempfile::tempdir().expect("temp dir"); + let ca = generate_ca("hyperdb-api test CA"); + let other_ca = generate_ca("unrelated test CA"); + let server = generate_leaf(&ca, &["localhost", "127.0.0.1"]); + let client = generate_leaf(&ca, &[]); + let other_client = generate_leaf(&other_ca, &[]); + + let write = |name: &str, contents: &str| { + std::fs::write(dir.path().join(name), contents).expect("write fixture file"); + }; + write("ca.pem", &ca.cert_pem); + write("other_ca.pem", &other_ca.cert_pem); + // hyperd also uses this file as its client trust store, so it holds + // the CA after the leaf; that is what lets it verify `client.pem`. + write("server.pem", &format!("{}{}", server.cert_pem, ca.cert_pem)); + write("server.key", &server.key_pem); + write("client.pem", &client.cert_pem); + write("client.key", &client.key_pem); + write("other_client.pem", &other_client.cert_pem); + write("other_client.key", &other_client.key_pem); + restrict_to_owner(&dir.path().join("server.key")); + + Fixture { dir } + } + + fn path(&self, name: &str) -> PathBuf { + self.dir.path().join(name) + } +} + +struct TestCa { + issuer: rcgen::Issuer<'static, rcgen::KeyPair>, + cert_pem: String, +} + +struct TestCert { + cert_pem: String, + key_pem: String, +} + +fn generate_ca(name: &str) -> TestCa { + let mut params = rcgen::CertificateParams::new(Vec::::new()).expect("CA params"); + params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params + .distinguished_name + .push(rcgen::DnType::CommonName, name); + let key_pair = rcgen::KeyPair::generate().expect("CA key"); + let cert = params.self_signed(&key_pair).expect("self-sign CA"); + TestCa { + cert_pem: cert.pem(), + issuer: rcgen::Issuer::new(params, key_pair), + } +} + +/// A leaf signed by `ca`. `sans` may mix DNS names and IP literals. +fn generate_leaf(ca: &TestCa, sans: &[&str]) -> TestCert { + let sans = sans.iter().map(|s| (*s).to_string()).collect::>(); + let params = rcgen::CertificateParams::new(sans).expect("leaf params"); + let key_pair = rcgen::KeyPair::generate().expect("leaf key"); + let cert = params.signed_by(&key_pair, &ca.issuer).expect("sign leaf"); + TestCert { + cert_pem: cert.pem(), + key_pem: key_pair.serialize_pem(), + } +} + +/// hyperd refuses a key file that group or other can read. The check is +/// POSIX-only, so Windows needs nothing. +#[cfg(unix)] +fn restrict_to_owner(path: &Path) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).expect("chmod key"); +} + +#[cfg(not(unix))] +fn restrict_to_owner(_path: &Path) {} + +/// Starts a TCP hyperd serving `fixture`'s server certificate. With `force`, +/// it rejects every plaintext startup. +fn tls_hyper(name: &str, fixture: &Fixture, force: bool) -> HyperProcess { + let mut params = common::test_hyper_params(name).expect("test params"); + params.set_transport_mode(TransportMode::Tcp); + params.set("ssl_key", fixture.path("server.key").to_string_lossy()); + params.set( + "ssl_certificate", + fixture.path("server.pem").to_string_lossy(), + ); + if force { + params.set("ssl_force", "true"); + } + HyperProcess::new(None, Some(¶ms)).expect("start TLS hyperd") +} + +fn plain_hyper(name: &str) -> HyperProcess { + let mut params = common::test_hyper_params(name).expect("test params"); + params.set_transport_mode(TransportMode::Tcp); + HyperProcess::new(None, Some(¶ms)).expect("start hyperd") +} + +fn endpoint(hyper: &HyperProcess) -> String { + hyper.require_endpoint().expect("endpoint").to_string() +} + +/// `hyper`'s port behind `host`, so a test picks the host name or the IP +/// literal regardless of which one `require_endpoint` reports. +fn endpoint_on(hyper: &HyperProcess, host: &str) -> String { + let endpoint = endpoint(hyper); + let (_, port) = endpoint + .rsplit_once(':') + .unwrap_or_else(|| panic!("no port in {endpoint}")); + format!("{host}:{port}") +} + +fn verify_ca(fixture: &Fixture) -> TlsConfig { + TlsConfig::new(TlsMode::VerifyCa).root_cert(fixture.path("ca.pem")) +} + +fn verify_full(fixture: &Fixture) -> TlsConfig { + TlsConfig::new(TlsMode::VerifyFull).root_cert(fixture.path("ca.pem")) +} + +fn connect(endpoint: &str, tls: impl Into) -> hyperdb_api::Result { + ConnectionBuilder::new(endpoint).tls(tls).build() +} + +fn assert_tls_error(result: hyperdb_api::Result) -> String { + match result { + Err(Error::Tls(message)) => message, + other => panic!("expected Error::Tls, got {other:?}"), + } +} + +/// Inserts [`ROWS`] rows through COPY, then reads them back as several chunks. +fn round_trip_sync(conn: &Connection) { + let table = TableDefinition::new("tls_rows").add_required_column("id", SqlType::big_int()); + conn.execute_command(&table.to_create_sql(true).expect("create sql")) + .expect("create table"); + let mut inserter = Inserter::new(conn, &table).expect("inserter"); + for id in 0..ROWS { + inserter.add_i64(id).expect("add"); + inserter.end_row().expect("end row"); + } + assert_eq!( + inserter.execute().expect("COPY"), + u64::try_from(ROWS).unwrap() + ); + + let mut rowset = conn + .execute_query("SELECT id FROM tls_rows ORDER BY id") + .expect("query"); + let (mut seen, mut chunks) = (0_i64, 0); + while let Some(chunk) = rowset.next_chunk().expect("chunk") { + chunks += 1; + for row in &chunk { + assert_eq!(row.get::(0), Some(seen)); + seen += 1; + } + } + assert_eq!(seen, ROWS); + assert!(chunks > 1, "expected a multi-chunk result, got {chunks}"); +} + +#[test] +fn plaintext_is_rejected_by_a_force_server() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_force_rejects_plaintext", &fixture, true); + + let err = connect(&endpoint(&hyper), TlsMode::Disable).expect_err("plaintext must fail"); + assert!(err.to_string().contains("SSL"), "got {err}"); +} + +#[test] +fn require_sync_round_trip() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_require_sync", &fixture, true); + let db = common::test_result_path("tls_require_sync", "hyper").unwrap(); + + let conn = ConnectionBuilder::new(endpoint(&hyper)) + .database(&db) + .create_mode(CreateMode::CreateAndReplace) + .tls(TlsMode::Require) + .build() + .expect("connect over TLS"); + assert!(conn.is_tls()); + round_trip_sync(&conn); +} + +#[tokio::test] +async fn require_async_round_trip() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_require_async", &fixture, true); + let db = common::test_result_path("tls_require_async", "hyper").unwrap(); + + let conn = AsyncConnectionBuilder::new(endpoint(&hyper)) + .database(&db) + .create_mode(CreateMode::CreateAndReplace) + .tls(TlsMode::Require) + .build() + .await + .expect("connect over TLS"); + assert!(conn.is_tls()); + + let table = TableDefinition::new("tls_rows").add_required_column("id", SqlType::big_int()); + conn.execute_command(&table.to_create_sql(true).expect("create sql")) + .await + .expect("create table"); + let mut inserter = AsyncInserter::new(&conn, &table).expect("inserter"); + for id in 0..ROWS { + inserter.add_i64(id).expect("add"); + inserter.end_row().await.expect("end row"); + } + assert_eq!( + inserter.execute().await.expect("COPY"), + u64::try_from(ROWS).unwrap() + ); + + let mut rowset = conn + .execute_query("SELECT id FROM tls_rows ORDER BY id") + .await + .expect("query"); + let (mut seen, mut chunks) = (0_i64, 0); + while let Some(chunk) = rowset.next_chunk().await.expect("chunk") { + chunks += 1; + for row in &chunk { + assert_eq!(row.get::(0), Some(seen)); + seen += 1; + } + } + assert_eq!(seen, ROWS); + assert!(chunks > 1, "expected a multi-chunk result, got {chunks}"); +} + +#[test] +fn verify_ca_checks_the_issuer() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_verify_ca", &fixture, true); + let endpoint = endpoint(&hyper); + + let conn = connect(&endpoint, verify_ca(&fixture)).expect("right CA"); + assert!(conn.is_tls()); + + let wrong = TlsConfig::new(TlsMode::VerifyCa).root_cert(fixture.path("other_ca.pem")); + assert_tls_error(connect(&endpoint, wrong)); + + match connect(&endpoint, TlsMode::VerifyCa) { + Err(Error::Config(_)) => {} + other => panic!("VerifyCa without a root certificate: expected Config, got {other:?}"), + } +} + +#[test] +fn verify_full_checks_the_host_name() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_verify_full", &fixture, true); + + // The certificate has both as SANs: a DNS name and an IP address. + for endpoint in [ + endpoint_on(&hyper, "localhost"), + endpoint_on(&hyper, "127.0.0.1"), + ] { + let conn = connect(&endpoint, verify_full(&fixture)) + .unwrap_or_else(|err| panic!("VerifyFull against {endpoint}: {err}")); + assert!(conn.is_tls()); + } + + let endpoint = endpoint(&hyper); + assert_tls_error(connect( + &endpoint, + verify_full(&fixture).server_name("wrong.example"), + )); + // VerifyCa skips the host-name check, so the same override passes. + let conn = connect(&endpoint, verify_ca(&fixture).server_name("wrong.example")) + .expect("VerifyCa ignores the host name"); + assert!(conn.is_tls()); +} + +#[test] +fn prefer_uses_tls_when_offered() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_prefer_tls_server", &fixture, true); + + let conn = connect(&endpoint(&hyper), TlsMode::Prefer).expect("Prefer"); + assert!(conn.is_tls()); +} + +#[test] +fn prefer_falls_back_to_plaintext_without_a_certificate() { + let hyper = plain_hyper("tls_prefer_plain_server"); + + let conn = connect(&endpoint(&hyper), TlsMode::Prefer).expect("Prefer"); + assert!(!conn.is_tls()); +} + +/// A failed verification under `Prefer` is an error, not a reason to retry +/// in plaintext: only a server that declines TLS is a fallback. +#[test] +fn prefer_does_not_downgrade_on_a_verification_failure() { + let fixture = Fixture::new(); + // Without ssl_force, a plaintext retry would succeed, so this would pass + // only if the client never attempted one. + let hyper = tls_hyper("tls_prefer_wrong_ca", &fixture, false); + + let prefer = TlsConfig::new(TlsMode::Prefer).root_cert(fixture.path("other_ca.pem")); + assert_tls_error(connect(&endpoint(&hyper), prefer)); +} + +#[test] +fn require_fails_against_a_server_without_tls() { + let hyper = plain_hyper("tls_require_plain_server"); + + let message = assert_tls_error(connect(&endpoint(&hyper), TlsMode::Require)); + assert!(message.contains("does not support TLS"), "got {message}"); +} + +#[test] +fn client_certificate_from_the_trusted_ca_is_accepted() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_mtls_accepted", &fixture, true); + + let tls = + verify_full(&fixture).client_cert(fixture.path("client.pem"), fixture.path("client.key")); + let conn = connect(&endpoint(&hyper), tls).expect("client certificate from the CA"); + assert!(conn.is_tls()); + let one: i64 = conn.fetch_scalar("SELECT 1").expect("query"); + assert_eq!(one, 1); +} + +#[test] +fn client_certificate_from_another_ca_is_rejected() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_mtls_rejected", &fixture, true); + + let tls = verify_full(&fixture).client_cert( + fixture.path("other_client.pem"), + fixture.path("other_client.key"), + ); + // Under TLS 1.3 the rejection can arrive after the handshake, on the + // first read; the builder's startup exchange is that read. + assert_tls_error(connect(&endpoint(&hyper), tls)); +} + +/// hyperd accepts a cancel request in plaintext too, even with `ssl_force`, +/// so this proves only that cancelling a TLS connection works. That the +/// request itself goes over TLS is covered by the fake-server tests in +/// `hyperdb_api_core::client::tls`. +#[test] +fn cancel_under_tls_sync() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_cancel_sync", &fixture, true); + + let conn = Arc::new(connect(&endpoint(&hyper), TlsMode::Require).expect("connect")); + assert!(conn.is_tls()); + + let query_conn = Arc::clone(&conn); + let start = Instant::now(); + let query = thread::spawn(move || query_conn.fetch_scalar::(SLOW_QUERY)); + thread::sleep(CANCEL_AFTER); + conn.cancel().expect("send cancel over TLS"); + + let err = query + .join() + .expect("query thread") + .expect_err("the slow query must be cancelled"); + assert_eq!(err.sqlstate(), Some("57014"), "got {err}"); + assert!( + start.elapsed() < Duration::from_secs(5), + "took {:?}", + start.elapsed() + ); +} + +/// See [`cancel_under_tls_sync`] for what this does and does not prove. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancel_under_tls_async() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_cancel_async", &fixture, true); + + let conn = AsyncConnectionBuilder::new(endpoint(&hyper)) + .tls(TlsMode::Require) + .build() + .await + .expect("connect"); + assert!(conn.is_tls()); + + let start = Instant::now(); + let (result, cancel) = tokio::join!(conn.fetch_scalar::(SLOW_QUERY), async { + tokio::time::sleep(CANCEL_AFTER).await; + conn.cancel().await + }); + cancel.expect("send cancel over TLS"); + let err = result.expect_err("the slow query must be cancelled"); + assert_eq!(err.sqlstate(), Some("57014"), "got {err}"); + assert!( + start.elapsed() < Duration::from_secs(5), + "took {:?}", + start.elapsed() + ); +} + +/// gRPC TLS is chosen by the `https://` scheme; `tls()` is rejected before +/// any connection attempt, so the unreachable port is never dialled. +#[test] +fn tls_on_grpc_is_rejected() { + let err = ConnectionBuilder::new("http://localhost:1") + .tls(TlsMode::Require) + .build() + .expect_err("gRPC + tls must fail"); + assert!(matches!(err, Error::FeatureNotSupported(_)), "got {err:?}"); +} + +#[cfg(unix)] +#[test] +fn ipc_rejects_required_tls_and_ignores_prefer() { + let mut params = common::test_hyper_params("tls_ipc").expect("test params"); + params.set_transport_mode(TransportMode::Ipc); + let hyper = HyperProcess::new(None, Some(¶ms)).expect("start IPC hyperd"); + // The socket path. `require_endpoint` keeps the descriptor's `/domain/` + // separator, which is not a filesystem path. + let endpoint = hyper.connection_endpoint_string().expect("IPC endpoint"); + assert!(endpoint.starts_with('/'), "not a socket path: {endpoint}"); + + for mode in [TlsMode::Require, TlsMode::VerifyCa, TlsMode::VerifyFull] { + match connect(&endpoint, mode) { + Err(Error::FeatureNotSupported(_)) => {} + other => { + panic!("{mode} over a Unix socket: expected FeatureNotSupported, got {other:?}") + } + } + } + let conn = connect(&endpoint, TlsMode::Prefer).expect("Prefer over a Unix socket"); + assert!(!conn.is_tls()); +} From 84ab5985e8a977b5731d309f9b41dc2baec2d0a9 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 03:15:40 -0700 Subject: [PATCH 23/61] feat(api)!: TLS for both connection pools PoolConfig and SyncPoolConfig gain a tls field and tls() builder, applied to every connection the pool opens. Both pools now open connections through AsyncConnectionBuilder / ConnectionBuilder unconditionally, so the async pool also picks up the builder's endpoint parsing (a malformed port is a Config error), IPC routing for authenticated pools, and its gRPC create-mode check. BREAKING CHANGE: PoolConfig and SyncPoolConfig have a new public field, so struct-literal construction no longer compiles. --- hyperdb-api/CHANGELOG.md | 10 +++++ hyperdb-api/src/pool.rs | 74 +++++++++++++++++++++++++--------- hyperdb-api/tests/tls_tests.rs | 61 ++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 88ffcc7f..0f92d61a 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -107,6 +107,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `tls()` the same way. - The `Connection::connect*` shortcuts and `Connection::new` stay plaintext. + - **BREAKING:** `PoolConfig` and `SyncPoolConfig` gain a public `tls` field + and a `tls()` builder, applied to every connection the pool opens, so a + struct-literal `PoolConfig { .. }` no longer compiles. The async pool now + opens connections through `AsyncConnectionBuilder`: a malformed port in + its endpoint is an `Error::Config` instead of silently becoming 7483, and + an authenticated pool on a Unix-socket or named-pipe endpoint connects + there instead of misreading the path as a TCP host. On a gRPC endpoint a + `create_mode` other than `DoNotCreate` now fails `Pool::get` with + `Error::FeatureNotSupported` instead of being ignored, as it already did + for the sync pool. - TLS session resumption is disabled: `hyperd` requests client certificates without an OpenSSL session ID context, so it aborts every resumed handshake (and a cancel would resume). diff --git a/hyperdb-api/src/pool.rs b/hyperdb-api/src/pool.rs index 17a9337c..38656532 100644 --- a/hyperdb-api/src/pool.rs +++ b/hyperdb-api/src/pool.rs @@ -83,6 +83,9 @@ //! long an acquire may block. The async pool enforces all three on the Tokio //! runtime; the sync pool enforces `wait_timeout` natively (see //! [`SyncPoolConfig`] for the create/recycle caveat). +//! - **TLS** ([`PoolConfig::tls`] / [`SyncPoolConfig::tls`]) applies to every +//! connection the pool opens, with the semantics of +//! [`ConnectionBuilder::tls`]. //! //! # Lifecycle hooks (async pool only) //! @@ -121,10 +124,10 @@ use deadpool::Runtime; use deadpool::managed::{self, Manager, Metrics, RecycleError, RecycleResult, Timeouts}; use tokio::sync::Mutex as AsyncMutex; -use crate::CreateMode; use crate::async_connection::AsyncConnection; use crate::connection::Connection; use crate::error::{Error, Result}; +use crate::{AsyncConnectionBuilder, ConnectionBuilder, CreateMode, TlsConfig}; /// Future returned by pool lifecycle hooks. /// @@ -240,6 +243,8 @@ pub struct PoolConfig { pub user: Option, /// Optional password for authentication pub password: Option, + /// TLS for every connection the pool opens (see [`tls`](Self::tls)). + pub tls: TlsConfig, /// Maximum number of connections in the pool pub max_size: usize, /// If `false`, skip the per-checkout health probe. Retained for backwards @@ -282,6 +287,7 @@ impl std::fmt::Debug for PoolConfig { .field("create_mode", &self.create_mode) .field("user", &self.user) .field("password", &self.password.as_ref().map(|_| "")) + .field("tls", &self.tls) .field("max_size", &self.max_size) .field("health_check", &self.health_check) .field("recycle", &self.recycle) @@ -316,6 +322,7 @@ impl PoolConfig { create_mode: CreateMode::DoNotCreate, user: None, password: None, + tls: TlsConfig::default(), max_size: 16, health_check: true, recycle: RecycleStrategy::SelectOne, @@ -345,6 +352,18 @@ impl PoolConfig { self } + /// Sets the TLS configuration for every connection the pool opens. + /// + /// Same semantics as [`AsyncConnectionBuilder::tls`]; the default is + /// [`TlsMode::Disable`](crate::TlsMode::Disable). A mode the endpoint + /// cannot honor surfaces from [`Pool::get`], since connections open + /// lazily. + #[must_use] + pub fn tls(mut self, tls: impl Into) -> Self { + self.tls = tls.into(); + self + } + /// Sets the maximum pool size. #[must_use] pub fn max_size(mut self, size: usize) -> Self { @@ -490,18 +509,14 @@ impl ConnectionManager { } async fn open(&self, mode: CreateMode) -> Result { + let mut builder = AsyncConnectionBuilder::new(&self.config.endpoint) + .database(&self.config.database) + .create_mode(mode) + .tls(self.config.tls.clone()); if let (Some(user), Some(password)) = (&self.config.user, &self.config.password) { - AsyncConnection::connect_with_auth( - &self.config.endpoint, - &self.config.database, - mode, - user, - password, - ) - .await - } else { - AsyncConnection::connect(&self.config.endpoint, &self.config.database, mode).await + builder = builder.user(user).password(password); } + builder.build().await } } @@ -791,6 +806,8 @@ pub struct SyncPoolConfig { pub user: Option, /// Optional password for authentication. pub password: Option, + /// TLS for every connection the pool opens (see [`tls`](Self::tls)). + pub tls: TlsConfig, /// Maximum number of connections in the pool. pub max_size: usize, /// Per-checkout recycle strategy. Defaults to [`SyncRecycleStrategy::SelectOne`]. @@ -815,6 +832,7 @@ impl std::fmt::Debug for SyncPoolConfig { .field("create_mode", &self.create_mode) .field("user", &self.user) .field("password", &self.password.as_ref().map(|_| "")) + .field("tls", &self.tls) .field("max_size", &self.max_size) .field("recycle", &self.recycle) .field("wait_timeout", &self.wait_timeout) @@ -834,6 +852,7 @@ impl SyncPoolConfig { create_mode: CreateMode::DoNotCreate, user: None, password: None, + tls: TlsConfig::default(), max_size: 16, recycle: SyncRecycleStrategy::SelectOne, wait_timeout: None, @@ -858,6 +877,18 @@ impl SyncPoolConfig { self } + /// Sets the TLS configuration for every connection the pool opens. + /// + /// Same semantics as [`ConnectionBuilder::tls`]; the default is + /// [`TlsMode::Disable`](crate::TlsMode::Disable). A mode the endpoint + /// cannot honor surfaces from [`ConnectionPool::get`], since connections + /// open lazily. + #[must_use] + pub fn tls(mut self, tls: impl Into) -> Self { + self.tls = tls.into(); + self + } + /// Sets the maximum pool size. #[must_use] pub fn max_size(mut self, size: usize) -> Self { @@ -956,17 +987,14 @@ impl SyncPoolInner { } else { CreateMode::DoNotCreate }; + let mut builder = ConnectionBuilder::new(&self.config.endpoint) + .database(&self.config.database) + .create_mode(mode) + .tls(self.config.tls.clone()); if let (Some(user), Some(password)) = (&self.config.user, &self.config.password) { - Connection::connect_with_auth( - &self.config.endpoint, - &self.config.database, - mode, - user, - password, - ) - } else { - Connection::connect(&self.config.endpoint, &self.config.database, mode) + builder = builder.user(user).password(password); } + builder.build() } /// Returns `true` if an idle connection should be retired before reuse. @@ -1304,6 +1332,7 @@ mod tests { let config = PoolConfig::new("localhost:7483", "test.hyper") .create_mode(CreateMode::CreateIfNotExists) .auth("user", "pass") + .tls(crate::TlsMode::Require) .max_size(32); assert_eq!(config.endpoint, "localhost:7483"); @@ -1312,6 +1341,7 @@ mod tests { assert_eq!(config.user, Some("user".to_string())); assert_eq!(config.password, Some("pass".to_string())); assert_eq!(config.max_size, 32); + assert_eq!(config.tls.mode(), crate::TlsMode::Require); } #[test] @@ -1327,6 +1357,7 @@ mod tests { assert_eq!(config.max_lifetime, None); assert_eq!(config.idle_timeout, None); assert_eq!(config.min_idle, None); + assert_eq!(config.tls, TlsConfig::default()); assert!(!config.has_timeout()); } @@ -1375,6 +1406,7 @@ mod tests { assert_eq!(config.max_lifetime, None); assert_eq!(config.idle_timeout, None); assert_eq!(config.min_idle, None); + assert_eq!(config.tls, TlsConfig::default()); let password: String = { use rand::RngExt; @@ -1385,12 +1417,14 @@ mod tests { .create_mode(CreateMode::CreateIfNotExists) .auth("u", password) .max_size(4) + .tls(crate::TlsMode::VerifyFull) .recycle(SyncRecycleStrategy::Ping) .wait_timeout(Some(Duration::from_millis(500))) .max_lifetime(Some(Duration::from_secs(10))) .idle_timeout(Some(Duration::from_secs(5))) .min_idle(Some(1)); assert_eq!(tuned.max_size, 4); + assert_eq!(tuned.tls.mode(), crate::TlsMode::VerifyFull); assert!(matches!(tuned.recycle, SyncRecycleStrategy::Ping)); assert_eq!(tuned.user, Some("u".to_string())); assert_eq!(tuned.wait_timeout, Some(Duration::from_millis(500))); diff --git a/hyperdb-api/tests/tls_tests.rs b/hyperdb-api/tests/tls_tests.rs index 941af106..b113952a 100644 --- a/hyperdb-api/tests/tls_tests.rs +++ b/hyperdb-api/tests/tls_tests.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; +use hyperdb_api::pool::{PoolConfig, SyncPoolConfig, create_pool}; use hyperdb_api::{ AsyncConnectionBuilder, AsyncInserter, Connection, ConnectionBuilder, CreateMode, Error, HyperProcess, Inserter, SqlType, TableDefinition, TlsConfig, TlsMode, TransportMode, @@ -442,6 +443,66 @@ async fn cancel_under_tls_async() { ); } +/// Every connection the pool opens is verified TLS, including the first +/// one, which creates the database. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_pool_connects_over_tls() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_async_pool", &fixture, true); + let db = common::test_result_path("tls_async_pool", "hyper").unwrap(); + + let config = PoolConfig::new(endpoint(&hyper), db.to_string_lossy()) + .create_mode(CreateMode::CreateAndReplace) + .tls(verify_full(&fixture)) + .max_size(2); + let pool = create_pool(config).expect("pool"); + + let (first, second) = tokio::join!(pool.get(), pool.get()); + for conn in [first.expect("first"), second.expect("second")] { + assert!(conn.is_tls()); + let one: i64 = conn.fetch_scalar("SELECT 1").await.expect("query"); + assert_eq!(one, 1); + } + assert_eq!(pool.status().size, 2); +} + +#[test] +fn sync_pool_connects_over_tls() { + let fixture = Fixture::new(); + let hyper = tls_hyper("tls_sync_pool", &fixture, true); + let db = common::test_result_path("tls_sync_pool", "hyper").unwrap(); + + let pool = SyncPoolConfig::new(endpoint(&hyper), db.to_string_lossy()) + .create_mode(CreateMode::CreateAndReplace) + .tls(verify_full(&fixture)) + .max_size(2) + .build(); + + let first = pool.get().expect("first"); + let second = pool.get().expect("second"); + for conn in [&first, &second] { + assert!(conn.is_tls()); + let one: i64 = conn.fetch_scalar("SELECT 1").expect("query"); + assert_eq!(one, 1); + } + assert_eq!(pool.status().size, 2); +} + +/// Pools open connections lazily, so a TLS failure surfaces from `get`. +#[tokio::test] +async fn pools_report_tls_failures_from_get() { + let hyper = plain_hyper("tls_pool_plain_server"); + + let pool = create_pool(PoolConfig::new(endpoint(&hyper), "unused").tls(TlsMode::Require)) + .expect("pool"); + assert_tls_error(pool.get().await.map(|_| ())); + + let pool = SyncPoolConfig::new(endpoint(&hyper), "unused") + .tls(TlsMode::Require) + .build(); + assert_tls_error(pool.get().map(|_| ())); +} + /// gRPC TLS is chosen by the `https://` scheme; `tls()` is rejected before /// any connection attempt, so the unreachable port is never dialled. #[test] From 1a3fe708cb7210d0735e5f3551357a03463dc1e7 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 03:28:03 -0700 Subject: [PATCH 24/61] feat(node): TLS options for ConnectionBuilder, ConnectionPool and HyperProcess Expose the Rust TLS support to the Node bindings: - ConnectionBuilder.tls({ mode, rootCert, clientCert, clientKey, serverName }) with libpq sslmode spellings, and Connection.isTls. - ConnectionPool's `tls` option; only a TLS pool goes through ConnectionBuilder, so a plain pool keeps Connection.connect's behavior. - new HyperProcess(hyperPath, { transport, parameters }) to start a TCP hyperd with ssl_key / ssl_certificate. __test__/tls.mjs runs against an ssl_force hyperd with openssl-generated certificates and is part of `npm test`; it fails on CI without openssl. --- hyperdb-api-node/AGENTS.md | 5 +- hyperdb-api-node/CHANGELOG.md | 15 +++ hyperdb-api-node/__test__/tls.mjs | 164 +++++++++++++++++++++++++++++ hyperdb-api-node/index.d.ts | 54 +++++++++- hyperdb-api-node/package.json | 2 +- hyperdb-api-node/pool.mjs | 28 +++-- hyperdb-api-node/src/connection.rs | 80 ++++++++++++++ hyperdb-api-node/src/process.rs | 48 ++++++++- 8 files changed, 384 insertions(+), 12 deletions(-) create mode 100644 hyperdb-api-node/__test__/tls.mjs diff --git a/hyperdb-api-node/AGENTS.md b/hyperdb-api-node/AGENTS.md index 7e824bdd..18b3ae66 100644 --- a/hyperdb-api-node/AGENTS.md +++ b/hyperdb-api-node/AGENTS.md @@ -64,7 +64,7 @@ npm run build:debug # Build native addon (release — slow compile, fast runtime) npm run build -# Run smoke tests +# Run the smoke and TLS tests npm test # Run benchmarks (build release first!) @@ -111,6 +111,7 @@ When modifying connection behavior, check both `src/connection.rs` (Rust) and `i ## Testing - **Smoke tests:** `__test__/smoke.mjs` — covers all major features (connection, queries, inserts, streams, pool, tagged templates, BigInt, dates, JSON) +- **TLS tests:** `__test__/tls.mjs` — `ConnectionBuilder.tls()`, `ConnectionPool`'s `tls` option and `HyperProcess` options against an `ssl_force` server; certificates come from the `openssl` CLI, and the test skips without it. Its certificates and database live in a temporary directory it removes - **Benchmarks:** `__test__/benchmark.mjs` — insert and query performance with configurable row counts - Tests need `hyperd`: run `make download-hyperd` (found via `.hyperd/current`), or set an absolute `HYPERD_PATH` - Test artifacts go into `test_results/` (gitignored) @@ -127,7 +128,7 @@ Uses napi-rs platform packages for cross-platform prebuilt binaries: Only these three platforms are published; macOS x64 (Intel), Linux x64 (musl) and Linux ARM64 have no prebuilt package. -On every PR and push, `ci.yml` builds the Linux debug addon and runs the smoke test (the `hyperdb-api-node (build + smoke)` job). The per-platform release builds run in `npm-build-publish.yml`, which the hand-created GitHub Release triggers; release-please only prepares the version bump and changelog in the release PR. +On every PR and push, `ci.yml` builds the Linux debug addon and runs `npm test`, the smoke and TLS tests (the `hyperdb-api-node (build + smoke)` job). The per-platform release builds run in `npm-build-publish.yml`, which the hand-created GitHub Release triggers; release-please only prepares the version bump and changelog in the release PR. ## Common Development Scenarios diff --git a/hyperdb-api-node/CHANGELOG.md b/hyperdb-api-node/CHANGELOG.md index 00da2639..23523240 100644 --- a/hyperdb-api-node/CHANGELOG.md +++ b/hyperdb-api-node/CHANGELOG.md @@ -13,6 +13,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +* **TLS for TCP connections:** `ConnectionBuilder.tls({ mode, rootCert, + clientCert, clientKey, serverName })`, with libpq `sslmode` semantics + (`"disable"`, `"prefer"`, `"require"`, `"verify-ca"`, `"verify-full"`), and + a `Connection.isTls` getter. An invalid mode, or `clientCert` without + `clientKey`, fails `build()`. `Connection.connect` and the other static + factories stay plaintext. +* **`ConnectionPool` takes a `tls` option** and opens every connection through + `ConnectionBuilder` with it. TLS failures surface from `acquire()`. +* **`new HyperProcess(hyperPath, options)`** takes an optional + `{ transport: 'ipc' | 'tcp', parameters }`; `parameters` are passed to + `hyperd` unchanged, for example `ssl_key` and `ssl_certificate` to serve + TLS. An unknown transport throws. + ## [1.0.0-rc.3] - 2026-09-07 ### Changed diff --git a/hyperdb-api-node/__test__/tls.mjs b/hyperdb-api-node/__test__/tls.mjs new file mode 100644 index 00000000..bc096f4f --- /dev/null +++ b/hyperdb-api-node/__test__/tls.mjs @@ -0,0 +1,164 @@ +/** + * TLS test for hyperdb-api-node against a real `hyperd` started with + * `ssl_key` / `ssl_certificate` / `ssl_force`. + * + * `ssl_force` makes hyperd reject any plaintext startup; the first case + * proves that, which is what makes every later successful connection + * evidence of TLS. Certificates come from the `openssl` CLI; without it the + * test is skipped, except under CI, where it fails. + * + * Run with: node __test__/tls.mjs + */ + +import { strict as assert } from 'assert'; +import { execFileSync } from 'child_process'; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { createRequire } from 'module'; +import { tmpdir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +const require = createRequire(import.meta.url); + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const { HyperProcess, Connection, ConnectionBuilder, CreateMode } = require('../index.js'); +const { ConnectionPool } = await import(join(__dirname, '..', 'pool.mjs')); + +function openssl(dir, ...args) { + execFileSync('openssl', args, { cwd: dir, stdio: 'pipe' }); +} + +function hasOpenssl() { + try { + execFileSync('openssl', ['version'], { stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +/** A CA, a server certificate it signs, and an unrelated CA. */ +function generateCertificates(dir) { + const newCa = (name, cn) => + openssl(dir, 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-keyout', `${name}.key`, '-out', `${name}.pem`, '-subj', `/CN=${cn}`); + newCa('ca', 'hyperdb-api-node test CA'); + newCa('other_ca', 'unrelated test CA'); + + openssl(dir, 'req', '-newkey', 'rsa:2048', '-nodes', '-keyout', 'server.key', + '-out', 'server.csr', '-subj', '/CN=localhost'); + writeFileSync(join(dir, 'san.ext'), 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + openssl(dir, 'x509', '-req', '-in', 'server.csr', '-CA', 'ca.pem', '-CAkey', 'ca.key', + '-CAcreateserial', '-days', '1', '-out', 'leaf.pem', '-extfile', 'san.ext'); + // hyperd also uses this file as its client trust store, so it holds the + // CA after the leaf. + writeFileSync( + join(dir, 'server.pem'), + readFileSync(join(dir, 'leaf.pem'), 'utf8') + readFileSync(join(dir, 'ca.pem'), 'utf8'), + ); + // hyperd refuses a key file that group or other can read. + chmodSync(join(dir, 'server.key'), 0o600); +} + +/** `endpoint`'s port behind `host`. */ +function onHost(endpoint, host) { + return `${host}:${endpoint.slice(endpoint.lastIndexOf(':') + 1)}`; +} + +function connect(endpoint, tls) { + return new ConnectionBuilder(endpoint).tls(tls).build(); +} + +async function main() { + console.log('=== hyperdb-api-node TLS test ===\n'); + if (!hasOpenssl()) { + if (process.env.CI) throw new Error('openssl not found; the TLS test needs it on CI'); + console.log('openssl not found; skipping the TLS test.'); + return; + } + + const dir = mkdtempSync(join(tmpdir(), 'hyperdb-node-tls-')); + let hyper; + try { + generateCertificates(dir); + hyper = new HyperProcess(undefined, { + transport: 'tcp', + parameters: { + ssl_key: join(dir, 'server.key'), + ssl_certificate: join(dir, 'server.pem'), + ssl_force: 'true', + }, + }); + const endpoint = hyper.endpoint; + const rootCert = join(dir, 'ca.pem'); + console.log(` Endpoint: ${endpoint}`); + + console.log('1. Plaintext is rejected by the ssl_force server...'); + await assert.rejects(Connection.withoutDatabase(endpoint), /SSL/); + await assert.rejects(connect(endpoint, { mode: 'disable' }), /SSL/); + + console.log('2. require: connects over TLS and runs a query...'); + { + const conn = await connect(endpoint, { mode: 'require' }); + assert.equal(conn.isTls, true); + const rows = await conn.executeQuery('SELECT 41 + 1'); + assert.equal(rows[0].getInt32(0), 42); + await conn.close(); + } + + console.log('3. verify-full: host name and IP literal both match the SANs...'); + for (const host of ['localhost', '127.0.0.1']) { + const conn = await connect(onHost(endpoint, host), { mode: 'verify-full', rootCert }); + assert.equal(conn.isTls, true, host); + await conn.close(); + } + + console.log('4. Verification failures...'); + await assert.rejects( + connect(endpoint, { mode: 'verify-ca', rootCert: join(dir, 'other_ca.pem') }), + /TLS error/, + ); + await assert.rejects( + connect(endpoint, { mode: 'verify-full', rootCert, serverName: 'wrong.example' }), + /TLS error/, + ); + + console.log('5. Invalid options fail build()...'); + await assert.rejects(connect(endpoint, { mode: 'verify_full' }), /verify_full/); + await assert.rejects( + connect(endpoint, { mode: 'require', clientCert: join(dir, 'leaf.pem') }), + /clientCert and clientKey must be set together/, + ); + + console.log('6. ConnectionPool with tls...'); + { + const pool = new ConnectionPool(endpoint, join(dir, 'pool.hyper'), { + createMode: CreateMode.CreateAndReplace, + tls: { mode: 'verify-full', rootCert }, + }); + const conn = await pool.acquire(); + assert.equal(conn.isTls, true); + pool.release(conn); + assert.equal((await pool.query('SELECT 1')).length, 1); + await pool.close(); + + const plaintext = new ConnectionPool(endpoint, join(dir, 'pool.hyper'), { + createMode: CreateMode.DoNotCreate, + }); + await assert.rejects(plaintext.acquire(), /SSL/); + await plaintext.close(); + } + + console.log('7. An unknown HyperProcess transport is rejected...'); + assert.throws(() => new HyperProcess(undefined, { transport: 'udp' }), /unknown transport `udp`/); + } finally { + hyper?.close(); + rmSync(dir, { recursive: true, force: true }); + } + + console.log('\n=== All TLS tests passed! ==='); +} + +main().catch((err) => { + console.error('\nTLS test FAILED:', err); + process.exit(1); +}); diff --git a/hyperdb-api-node/index.d.ts b/hyperdb-api-node/index.d.ts index dac2bdf0..52ad313f 100644 --- a/hyperdb-api-node/index.d.ts +++ b/hyperdb-api-node/index.d.ts @@ -89,6 +89,39 @@ export const enum CreateMode { /** Drop and recreate the database. */ CreateAndReplace = 'CreateAndReplace' } +/** + * TLS settings for `ConnectionBuilder.tls()`, with libpq `sslmode` + * semantics. + */ +export interface TlsOptions { + /** `"disable"`, `"prefer"`, `"require"`, `"verify-ca"` or `"verify-full"`. */ + mode: string + /** + * PEM file of trusted root certificates. Required by `"verify-ca"` and + * `"verify-full"`; with `"prefer"` or `"require"` it turns on chain + * verification. + */ + rootCert?: string + /** PEM client certificate for mutual TLS. Requires `clientKey`. */ + clientCert?: string + /** PEM private key for `clientCert`. */ + clientKey?: string + /** Host name to verify and send as SNI instead of the endpoint's host. */ + serverName?: string +} +/** Options for starting a `HyperProcess`. */ +export interface HyperProcessOptions { + /** + * `"ipc"` (the default: a Unix domain socket, or a named pipe on + * Windows) or `"tcp"`. TLS needs `"tcp"`. + */ + transport?: string + /** + * `hyperd` settings, passed through unchanged, for example `ssl_key` + * and `ssl_certificate` to serve TLS. + */ + parameters?: Record +} /** Information about a single column in a table definition. */ export interface ColumnInfo { /** Column name. */ @@ -311,6 +344,14 @@ export declare class Connection { get database(): string | null /** Returns true if the connection is alive. */ get isAlive(): boolean + /** + * Returns true if the session is encrypted with TLS. + * + * `false` for a plaintext TCP connection, including a + * `{ mode: 'prefer' }` connection whose server declined TLS, and for a + * Unix domain socket or named pipe. + */ + get isTls(): boolean /** * Closes the connection. * @@ -409,6 +450,16 @@ export declare class ConnectionBuilder { password(password: string): this /** Sets the login timeout in milliseconds. */ loginTimeout(ms: number): this + /** + * Sets the TLS options for a TCP connection; the default is + * `{ mode: 'disable' }`. + * + * Over a Unix domain socket or a named pipe, `"prefer"` connects in + * plaintext and the modes that require TLS fail `build()`. A gRPC + * endpoint picks TLS through its `https://` scheme, so any mode but + * `"disable"` fails there too. The options are checked by `build()`. + */ + tls(options: TlsOptions): this /** Builds and establishes the connection. */ build(): Promise } @@ -522,8 +573,9 @@ export declare class HyperProcess { * path if it's not in the standard location. * * @param hyperPath - Optional path to the `hyperd` binary. + * @param options - Optional transport and `hyperd` settings. */ - constructor(hyperPath?: string | undefined | null) + constructor(hyperPath?: string | undefined | null, options?: HyperProcessOptions | undefined | null) /** * Returns the server endpoint (e.g., "localhost:7483"). * diff --git a/hyperdb-api-node/package.json b/hyperdb-api-node/package.json index ac75e717..91b6727a 100644 --- a/hyperdb-api-node/package.json +++ b/hyperdb-api-node/package.json @@ -17,7 +17,7 @@ "artifacts": "napi artifacts", "prepublishOnly": "napi prepublish -t npm", "version": "napi version", - "test": "node __test__/smoke.mjs", + "test": "node __test__/smoke.mjs && node __test__/tls.mjs", "benchmark": "node __test__/benchmark.mjs", "example:ts": "npx tsx examples/typed-analytics.mts" }, diff --git a/hyperdb-api-node/pool.mjs b/hyperdb-api-node/pool.mjs index be6b20e3..a4359178 100644 --- a/hyperdb-api-node/pool.mjs +++ b/hyperdb-api-node/pool.mjs @@ -33,7 +33,7 @@ import { createRequire } from 'module'; const require = createRequire(import.meta.url); -const { Connection, CreateMode } = require('./index.js'); +const { Connection, ConnectionBuilder, CreateMode } = require('./index.js'); /** * A connection pool that manages reusable Hyper database connections. @@ -42,6 +42,7 @@ export class ConnectionPool { #endpoint; #databasePath; #createMode; + #tls; #min; #max; #idleTimeoutMs; @@ -65,6 +66,8 @@ export class ConnectionPool { * @param {number} [options.idleTimeoutMs=30000] - Close idle connections after this many ms. * @param {number} [options.acquireTimeoutMs=30000] - Max ms to wait for a connection (0 = no limit). * @param {string} [options.createMode='CreateIfNotExists'] - Database creation mode. + * @param {import('./index.js').TlsOptions} [options.tls] - TLS options for every + * connection, as for `ConnectionBuilder.tls()`. Errors surface from `acquire()`. */ constructor(endpoint, databasePath, options = {}) { this.#endpoint = endpoint; @@ -74,6 +77,7 @@ export class ConnectionPool { this.#idleTimeoutMs = options.idleTimeoutMs ?? 30_000; this.#acquireTimeoutMs = options.acquireTimeoutMs ?? 30_000; this.#createMode = options.createMode ?? CreateMode.CreateIfNotExists; + this.#tls = options.tls; if (this.#idleTimeoutMs > 0) { this.#idleTimer = setInterval(() => this.#evictIdle(), this.#idleTimeoutMs / 2); @@ -141,11 +145,7 @@ export class ConnectionPool { // Create a new connection if below max if (this.size < this.#max) { - const conn = await Connection.connect( - this.#endpoint, - this.#databasePath, - this.#createMode, - ); + const conn = await this.#connect(); this.#active.add(conn); return conn; } @@ -303,6 +303,22 @@ export class ConnectionPool { await Promise.all(closePromises); } + /** + * Opens a new connection. Only a TLS pool goes through the builder: + * `Connection.connect` ignores the create mode on a read-only gRPC + * endpoint, while the builder rejects any but `DoNotCreate`. + */ + #connect() { + if (this.#tls === undefined) { + return Connection.connect(this.#endpoint, this.#databasePath, this.#createMode); + } + return new ConnectionBuilder(this.#endpoint) + .database(this.#databasePath) + .createMode(this.#createMode) + .tls(this.#tls) + .build(); + } + /** Evicts idle connections that have been unused longer than idleTimeoutMs. */ #evictIdle() { const now = Date.now(); diff --git a/hyperdb-api-node/src/connection.rs b/hyperdb-api-node/src/connection.rs index 18e2bfca..5540288e 100644 --- a/hyperdb-api-node/src/connection.rs +++ b/hyperdb-api-node/src/connection.rs @@ -224,6 +224,16 @@ impl Connection { self.inner.database().map(std::string::ToString::to_string) } + /// Returns true if the session is encrypted with TLS. + /// + /// `false` for a plaintext TCP connection, including a + /// `{ mode: 'prefer' }` connection whose server declined TLS, and for a + /// Unix domain socket or named pipe. + #[napi(getter)] + pub fn is_tls(&self) -> bool { + self.inner.is_tls() + } + /// Returns true if the connection is alive and has not been closed. #[napi(getter)] pub fn is_alive(&self) -> bool { @@ -289,6 +299,58 @@ impl Connection { } } +// ============================================================================= +// TlsOptions +// ============================================================================= + +/// TLS settings for [`ConnectionBuilder::tls`], with libpq `sslmode` +/// semantics. +#[napi(object)] +#[derive(Debug, Clone)] +pub struct TlsOptions { + /// `"disable"`, `"prefer"`, `"require"`, `"verify-ca"` or + /// `"verify-full"`. + pub mode: String, + /// PEM file of trusted root certificates. Required by `"verify-ca"` and + /// `"verify-full"`; with `"prefer"` or `"require"` it turns on chain + /// verification. + pub root_cert: Option, + /// PEM client certificate for mutual TLS. Requires `clientKey`. + pub client_cert: Option, + /// PEM private key for `clientCert`. + pub client_key: Option, + /// Host name to verify and send as SNI instead of the endpoint's host. + pub server_name: Option, +} + +impl TryFrom<&TlsOptions> for hyperdb_api::TlsConfig { + type Error = Error; + + fn try_from(options: &TlsOptions) -> Result { + let mode: hyperdb_api::TlsMode = options + .mode + .parse() + .map_err(|e: hyperdb_api::ParseTlsModeError| Error::from_reason(e.to_string()))?; + let mut tls = hyperdb_api::TlsConfig::new(mode); + if let Some(root_cert) = &options.root_cert { + tls = tls.root_cert(root_cert); + } + match (&options.client_cert, &options.client_key) { + (Some(cert), Some(key)) => tls = tls.client_cert(cert, key), + (None, None) => {} + _ => { + return Err(Error::from_reason( + "clientCert and clientKey must be set together", + )); + } + } + if let Some(server_name) = &options.server_name { + tls = tls.server_name(server_name); + } + Ok(tls) + } +} + // ============================================================================= // ConnectionBuilder // ============================================================================= @@ -303,6 +365,7 @@ pub struct ConnectionBuilder { user: Option, password: Option, login_timeout_ms: Option, + tls: Option, } #[napi] @@ -317,6 +380,7 @@ impl ConnectionBuilder { user: None, password: None, login_timeout_ms: None, + tls: None, } } @@ -355,6 +419,19 @@ impl ConnectionBuilder { self } + /// Sets the TLS options for a TCP connection; the default is + /// `{ mode: 'disable' }`. + /// + /// Over a Unix domain socket or a named pipe, `"prefer"` connects in + /// plaintext and the modes that require TLS fail `build()`. A gRPC + /// endpoint picks TLS through its `https://` scheme, so any mode but + /// `"disable"` fails there too. The options are checked by `build()`. + #[napi] + pub fn tls(&mut self, options: TlsOptions) -> &Self { + self.tls = Some(options); + self + } + /// Builds and establishes the connection. #[napi] pub async fn build(&self) -> Result { @@ -376,6 +453,9 @@ impl ConnectionBuilder { if let Some(ms) = self.login_timeout_ms { builder = builder.login_timeout(Duration::from_millis(u64::from(ms))); } + if let Some(options) = &self.tls { + builder = builder.tls(hyperdb_api::TlsConfig::try_from(options)?); + } let conn = builder .build() diff --git a/hyperdb-api-node/src/process.rs b/hyperdb-api-node/src/process.rs index 4d27bd32..f710022e 100644 --- a/hyperdb-api-node/src/process.rs +++ b/hyperdb-api-node/src/process.rs @@ -4,6 +4,7 @@ use napi::bindgen_prelude::*; use napi_derive::napi; +use std::collections::HashMap; use std::path::Path; use crate::connection::Connection; @@ -33,6 +34,44 @@ pub struct HyperProcess { inner: Option, } +/// Options for starting a [`HyperProcess`]. +#[napi(object)] +#[derive(Debug, Clone, Default)] +pub struct HyperProcessOptions { + /// `"ipc"` (the default: a Unix domain socket, or a named pipe on + /// Windows) or `"tcp"`. TLS needs `"tcp"`. + pub transport: Option, + /// `hyperd` settings, passed through unchanged, for example `ssl_key` + /// and `ssl_certificate` to serve TLS. + pub parameters: Option>, +} + +impl TryFrom<&HyperProcessOptions> for hyperdb_api::Parameters { + type Error = Error; + + fn try_from(options: &HyperProcessOptions) -> Result { + let mut params = hyperdb_api::Parameters::new(); + match options.transport.as_deref() { + None | Some("ipc") => {} + Some("tcp") => { + params.set_transport_mode(hyperdb_api::TransportMode::Tcp); + } + Some(other) => { + return Err(Error::from_reason(format!( + "unknown transport `{other}`; expected \"ipc\" or \"tcp\"" + ))); + } + } + // Sorted, so the `hyperd` command line does not depend on hash order. + let mut settings: Vec<_> = options.parameters.iter().flatten().collect(); + settings.sort(); + for (key, value) in settings { + params.set(key, value); + } + Ok(params) + } +} + #[napi] impl HyperProcess { #[allow( @@ -45,10 +84,15 @@ impl HyperProcess { /// path if it's not in the standard location. /// /// @param hyperPath - Optional path to the `hyperd` binary. + /// @param options - Optional transport and `hyperd` settings. #[napi(constructor)] - pub fn new(hyper_path: Option) -> Result { + pub fn new(hyper_path: Option, options: Option) -> Result { let path = hyper_path.as_ref().map(|p| Path::new(p.as_str())); - let process = hyperdb_api::HyperProcess::new(path, None) + let params = options + .as_ref() + .map(hyperdb_api::Parameters::try_from) + .transpose()?; + let process = hyperdb_api::HyperProcess::new(path, params.as_ref()) .map_err(|e| Error::from_reason(e.to_string()))?; Ok(HyperProcess { inner: Some(process), From 1f7562b7331a11da37a4164da3dd4e6c41965d11 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 03:45:02 -0700 Subject: [PATCH 25/61] fix(api): async without_database/connect_with_auth reach IPC endpoints Both parsed every endpoint as host:port, so a Unix socket path failed with "failed to lookup address information". They now delegate to AsyncConnectionBuilder, like their sync counterparts: a malformed port is Error::Config instead of a fallback to 7483, and without_database also reaches gRPC. The now-unused connect_tcp_with_auth is removed from both transports. New test string_constructors_connect_over_a_unix_socket fails on the old code and passes now. --- hyperdb-api/CHANGELOG.md | 7 ++++ hyperdb-api/src/async_connection.rs | 41 ++++++++++----------- hyperdb-api/src/async_transport.rs | 16 -------- hyperdb-api/src/transport.rs | 16 -------- hyperdb-api/tests/async_connection_tests.rs | 30 +++++++++++++++ 5 files changed, 56 insertions(+), 54 deletions(-) diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 0f92d61a..7a76e601 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -141,6 +141,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). connecting, so `hyperd` cancels a statement that runs longer (SQLSTATE 57014, "canceled"). A zero timeout and a timeout on a gRPC connection are rejected at `build()` instead of being ignored. +- **`AsyncConnection::without_database` and `AsyncConnection::connect_with_auth` + could not reach a Unix domain socket or named pipe.** They parsed every + endpoint as `host:port`, so a socket path failed with "failed to lookup + address information". Like their sync counterparts, they now go through + `AsyncConnectionBuilder`, which also makes a malformed port an + `Error::Config` instead of a silent fallback to port 7483, and lets + `without_database` connect to a gRPC endpoint. - **`Date`, `Timestamp` and `OffsetTimestamp` query parameters were bound with the wrong epoch.** `ToSqlParam::encode_param` sent absolute Julian-based diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index fe9746a7..ab03868e 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -97,6 +97,12 @@ impl AsyncConnection { /// Connects with authentication (async). /// + /// Equivalent to [`AsyncConnectionBuilder`](crate::AsyncConnectionBuilder) + /// with `database`, `create_mode`, `user` and `password` set, so the + /// endpoint may be TCP, a Unix domain socket or a named pipe. A gRPC + /// endpoint is accepted only with [`CreateMode::DoNotCreate`], and + /// ignores `user` and `password`. + /// /// # Errors /// /// - Returns [`Error::Authentication`] if authentication is rejected. @@ -109,37 +115,28 @@ impl AsyncConnection { user: &str, password: &str, ) -> Result { - let transport = AsyncTransport::connect_tcp_with_auth(endpoint, user, password).await?; - let conn = AsyncConnection { - transport, - database: Some(database.to_string()), - stats_provider: Mutex::new(None), - pending_stats: Mutex::new(None), - }; - - conn.handle_creation_mode(database, mode).await?; - conn.attach_and_set_path(database).await?; - - Ok(conn) + crate::AsyncConnectionBuilder::new(endpoint) + .database(database) + .create_mode(mode) + .user(user) + .password(password) + .build() + .await } /// Connects to a server without attaching any database (async). /// /// Useful for running `CREATE DATABASE` / `DROP DATABASE` without an - /// active attachment. + /// active attachment. Equivalent to + /// [`AsyncConnectionBuilder::new(endpoint).build()`](crate::AsyncConnectionBuilder), + /// so the endpoint may be TCP, gRPC, a Unix domain socket or a named pipe. /// /// # Errors /// - /// Returns [`Error::Connection`] if the TCP handshake - /// with `endpoint` fails. + /// Returns [`Error::Connection`] if the handshake with `endpoint` fails + /// or the endpoint cannot be reached. pub async fn without_database(endpoint: &str) -> Result { - let transport = AsyncTransport::connect_tcp(endpoint).await?; - Ok(AsyncConnection { - transport, - database: None, - stats_provider: Mutex::new(None), - pending_stats: Mutex::new(None), - }) + crate::AsyncConnectionBuilder::new(endpoint).build().await } /// Builds an `AsyncConnection` from a pre-existing `AsyncClient` (TCP only). diff --git a/hyperdb-api/src/async_transport.rs b/hyperdb-api/src/async_transport.rs index 978a9a66..b8f2f7a1 100644 --- a/hyperdb-api/src/async_transport.rs +++ b/hyperdb-api/src/async_transport.rs @@ -74,22 +74,6 @@ impl AsyncTransport { Ok(AsyncTransport::Tcp(AsyncTcpTransport { client })) } - /// Connect using TCP transport with authentication (async). - pub(crate) async fn connect_tcp_with_auth( - endpoint: &str, - user: &str, - password: &str, - ) -> Result { - let (host, port) = parse_endpoint(endpoint); - let config = hyperdb_api_core::client::Config::new() - .with_host(host) - .with_port(port) - .with_user(user) - .with_password(password); - let client = hyperdb_api_core::client::AsyncClient::connect(&config).await?; - Ok(AsyncTransport::Tcp(AsyncTcpTransport { client })) - } - /// Connect using gRPC transport (async). pub(crate) async fn connect_grpc( config: hyperdb_api_core::client::grpc::GrpcConfig, diff --git a/hyperdb-api/src/transport.rs b/hyperdb-api/src/transport.rs index f6c1bde3..526ea85a 100644 --- a/hyperdb-api/src/transport.rs +++ b/hyperdb-api/src/transport.rs @@ -135,22 +135,6 @@ impl Transport { Ok(Transport::Tcp(Box::new(TcpTransport { client }))) } - /// Connect using TCP transport with authentication. - pub(crate) fn connect_tcp_with_auth( - endpoint: &str, - user: &str, - password: &str, - ) -> Result { - let (host, port) = parse_endpoint(endpoint); - let config = hyperdb_api_core::client::Config::new() - .with_host(host) - .with_port(port) - .with_user(user) - .with_password(password); - let client = hyperdb_api_core::client::Client::connect(&config)?; - Ok(Transport::Tcp(Box::new(TcpTransport { client }))) - } - /// Connect using gRPC transport. pub(crate) fn connect_grpc(config: hyperdb_api_core::client::grpc::GrpcConfig) -> Result { let client = hyperdb_api_core::client::grpc::GrpcClientSync::connect(config.clone())?; diff --git a/hyperdb-api/tests/async_connection_tests.rs b/hyperdb-api/tests/async_connection_tests.rs index 37bf2737..c2bba6f6 100644 --- a/hyperdb-api/tests/async_connection_tests.rs +++ b/hyperdb-api/tests/async_connection_tests.rs @@ -628,3 +628,33 @@ async fn copy_in_hyperbinary_on_grpc_is_feature_not_supported() { "expected FeatureNotSupported, got {err:?}" ); } + +/// The string-endpoint constructors reach a Unix domain socket like +/// `connect` and the builder do, rather than parsing the path as `host:port`. +#[cfg(unix)] +#[tokio::test(flavor = "current_thread")] +async fn string_constructors_connect_over_a_unix_socket() { + use hyperdb_api::TransportMode; + + let mut params = test_hyper_params("async_string_constructors_ipc").unwrap(); + params.set_transport_mode(TransportMode::Ipc); + let hyper = HyperProcess::new(None, Some(¶ms)).unwrap(); + let endpoint = hyper.connection_endpoint_string().expect("IPC endpoint"); + assert!(endpoint.starts_with('/'), "not a socket path: {endpoint}"); + + let conn = AsyncConnection::without_database(&endpoint).await.unwrap(); + let one: i64 = conn.fetch_scalar("SELECT 1").await.unwrap(); + assert_eq!(one, 1); + + let db = test_result_path("async_string_constructors_ipc", "hyper").unwrap(); + let conn = AsyncConnection::connect_with_auth( + &endpoint, + db.to_str().expect("path"), + CreateMode::CreateAndReplace, + "tableau_internal_user", + "unused", + ) + .await + .unwrap(); + assert_eq!(conn.database(), db.to_str()); +} From 0fd87e35e46b136d263d920589a0aa1d321a01f0 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 03:45:03 -0700 Subject: [PATCH 26/61] fix(node): HyperProcess transport 'ipc' works and its endpoint is connectable `transport: 'ipc'` was a no-op and the docs called IPC the default; HyperProcess actually defaults to TCP. 'tcp' and 'ipc' now both set the mode and anything else is rejected. `hyper.endpoint` uses connection_endpoint_string(), so over IPC it is the socket path every connect call accepts rather than the listen descriptor `/domain/`. tls.mjs gains an IPC case: withoutDatabase and `prefer` connect in plaintext, `require` is rejected. README documents TLS for ConnectionBuilder, the pool and HyperProcess options. --- hyperdb-api-node/AGENTS.md | 2 +- hyperdb-api-node/CHANGELOG.md | 8 +++++--- hyperdb-api-node/README.md | 21 ++++++++++++++++++++- hyperdb-api-node/__test__/tls.mjs | 17 +++++++++++++++++ hyperdb-api-node/index.d.ts | 7 ++++--- hyperdb-api-node/src/process.rs | 30 +++++++++++++----------------- 6 files changed, 60 insertions(+), 25 deletions(-) diff --git a/hyperdb-api-node/AGENTS.md b/hyperdb-api-node/AGENTS.md index 18b3ae66..060f1fd7 100644 --- a/hyperdb-api-node/AGENTS.md +++ b/hyperdb-api-node/AGENTS.md @@ -111,7 +111,7 @@ When modifying connection behavior, check both `src/connection.rs` (Rust) and `i ## Testing - **Smoke tests:** `__test__/smoke.mjs` — covers all major features (connection, queries, inserts, streams, pool, tagged templates, BigInt, dates, JSON) -- **TLS tests:** `__test__/tls.mjs` — `ConnectionBuilder.tls()`, `ConnectionPool`'s `tls` option and `HyperProcess` options against an `ssl_force` server; certificates come from the `openssl` CLI, and the test skips without it. Its certificates and database live in a temporary directory it removes +- **TLS tests:** `__test__/tls.mjs` — `ConnectionBuilder.tls()`, `ConnectionPool`'s `tls` option and `HyperProcess` options against an `ssl_force` server, plus TLS modes over an IPC socket; certificates come from the `openssl` CLI, and the test skips without it. Its certificates and database live in a temporary directory it removes - **Benchmarks:** `__test__/benchmark.mjs` — insert and query performance with configurable row counts - Tests need `hyperd`: run `make download-hyperd` (found via `.hyperd/current`), or set an absolute `HYPERD_PATH` - Test artifacts go into `test_results/` (gitignored) diff --git a/hyperdb-api-node/CHANGELOG.md b/hyperdb-api-node/CHANGELOG.md index 23523240..eb957b66 100644 --- a/hyperdb-api-node/CHANGELOG.md +++ b/hyperdb-api-node/CHANGELOG.md @@ -24,9 +24,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * **`ConnectionPool` takes a `tls` option** and opens every connection through `ConnectionBuilder` with it. TLS failures surface from `acquire()`. * **`new HyperProcess(hyperPath, options)`** takes an optional - `{ transport: 'ipc' | 'tcp', parameters }`; `parameters` are passed to - `hyperd` unchanged, for example `ssl_key` and `ssl_certificate` to serve - TLS. An unknown transport throws. + `{ transport: 'tcp' | 'ipc', parameters }` (TCP stays the default); + `parameters` are passed to `hyperd` unchanged, for example `ssl_key` and + `ssl_certificate` to serve TLS. An unknown transport throws. With `'ipc'`, + `hyper.endpoint` is the Unix socket path or named pipe, which every + `Connection` factory and `ConnectionBuilder` accept. ## [1.0.0-rc.3] - 2026-09-07 diff --git a/hyperdb-api-node/README.md b/hyperdb-api-node/README.md index 1427d13d..53a4d25d 100644 --- a/hyperdb-api-node/README.md +++ b/hyperdb-api-node/README.md @@ -135,8 +135,13 @@ Manages a local Hyper server process (`hyperd`). ```typescript const hyper = new HyperProcess(); // auto-detect hyperd location const hyper = new HyperProcess('/path/to/hyperd'); // or specify path +// TCP, with hyperd settings passed through unchanged (here: serve TLS) +const hyper = new HyperProcess(undefined, { + transport: 'tcp', + parameters: { ssl_key: 'server.key', ssl_certificate: 'server.pem' }, +}); -hyper.endpoint; // e.g., "localhost:7483" +hyper.endpoint; // e.g., "127.0.0.1:7483", or a socket path with transport: 'ipc' hyper.isOpen; // true // Convenience: connect directly @@ -167,6 +172,7 @@ const schema = await conn.querySchema('SELECT * FROM users'); conn.database; // 'db.hyper' or null conn.isAlive; // true +conn.isTls; // true if the session is encrypted with TLS await conn.close(); // MUST be called when done ``` @@ -184,6 +190,18 @@ const conn = await new ConnectionBuilder('localhost:7483') .build(); ``` +`tls()` turns on TLS for a TCP connection, with libpq `sslmode` names: +`'disable'` (the default), `'prefer'`, `'require'`, `'verify-ca'` and +`'verify-full'`. `Connection.connect` and the other static factories stay +plaintext. + +```typescript +const conn = await new ConnectionBuilder('hyper.example.com:7483') + .database('my.hyper') + .tls({ mode: 'verify-full', rootCert: 'ca.pem' }) // also clientCert, clientKey, serverName + .build(); +``` + ### `CreateMode` Controls database creation behavior when connecting. @@ -419,6 +437,7 @@ await pool.close(); | `idleTimeoutMs` | 30000 | Close idle connections after this many ms | | `acquireTimeoutMs` | 30000 | Max ms to wait for a connection (0 = no limit) | | `createMode` | `CreateIfNotExists` | Database creation mode | +| `tls` | none | TLS options for every connection, as for `ConnectionBuilder.tls()` | ### Lossless integers and keyed rows diff --git a/hyperdb-api-node/__test__/tls.mjs b/hyperdb-api-node/__test__/tls.mjs index bc096f4f..d4fae94c 100644 --- a/hyperdb-api-node/__test__/tls.mjs +++ b/hyperdb-api-node/__test__/tls.mjs @@ -150,6 +150,23 @@ async function main() { console.log('7. An unknown HyperProcess transport is rejected...'); assert.throws(() => new HyperProcess(undefined, { transport: 'udp' }), /unknown transport `udp`/); + + if (process.platform !== 'win32') { + console.log('8. IPC: the endpoint is a socket path; prefer is plaintext, require fails...'); + const ipc = new HyperProcess(undefined, { transport: 'ipc' }); + try { + assert.match(ipc.endpoint, /^\//); + const conn = await Connection.withoutDatabase(ipc.endpoint); + assert.equal(conn.isTls, false); + await conn.close(); + const prefer = await connect(ipc.endpoint, { mode: 'prefer' }); + assert.equal(prefer.isTls, false); + await prefer.close(); + await assert.rejects(connect(ipc.endpoint, { mode: 'require' })); + } finally { + ipc.close(); + } + } } finally { hyper?.close(); rmSync(dir, { recursive: true, force: true }); diff --git a/hyperdb-api-node/index.d.ts b/hyperdb-api-node/index.d.ts index 52ad313f..caaa36aa 100644 --- a/hyperdb-api-node/index.d.ts +++ b/hyperdb-api-node/index.d.ts @@ -112,8 +112,8 @@ export interface TlsOptions { /** Options for starting a `HyperProcess`. */ export interface HyperProcessOptions { /** - * `"ipc"` (the default: a Unix domain socket, or a named pipe on - * Windows) or `"tcp"`. TLS needs `"tcp"`. + * `"tcp"` (the default) or `"ipc"`: a Unix domain socket, or a named + * pipe on Windows. TLS needs `"tcp"`. */ transport?: string /** @@ -577,7 +577,8 @@ export declare class HyperProcess { */ constructor(hyperPath?: string | undefined | null, options?: HyperProcessOptions | undefined | null) /** - * Returns the server endpoint (e.g., "localhost:7483"). + * Returns the server endpoint: `host:port` (e.g., "127.0.0.1:7483"), + * or with `transport: 'ipc'` the Unix socket path or named pipe. * * Use this to connect to the server via `Connection.connect()`. */ diff --git a/hyperdb-api-node/src/process.rs b/hyperdb-api-node/src/process.rs index f710022e..5e2183c6 100644 --- a/hyperdb-api-node/src/process.rs +++ b/hyperdb-api-node/src/process.rs @@ -38,8 +38,8 @@ pub struct HyperProcess { #[napi(object)] #[derive(Debug, Clone, Default)] pub struct HyperProcessOptions { - /// `"ipc"` (the default: a Unix domain socket, or a named pipe on - /// Windows) or `"tcp"`. TLS needs `"tcp"`. + /// `"tcp"` (the default) or `"ipc"`: a Unix domain socket, or a named + /// pipe on Windows. TLS needs `"tcp"`. pub transport: Option, /// `hyperd` settings, passed through unchanged, for example `ssl_key` /// and `ssl_certificate` to serve TLS. @@ -52,13 +52,16 @@ impl TryFrom<&HyperProcessOptions> for hyperdb_api::Parameters { fn try_from(options: &HyperProcessOptions) -> Result { let mut params = hyperdb_api::Parameters::new(); match options.transport.as_deref() { - None | Some("ipc") => {} + None => {} Some("tcp") => { params.set_transport_mode(hyperdb_api::TransportMode::Tcp); } + Some("ipc") => { + params.set_transport_mode(hyperdb_api::TransportMode::Ipc); + } Some(other) => { return Err(Error::from_reason(format!( - "unknown transport `{other}`; expected \"ipc\" or \"tcp\"" + "unknown transport `{other}`; expected \"tcp\" or \"ipc\"" ))); } } @@ -99,7 +102,8 @@ impl HyperProcess { }) } - /// Returns the server endpoint (e.g., "localhost:7483"). + /// Returns the server endpoint: `host:port` (e.g., "127.0.0.1:7483"), + /// or with `transport: 'ipc'` the Unix socket path or named pipe. /// /// Use this to connect to the server via `Connection.connect()`. #[napi(getter)] @@ -108,9 +112,10 @@ impl HyperProcess { .inner .as_ref() .ok_or_else(|| Error::from_reason("HyperProcess is closed"))?; + // Not `process.endpoint()`: over IPC that is the listen descriptor's + // `/domain/`, which no connect call accepts. process - .endpoint() - .map(std::string::ToString::to_string) + .connection_endpoint_string() .ok_or_else(|| Error::from_reason("No endpoint available")) } @@ -124,16 +129,7 @@ impl HyperProcess { database_path: String, create_mode: CreateMode, ) -> Result { - let process = self - .inner - .as_ref() - .ok_or_else(|| Error::from_reason("HyperProcess is closed"))?; - let endpoint = process - .endpoint() - .map(std::string::ToString::to_string) - .ok_or_else(|| Error::from_reason("No endpoint available"))?; - - Connection::connect(endpoint, database_path, create_mode).await + Connection::connect(self.endpoint()?, database_path, create_mode).await } #[allow( From e9447410223bea2d22cd4b34f775a4df8cef0cd5 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 03:45:03 -0700 Subject: [PATCH 27/61] docs: TLS for TCP connections and pools README files describe the libpq sslmode support, the hyperdb-api README gains a TLS section, and DEVELOPMENT-client.md's TLS internals and tests match the implementation. The hyperdb-mcp attach comment records where a future TCP attach would take its TlsConfig. --- README.md | 2 +- hyperdb-api-core/docs/DEVELOPMENT-client.md | 37 +++++++++++++-------- hyperdb-api/README.md | 22 ++++++++++++ hyperdb-mcp/src/attach.rs | 3 +- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2c87e8cc..7498ba9c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ executable, obtained with `make download-hyperd`. See - **gRPC Transport** — read-only access with Arrow IPC and load balancing support - **Full Type Support** — all Hyper types including Numeric, Geography, Intervals - **Salesforce Auth** — OAuth 2.0 and JWT Bearer Token flows for Data Cloud -- **TLS** — via rustls (always-on, pure Rust) +- **TLS** — for TCP connections and pools, with libpq `sslmode` semantics, via rustls (always-on, pure Rust) - **Formal Verification** — Kani proof harnesses for model-checked correctness ## Quick Start diff --git a/hyperdb-api-core/docs/DEVELOPMENT-client.md b/hyperdb-api-core/docs/DEVELOPMENT-client.md index 8438aff6..76e48286 100644 --- a/hyperdb-api-core/docs/DEVELOPMENT-client.md +++ b/hyperdb-api-core/docs/DEVELOPMENT-client.md @@ -27,7 +27,7 @@ src/ connection.rs # RawConnection -- sync wire protocol engine async_connection.rs # AsyncRawConnection -- async wire protocol engine auth.rs # Authentication: cleartext, MD5, SCRAM-SHA-256 - tls.rs # TLS config and rustls integration + tls.rs # TlsConfig/TlsMode, SSLRequest negotiation, verifiers, TLS cancel cancel.rs # Cancellable trait (transport-agnostic cancel) endpoint.rs # ConnectionEndpoint (TCP, Unix, Named Pipe) sync_stream.rs # SyncStream (TCP/Unix/Pipe wrapper for sync I/O) @@ -52,7 +52,6 @@ tests/ client_tests.rs # Sync client integration tests copy_tests.rs # COPY protocol tests prepared_statement_tests.rs - tls_tests.rs # TLS with self-signed certs (rcgen) async_copy_cancel_tests.rs ``` @@ -178,13 +177,21 @@ handshake are documented in `auth.rs` module-level rustdoc. ## TLS Internals -TLS is implemented via `rustls` (pure Rust, no OpenSSL dependency). The -implementation lives in `tls.rs` and `tls::rustls_impl`. - -- Root certificates: system roots via `webpki-roots` + optional custom CA -- Client certificates: optional mutual TLS (mTLS) -- Key formats: PEM (PKCS#8 or PKCS#1 for private keys) -- TLS modes: `Disable`, `Prefer`, `Require`, `VerifyCA`, `VerifyFull` +TLS for PG-wire TCP connections uses `rustls` (pure Rust, no OpenSSL) and +lives in `tls.rs`. `Config::with_tls` takes a `TlsConfig`; `hyperdb-api`'s +connection builders and pools pass theirs through it. The rustdoc on +`client::tls` is the reference for the details: + +- `TlsMode` follows libpq's `sslmode` names, with sqlx's choices where they + differ: `Disable` is the default, and `Prefer` falls back to plaintext only + when the server declines the `SSLRequest`, never after a failed handshake + or certificate check. +- A configured root certificate is the only trust anchor. `VerifyFull` + without one trusts the bundled `webpki-roots`; the OS store and + `~/.postgresql/` are never read. +- A cancel request for a TLS session goes over TLS, never plaintext. +- Session resumption is disabled: `hyperd` aborts a resumed handshake with + `internal_error`. gRPC TLS is handled separately by `tonic`'s built-in TLS support, configured via `GrpcConfig` (auto-detected from `https://` endpoints). @@ -232,9 +239,13 @@ cargo test -p hyperdb-api-core --test client_tests ### TLS Tests -`tls_tests.rs` generates self-signed certificates at runtime using `rcgen` -(dev-dependency) to test TLS handshake, mTLS, and certificate verification -without requiring pre-generated certificates. +The unit tests in `tls.rs` run the client against a fake PG-wire server on a +local thread, which answers the `SSLRequest` and runs a rustls server. They +cover the verifiers, each `SSLRequest` answer, and cancel over TLS, which +`hyperd` cannot prove because it accepts a plaintext cancel too. The end-to-end tests against a real `hyperd` +started with `ssl_key` / `ssl_certificate` are in +`hyperdb-api/tests/tls_tests.rs`. Both generate their certificates at runtime +with `rcgen`. ### Writing New Tests @@ -298,8 +309,6 @@ See `cancel.rs` for the full rationale. - Connection pooling is not part of this crate. It lives in `hyperdb-api`'s `pool` module, which wraps `deadpool` behind its own `Pool` and `PooledConnection` types -- `TlsConfig` / `TlsMode` are defined but not yet wired into `Config`'s - builder (TLS is configured at a lower level currently) --- diff --git a/hyperdb-api/README.md b/hyperdb-api/README.md index f234a816..a300cfe1 100644 --- a/hyperdb-api/README.md +++ b/hyperdb-api/README.md @@ -475,6 +475,28 @@ ASCII `A-Z a-z 0-9 _ . -`; anything else returns `Error::InvalidName`. ## Connection Features +### TLS + +TCP connections use TLS through the builders' `tls()`, with libpq `sslmode` +names: `Disable` (the default), `Prefer`, `Require`, `VerifyCa` and +`VerifyFull`. `PoolConfig` and `SyncPoolConfig` take the same setting; the +`Connection::new` / `connect*` shortcuts stay plaintext. + +```rust +use hyperdb_api::{ConnectionBuilder, TlsConfig, TlsMode}; + +let conn = ConnectionBuilder::new("hyper.example.com:7483") + .database("data.hyper") + .tls(TlsConfig::new(TlsMode::VerifyFull).root_cert("ca.pem")) + .build()?; +assert!(conn.is_tls()); +``` + +A `hyperd` serves TLS when started with the `ssl_key` and `ssl_certificate` +settings (`Parameters::set`) over the TCP transport. gRPC picks TLS from its +`https://` scheme instead. See `TlsMode` for how each mode treats a server +without TLS and which certificate checks it runs. + ### Query Cancellation Thread-safe cancellation from another thread: diff --git a/hyperdb-mcp/src/attach.rs b/hyperdb-mcp/src/attach.rs index 71d19135..f119d3bd 100644 --- a/hyperdb-mcp/src/attach.rs +++ b/hyperdb-mcp/src/attach.rs @@ -134,7 +134,8 @@ pub enum AttachSource { /// Canonical absolute path to the `.hyper` file. path: PathBuf, }, - // Future: Tcp { endpoint: String, auth: Option }, + // Future: Tcp { endpoint: String, auth: Option, tls: TlsConfig }, + // (`hyperdb_api::TlsConfig`, passed to the builder's `tls()`) // Future: Grpc { endpoint: String, auth: Option }, // writable always false } From ae273feba854a69b7ce5f375b4c6f8e604d2d605 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 04:05:11 -0700 Subject: [PATCH 28/61] fix(core,node): TLS final-review follow-ups - Core CHANGELOG lists the breaking reshape of the old client::tls surface, the new client::Error::Tls variant and is_tcp() on TLS streams. - `allow` is documented as unsupported, and parsing it hints at `prefer`. - The connection-string TLS-key error names Config::with_tls. - The builders' tls() docs say the TLS handshake is bounded by login_timeout, not the TCP connect. - Client/AsyncClient Debug redacts the cancel secret_key. - ConnectionPool treats `tls: null` as no TLS; it used to fail every acquire() in the napi conversion (red-checked). --- hyperdb-api-core/CHANGELOG.md | 16 ++++++++++++++++ hyperdb-api-core/src/client/async_client.rs | 2 +- hyperdb-api-core/src/client/client.rs | 2 +- hyperdb-api-core/src/client/config.rs | 7 ++----- hyperdb-api-core/src/client/tls.rs | 12 +++++++++++- hyperdb-api-node/__test__/tls.mjs | 8 ++++++++ hyperdb-api-node/pool.mjs | 2 +- hyperdb-api/src/async_connection_builder.rs | 4 ++-- hyperdb-api/src/connection_builder.rs | 4 ++-- 9 files changed, 44 insertions(+), 13 deletions(-) diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index a54d1246..53b4f2ca 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -27,9 +27,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **`SyncStream::try_clone`.** A rustls stream cannot be cloned, and nothing called it. +- **The old, never-wired `client::tls` surface:** `TlsConfig`'s public fields + (`verify_server`, `ca_cert_path`, `client_cert_path`, `client_key_path`, + `server_name`), `TlsConfig::danger_accept_invalid_certs`, the `TlsMode` + helpers `is_enabled` / `is_required` / `verify_server` / `verify_hostname`, + and the `rustls_impl` module (`create_connector`, `TlsStream`, + `wrap_stream`). ### Changed +- **`client::tls` is reshaped around libpq `sslmode`** (BREAKING). + `TlsConfig::new` takes the `TlsMode`; `ca_cert` is now `root_cert`; + `TlsMode::VerifyCA` is now `TlsMode::VerifyCa`; `TlsConfig`'s fields are + private (`TlsConfig::mode` reads the mode back). +- **`client::Error` has a new `Tls` variant** (BREAKING for exhaustive + matches; the enum is not `#[non_exhaustive]`), built with `Error::tls`. + Handshake and certificate failures, including a `rustls::Error` surfacing + through I/O, map to it. +- **`SyncStream` / `AsyncStream` gained a `Tls` variant, and `is_tcp()` is + `true` for it** (a TLS stream runs over TCP); `is_tls()` tells them apart. - **`Config`'s `FromStr` error type is `client::Error`** (was `String`). The libpq TLS keys (`sslmode`, `sslrootcert`, `sslcert`, `sslkey`, ...) are rejected with `Error::Config` instead of being forwarded to the server as diff --git a/hyperdb-api-core/src/client/async_client.rs b/hyperdb-api-core/src/client/async_client.rs index 83625c10..be2d4422 100644 --- a/hyperdb-api-core/src/client/async_client.rs +++ b/hyperdb-api-core/src/client/async_client.rs @@ -72,7 +72,7 @@ impl std::fmt::Debug for AsyncClient { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("AsyncClient") .field("process_id", &self.process_id) - .field("secret_key", &self.secret_key) + .field("secret_key", &"") .field("endpoint", &self.endpoint) .field( "notice_receiver", diff --git a/hyperdb-api-core/src/client/client.rs b/hyperdb-api-core/src/client/client.rs index 20237299..02f7e6f0 100644 --- a/hyperdb-api-core/src/client/client.rs +++ b/hyperdb-api-core/src/client/client.rs @@ -130,7 +130,7 @@ impl std::fmt::Debug for Client { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Client") .field("process_id", &self.process_id) - .field("secret_key", &self.secret_key) + .field("secret_key", &"") .field("endpoint", &self.endpoint) .field( "notice_receiver", diff --git a/hyperdb-api-core/src/client/config.rs b/hyperdb-api-core/src/client/config.rs index e8664d40..98d46624 100644 --- a/hyperdb-api-core/src/client/config.rs +++ b/hyperdb-api-core/src/client/config.rs @@ -264,7 +264,7 @@ impl std::str::FromStr for Config { if LIBPQ_TLS_KEYS.contains(&name) { return Err(Error::config(format!( "TLS options are not accepted in the connection string (found `{name}`); \ - use ConnectionBuilder::tls" + use Config::with_tls (ConnectionBuilder::tls in hyperdb-api)" ))); } if let Some(idx) = param.find('=') { @@ -308,10 +308,7 @@ mod tests { .parse::() .expect_err("TLS key must be rejected"); assert!(matches!(err, Error::Config(_)), "{key}: got {err:?}"); - assert!( - err.to_string().contains("ConnectionBuilder::tls"), - "got {err}" - ); + assert!(err.to_string().contains("Config::with_tls"), "got {err}"); } let err = "h:1/db?sslmode=require".parse::().unwrap_err(); assert!(err.to_string().contains("sslmode"), "got {err}"); diff --git a/hyperdb-api-core/src/client/tls.rs b/hyperdb-api-core/src/client/tls.rs index e7d30391..bd3f1135 100644 --- a/hyperdb-api-core/src/client/tls.rs +++ b/hyperdb-api-core/src/client/tls.rs @@ -84,6 +84,8 @@ use crate::protocol::message::frontend; /// /// - The default is `Disable`, not libpq's `prefer`: `hyperd` is usually a /// local engine with TLS off. +/// - libpq's `allow` (plaintext first, TLS only if the server insists) is not +/// offered; use `Prefer`. /// - `Prefer` falls back to plaintext only when the server answers that it /// has no TLS. A failed handshake or certificate check is an error, never a /// downgrade. @@ -163,7 +165,11 @@ impl fmt::Display for ParseTlsModeError { f, "invalid TLS mode `{}`; expected one of: disable, prefer, require, verify-ca, verify-full", self.input - ) + )?; + if self.input == "allow" { + f.write_str(" (libpq's `allow` is not supported; use `prefer`)")?; + } + Ok(()) } } @@ -1554,6 +1560,10 @@ mod tests { for name in ["disable", "prefer", "require", "verify-ca", "verify-full"] { assert!(message.contains(name), "{name} missing from {message}"); } + assert!(!message.contains("allow"), "got {message}"); + + let message = "allow".parse::().unwrap_err().to_string(); + assert!(message.contains("use `prefer`"), "got {message}"); } #[test] diff --git a/hyperdb-api-node/__test__/tls.mjs b/hyperdb-api-node/__test__/tls.mjs index d4fae94c..62fbf0c7 100644 --- a/hyperdb-api-node/__test__/tls.mjs +++ b/hyperdb-api-node/__test__/tls.mjs @@ -146,6 +146,14 @@ async function main() { }); await assert.rejects(plaintext.acquire(), /SSL/); await plaintext.close(); + + // `tls: null` means no TLS, like leaving it out. + const nullTls = new ConnectionPool(endpoint, join(dir, 'pool.hyper'), { + createMode: CreateMode.DoNotCreate, + tls: null, + }); + await assert.rejects(nullTls.acquire(), /SSL/); + await nullTls.close(); } console.log('7. An unknown HyperProcess transport is rejected...'); diff --git a/hyperdb-api-node/pool.mjs b/hyperdb-api-node/pool.mjs index a4359178..f7345e05 100644 --- a/hyperdb-api-node/pool.mjs +++ b/hyperdb-api-node/pool.mjs @@ -309,7 +309,7 @@ export class ConnectionPool { * endpoint, while the builder rejects any but `DoNotCreate`. */ #connect() { - if (this.#tls === undefined) { + if (this.#tls == null) { return Connection.connect(this.#endpoint, this.#databasePath, this.#createMode); } return new ConnectionBuilder(this.#endpoint) diff --git a/hyperdb-api/src/async_connection_builder.rs b/hyperdb-api/src/async_connection_builder.rs index 45b31940..32522e46 100644 --- a/hyperdb-api/src/async_connection_builder.rs +++ b/hyperdb-api/src/async_connection_builder.rs @@ -196,9 +196,9 @@ impl AsyncConnectionBuilder { /// endpoint picks TLS through its `https://` scheme, so any mode other /// than [`TlsMode::Disable`] fails there with /// [`Error::FeatureNotSupported`] too. A query cancel for a TLS session - /// is sent over TLS. TLS negotiation counts against + /// is sent over TLS. The TLS handshake must finish within /// [`login_timeout`](Self::login_timeout) (30 seconds by default; zero - /// means no limit). + /// means no limit); the TCP connect itself is not bounded by it. /// /// The `AsyncConnection::connect*` shortcuts and `AsyncConnection::new` always /// connect in plaintext; use this builder for TLS. diff --git a/hyperdb-api/src/connection_builder.rs b/hyperdb-api/src/connection_builder.rs index 2a00c733..4c265029 100644 --- a/hyperdb-api/src/connection_builder.rs +++ b/hyperdb-api/src/connection_builder.rs @@ -231,9 +231,9 @@ impl ConnectionBuilder { /// endpoint picks TLS through its `https://` scheme, so any mode other /// than [`TlsMode::Disable`] fails there with /// [`Error::FeatureNotSupported`] too. A query cancel for a TLS session - /// is sent over TLS. TLS negotiation counts against + /// is sent over TLS. The TLS handshake must finish within /// [`login_timeout`](Self::login_timeout) (30 seconds by default; zero - /// means no limit). + /// means no limit); the TCP connect itself is not bounded by it. /// /// The `Connection::connect*` shortcuts and `Connection::new` always /// connect in plaintext; use this builder for TLS. From 37dfd45d5fe28dc7c3f35126d7d8e35c10f5052b Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 11:44:47 -0700 Subject: [PATCH 29/61] fix(core): drop an IPv6 zone id from the TLS server name A link-local host such as `[fe80::1%1]` connects in plaintext, but every TLS mode failed before connecting with "invalid TLS server name", because rustls cannot parse the `%zone` suffix. The zone names the local interface, not the server, and an IP SAN cannot hold one, so the derived server name is now the bare address: prefer/require/verify-ca connect, and verify-full checks the IP SAN `fe80::1`. An explicit server_name and a non-IPv6 host are unchanged. Both new unit tests fail with the zone kept and pass with it dropped. --- hyperdb-api-core/src/client/tls.rs | 41 +++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/hyperdb-api-core/src/client/tls.rs b/hyperdb-api-core/src/client/tls.rs index bd3f1135..bb0aa3ce 100644 --- a/hyperdb-api-core/src/client/tls.rs +++ b/hyperdb-api-core/src/client/tls.rs @@ -45,7 +45,7 @@ use std::fmt; use std::io::{self, Read, Write}; -use std::net::{Shutdown, TcpStream}; +use std::net::{Ipv6Addr, Shutdown, TcpStream}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; @@ -325,9 +325,17 @@ impl TlsConnector { let name = tls.server_name.as_deref().unwrap_or_else(|| { // An IPv6 host arrives as `[::1]`; the brackets are URL syntax. - host.strip_prefix('[') + let host = host + .strip_prefix('[') .and_then(|inner| inner.strip_suffix(']')) - .unwrap_or(host) + .unwrap_or(host); + // A link-local literal can carry a zone (`fe80::1%1`). The zone + // picks the local interface, not the server, and an IP SAN cannot + // hold one, so the name is the bare address. + match host.split_once('%') { + Some((address, _zone)) if address.parse::().is_ok() => address, + _ => host, + } }); let server_name = ServerName::try_from(name) .map_err(|e| Error::config(format!("invalid TLS server name `{name}`: {e}")))? @@ -1206,6 +1214,33 @@ mod tests { assert!(matches!(err, Error::Config(_)), "got {err:?}"); } + #[test] + fn ipv6_zone_is_not_part_of_the_server_name() { + let tls = TlsConfig::from(TlsMode::Require); + for host in ["[fe80::1%1]", "[fe80::1%en0]", "fe80::1%1"] { + let built = connector(&tls, host); + let expected = ServerName::from("fe80::1".parse::().unwrap()); + assert_eq!(built.server_name, expected, "{host}"); + } + // Only an IPv6 address loses its `%` suffix. + let err = TlsConnector::build(&tls, "host%1").unwrap_err(); + assert!(matches!(err, Error::Config(_)), "got {err:?}"); + } + + #[test] + fn verify_full_checks_zoned_ipv6_host_against_ip_san() { + let ca = generate_ca("Test CA"); + let server = generate_leaf(&ca, &["fe80::1"]); + let ca_file = pem_file(&ca.cert_pem); + let tls = TlsConfig::new(TlsMode::VerifyFull).root_cert(ca_file.path()); + let (outcome, seen) = handshake(&server, &tls, "[fe80::1%1]"); + assert!(outcome.unwrap()); + assert_payload_received(&seen); + + let (outcome, _) = handshake(&server, &tls, "[fe80::2%1]"); + assert!(matches!(outcome, Err(Error::Tls(_))), "got {outcome:?}"); + } + #[test] fn disable_builds_no_connector() { assert!( From b8d85904e5e6640bf68cbc6f40cda84ba18dc40c Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 13:43:46 -0700 Subject: [PATCH 30/61] fix(mcp): read-only SQL checks follow Hyper's grammar and cover every SQL tool query_data, query_file and export ran any statement, so `query_data(sql: "DROP TABLE persistent.public.t")` dropped a persistent table even under --read-only. They now refuse non-read-only SQL with SQL_ERROR, as query does; query_data/query_file also check the SQL after the table-name substitution. The classifier moves to sql_classify.rs and tokenizes like Hyper's lexer (non-nesting block comments, E-strings, string continuation, dollar-quote tags, Unicode whitespace). WITH and EXPLAIN take the kind of the statement they run, so `WITH ... DELETE` and `EXPLAIN (ANALYZE) DELETE` are no longer read-only; SELECT ... INTO is a write; more than one statement, unbalanced or over-deep parentheses are refused. export checks its SQL, has Hyper parse it alone, and puts it on its own lines inside COPY (...) TO, so it can neither pick its own target nor be broken by a trailing -- comment. --- .../plans/2026-10-10-mcp-security.md | 119 ++++ .../specs/2026-10-10-mcp-security-design.md | 450 +++++++++++++++ hyperdb-mcp/CHANGELOG.md | 18 + hyperdb-mcp/README.md | 8 +- hyperdb-mcp/src/engine.rs | 212 ++----- hyperdb-mcp/src/export.rs | 23 +- hyperdb-mcp/src/lib.rs | 1 + hyperdb-mcp/src/readme.rs | 6 +- hyperdb-mcp/src/server.rs | 41 +- hyperdb-mcp/src/sql_classify.rs | 546 ++++++++++++++++++ hyperdb-mcp/tests/end_to_end_mcp_tests.rs | 125 +++- hyperdb-mcp/tests/export_tests.rs | 83 +++ hyperdb-mcp/tests/read_only_tests.rs | 136 ++++- 13 files changed, 1555 insertions(+), 213 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-10-mcp-security.md create mode 100644 docs/superpowers/specs/2026-10-10-mcp-security-design.md create mode 100644 hyperdb-mcp/src/sql_classify.rs diff --git a/docs/superpowers/plans/2026-10-10-mcp-security.md b/docs/superpowers/plans/2026-10-10-mcp-security.md new file mode 100644 index 00000000..d7f6416f --- /dev/null +++ b/docs/superpowers/plans/2026-10-10-mcp-security.md @@ -0,0 +1,119 @@ +# hyperdb-mcp security fixes Implementation Plan + +**Goal:** Close the four decision-9 findings in `hyperdb-mcp`: the `--read-only` bypass, the destructive export default, the unauthenticated loopback health protocol, and the takeover race. + +**Spec:** [`docs/superpowers/specs/2026-10-10-mcp-security-design.md`](../specs/2026-10-10-mcp-security-design.md). It holds the design, the platform details, and the measured facts (Hyper's SQL behaviour, the file magic, the race timeline). Read it first and do not re-derive any of it. + +**Tech stack:** std `UnixListener` / `UnixStream`, `libc` (`poll`, `flock`; already a Unix dependency), `windows-sys` 0.61 (new direct Windows dependency; 0.61.2 is already in `Cargo.lock`), `tokio::sync::Notify` (add tokio's `sync` feature to `hyperdb-mcp`). + +## Global constraints + +- **Engine.** Pinned hyperd only, `HYPERD_PATH="$PWD/.hyperd/current"` absolute. No invented hyperd flags. +- **Code rules.** No narrowing integer `as` casts (AGENTS.md reminder 7); `libc` pid/fd conversions go through `TryFrom`. Library code returns errors. Lifetimes get descriptive names. Every `unsafe` block has a `// SAFETY:` comment. +- **Commits.** Explicit `git add `; never stage `.agents/` or `.codex/`. Conventional Commits, `!` on breaking iterations. No push. +- **MSRV 1.88.** `cargo +1.88 check --workspace --all-targets` stays clean (so no `File::try_lock`). +- **Windows.** No local Windows C toolchain: `cargo check -p hyperdb-mcp --target x86_64-pc-windows-msvc` fails in `ring`'s build script (measured: `cc` exit 1, cargo exit 101). So the modules with Windows code, `daemon/control.rs`, `daemon/lock.rs` and `src/file_identity.rs`, depend only on `std`, `libc` and `windows-sys` (no `crate::` imports), and each iteration that touches them compiles them for Windows in a scratch crate (`/tmp/mcpsec/wincheck`, `#[path = "..."] mod control;` and so on, `windows-sys` with the same features) with `cargo clippy --target x86_64-pc-windows-msvc -- -D warnings`. The `x86_64-pc-windows-msvc` target is installed (checked). Windows-only code elsewhere stays minimal and `cfg`-gated. CI's Windows job is the real check, and the user pushes. +- **Gate for every iteration:** + - `cargo fmt --all --check` + - `cargo clippy --workspace --all-targets --all-features -- -D warnings` + - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace` + - `cargo +1.88 check --workspace --all-targets` + - `cargo test -p hyperdb-mcp` (whole crate; real output, exit code checked via `>log 2>&1; echo $?`) + - `npx markdownlint-cli2` on touched Markdown, judged against `git show HEAD:` +- **Doc contracts.** Each iteration updates the docs its change makes wrong *and* the doc-contract tests that pin them, in the same commit; never weaken a test to pass. `hyperdb-mcp/src/readme.rs` must stay under 24 000 bytes (`readme_tests.rs` `readme_is_non_trivial`; headroom is about 240 bytes, so trim when adding). The tool catalog must stay under `CATALOG_BYTE_BUDGET` (57 344, `tool_schema_tests.rs`). The read-only and writable catalogs must stay identical. Known pins: the public README's blocked-tools slice (`**blocked:**` up to `- **resources`) must not contain `export` (`readme_tests.rs` ~611); the get_readme rules keep `export`/`hyper` "remain allowed" wording (~617-635); get_readme chart guidance keeps the word `overwrite` (~672); the CLI reference and `daemon --help` pins (`readme_tests.rs` ~740-747, `doctor_tests.rs` ~2268-2302). +- **Changelog.** `hyperdb-mcp/CHANGELOG.md` `## [Unreleased]`: each iteration adds its own bullets, merged into the existing `### Changed` / `### Fixed` / `### Removed` headings (MD024). Do not rewrite the historic entries that carry "health port" (~215, ~473, ~725). + +## Iteration 1: read-only SQL + +**Files:** `hyperdb-mcp/src/engine.rs` (tokenizer, `classify_statement`, `strip_leading_sql_comments`, doc comments; a `check_embeddable_query` helper), `hyperdb-mcp/src/server.rs`, `hyperdb-mcp/src/export.rs`, `hyperdb-mcp/tests/read_only_tests.rs`, `hyperdb-mcp/tests/end_to_end_mcp_tests.rs`, `hyperdb-mcp/tests/export_tests.rs`, `hyperdb-mcp/src/readme.rs`, `hyperdb-mcp/README.md`, `hyperdb-mcp/src/error.rs` (only if the `READ_ONLY_VIOLATION` suggestion implies those tools accept writes), `hyperdb-mcp/CHANGELOG.md`. + +1. Tokenizer and classifier per spec section 1, design point 1. `strip_leading_sql_comments` stops nesting. Unit tests (in `read_only_tests.rs`): every row of the spec's measured table that the classifier decides; `WITH ... DELETE`, `WITH ... INSERT`, `EXPLAIN (ANALYZE) DELETE`, `EXPLAIN (ANALYZE) WITH ... DELETE` not read-only; `EXPLAIN SELECT`, `EXPLAIN DELETE` (no analyze), `WITH x AS (...) (SELECT ...)`, `TABLE t`, dollar-quoted and `E'...'` literals holding `)`, `;` or `DELETE` read-only; a non-nesting comment that hides a statement from a nesting stripper is not read-only; an unterminated literal or comment, an unbalanced `)`, and a token after a depth-0 `;` are `Other`; the `execute` batch validator's view of `EXPLAIN (ANALYZE) DELETE` (DML). +2. `query_data` and `query_file`: before any ingest, reject `!is_read_only_sql(¶ms.sql)` with `ErrorCode::SqlError` and the `query` wording, naming the tool. `export` in SQL mode: same check, then `check_embeddable_query` (spec point 3: `prepare` alone, dropped), then the two `COPY` sites splice `COPY (\n{sql}\n) TO ...`. Classify the original SQL, not the `replace_identifier` output. +3. End-to-end tests (`TestHarness::start(read_only, ...)`), red-before-green for each new guard: + - `query_data`, `query_file`, `export`: a `DROP TABLE persistent.public.t` and a `DELETE` are rejected and the table survives, with and without `--read-only`; a plain `SELECT` still works under `--read-only` for each; + - `query`: `WITH ... DELETE` and `EXPLAIN (ANALYZE) DELETE` are rejected and the rows survive (red on today's classifier); + - `export` (csv and iceberg): SQL that closes the `COPY` parenthesis and names its own target is rejected; the target named inside the SQL does not exist afterwards, and a persistent DB named there is unchanged; + - `export`: a query ending in a `--` comment still exports. +4. Docs: README "Read-Only Mode" (the SQL-level rule now names `query`, `query_data`, `query_file`, `export` and `chart`, written **outside** the blocked-tools slice), readme.rs `query_data` / `query_file` / export lines (keep within budget). CHANGELOG `### Fixed`: the four tools' read-only check, the `WITH` / `EXPLAIN (ANALYZE)` / comment classifier holes, the export wrapper escape. + +Commit: `fix(mcp): read-only SQL checks follow Hyper's grammar and cover every SQL tool`. + +## Iteration 2: export and chart never destroy what they did not write + +**Files:** `hyperdb-mcp/src/server.rs` (`ExportParams`, `ChartParams` docs and defaults, export and chart handlers, protected-set builder), `hyperdb-mcp/src/export.rs`, `hyperdb-mcp/src/chart.rs`, `hyperdb-mcp/src/attach.rs` (stale `validate_output_path` doc), new `hyperdb-mcp/src/file_identity.rs`, `hyperdb-mcp/src/lib.rs`, `hyperdb-mcp/Cargo.toml` (`windows-sys` with `Win32_Foundation`, `Win32_Storage_FileSystem` for now), `Cargo.lock`, `hyperdb-mcp/tests/export_tests.rs`, `hyperdb-mcp/tests/end_to_end_mcp_tests.rs` (and any chart test that wrote onto an existing path relying on the old `true` default), readme.rs, README.md, CHANGELOG.md. + +1. First, measure the top-level layout of a real Iceberg export (one-off, from an existing test's output) and pin it as the allowed set. +2. Default `overwrite` to `false` for `export` (server.rs ~3311) and `chart`; rewrite both param docs ("Defaults to false: an existing destination is refused with PERMISSION_DENIED; pass true to replace it"). +3. Use `validate_output_path`'s returned path for the checks and the write (spec section 2, point 2). +4. `file_identity.rs`: `same_file(a, b) -> io::Result` (Unix `MetadataExt::{dev, ino}`; Windows `GetFileInformationByHandle` volume serial + file index). The server builds the protected set (persistent, ephemeral, each `AttachSource::LocalFile`) and passes it to `export_to_file` (`ExportOptions::protected_paths`) and to the chart write path. An existing destination that is the same file as a protected one is `INVALID_ARGUMENT`, for every format and for chart, before any other check. +5. Iceberg: replace the `remove_dir_all` block with the shape check (empty, or top-level entries within the measured set with a `metadata/*.metadata.json`) and `ReplaceGuard` (rename to `.hyperdb-mcp-old--`; `commit()` deletes the backup; `Drop` without commit removes what is now at `dest`, renames the backup back, and reports the backup path if that fails). +6. Hyper: refuse unless `symlink_metadata` is a regular file starting with `b"Hyper"`; create the database at `.hyperdb-mcp-tmp--`, fill and detach it, rename it over `path`; on failure remove the temp file. Keep the `is_file_in_use` → `RESOURCE_BUSY` mapping on the rename. +7. Tests, red-before-green for the non-Iceberg refusal and the protected-file refusal: + - iceberg over a non-Iceberg directory (with a sentinel file), and over a directory holding a `.hyper` file, is refused with `overwrite=true`, and the contents survive; + - iceberg over an existing Iceberg export still replaces it (`iceberg_export_overwrite_replaces_directory` keeps passing); + - iceberg and hyper exports that fail after creating output (a bad `source_db`, which fails after create and attach, export.rs ~538; a SQL error for iceberg) leave the previous destination byte-identical and no temp or backup behind; + - hyper over a non-Hyper regular file is refused and the file is untouched; over a real `.hyper` it replaces; + - export (csv and hyper) and chart to the persistent DB, to an attached file, and to a symlink pointing at the persistent DB are refused even with `overwrite=true`; + - MCP level: `export` and `chart` without `overwrite` onto an existing file are `PERMISSION_DENIED`. +8. Docs: README export parameter table gains `overwrite` (and the missing `format_options`), the chart row default `false`, the `PERMISSION_DENIED` row, the leftover-backup note; readme.rs chart/export lines (keep `overwrite` in the chart guidance). CHANGELOG `### Changed` (both default flips, BREAKING) and `### Fixed` (destructive replace, protected files). Mind the `markdown_bullet(changelog, "hyper-format export")` contract if a new bullet uses that phrase. +9. Windows scratch-crate clippy for `file_identity.rs`. + +Commit: `fix(mcp)!: export and chart refuse to overwrite by default and never replace what they did not write`. + +## Iteration 3: control transport and lock modules (not wired yet) + +**Files:** new `hyperdb-mcp/src/daemon/control.rs`, new `hyperdb-mcp/src/daemon/lock.rs`, `hyperdb-mcp/src/daemon/mod.rs`, `hyperdb-mcp/Cargo.toml` (`windows-sys` features grow to `Win32_Foundation`, `Win32_Security`, `Win32_Security_Authorization`, `Win32_System_Pipes`, `Win32_Storage_FileSystem`, `Win32_System_Threading`, `Win32_System_IO`, `Win32_System_Memory`; tokio `sync`), `Cargo.lock`. + +1. `control.rs` per spec section 3, Transport: `HealthEndpoint` (`for_new_daemon`, `from_record`, `as_str`, `Display`), `ControlListener` (`bind`, a bounded-wait accept shaped to slot into `HealthListener::run`'s loop, `wake`, `remove`), `ControlStream` (`Read + Write`, per-operation timeouts), `connect`. +2. `lock.rs`: `DaemonLock::try_acquire(state_dir)` per spec; a module doc saying the file is never unlinked. +3. Unit tests (cfg as needed): bind/connect/round-trip a line; a second `bind` on a live endpoint fails and does not unlink the live socket; a stale socket file is reclaimed; a non-socket file at the path is refused; an over-long path error names `HYPERDB_STATE_DIR`; socket mode is `0600`; a silent peer hits the read timeout; lock: a second in-process acquire is `None`, released on drop, the lock file survives release. +4. Windows scratch-crate clippy for `control.rs` and `lock.rs`. + +Modules are `pub` (tests use them; `daemon` is `pub` in the lib — confirm), so no dead-code `expect` is needed. + +Commit: `feat(mcp): per-user control transport and daemon lock`. + +## Iteration 4: switch the daemon to the control transport + +**Files:** `daemon/health.rs`, `daemon/discovery.rs`, `daemon/spawn.rs`, `daemon/run.rs`, `daemon/mod.rs`, `daemon/state_perms.rs`, `main.rs`, `engine.rs`, `server.rs`, `diagnostics.rs`; tests `daemon_tests.rs`, `recovery_tests.rs`, `doctor_tests.rs`, `health_idle_cost_tests.rs`, `engine_tests.rs`, `readme_tests.rs` and the in-file unit tests of the touched modules; docs `hyperdb-mcp/README.md` (daemon section ~321-325, CLI reference ~1044-1060), `hyperdb-mcp/DEVELOPMENT.md` (~244-253 and the discovery/identity sections), `src/readme.rs` if it mentions the port, `.claude/workflows/plan-to-release.js` (~548), CHANGELOG.md. + +Roughly 120 references change (daemon_tests ~48, diagnostics ~30, spawn ~15, engine ~11, server ~10); grep `health_port`, `HYPERDB_DAEMON_PORT`, `isolated_daemon_port`, `--port`, `LiveFromScan`. + +1. `state_perms.rs`: the ownership and no-group/other-write check (spec section 3, State directory), used by the daemon and the client. +2. `health.rs`: `HealthListener` wraps `ControlListener`; `DaemonState` stores the endpoint at construction plus a SeqCst `AtomicBool` registered flag (spec); `send_command`, `send_command_with_timeout`, `ping_identified`, `report_hyperd_error_to_daemon` take `&HealthEndpoint`. Keep the absolute I/O deadline and the 64 KiB response cap. Rewrite the module doc's security claims to what is now true. +3. `discovery.rs`: `DaemonInfo.health_endpoint: String`; delete the scan machinery listed in the spec; `discover()` = record + identified `PING`, nothing else. +4. `run.rs`: create and check the state dir, then acquire `DaemonLock` with the `STARTUP_WAIT` retry and the debounced "already running" check (spec section 4, point 3; the pre-flight joins the same budget), bind the control endpoint, start hyperd, write the record. Hold the lock until after `drop(hyper_state)` and socket removal. `DaemonConfig` loses `port`. +5. `spawn.rs`: `ensure_daemon()` per spec (no scan; held lock → wait, free lock → stale cleanup then spawn); `spawn_detached()` without `--port`. +6. `main.rs`: drop `--port` from `daemon`, `daemon stop`, `daemon status`; status prints "Health endpoint:"; rewrite the `daemon --help` text. `engine.rs`/`server.rs`: `daemon_health_endpoint`, status JSON key `daemon_health_endpoint`. `diagnostics.rs`: `health_endpoint`, drop `LiveFromScan` and the scan request/candidate types, `StatusHealthPortMismatch` → endpoint mismatch, report the lock state; still side-effect free. +7. Tests: migrate every TCP fake peer (`CliStatusListener`, `run_controlled_health_peer`, `RunningHealthListener`, `HeldHeartbeatPeer`, health.rs `TestPeer`, …) to `ControlListener` on a per-test endpoint; `TestDaemon` (`daemon_tests.rs` ~2008-2103) drops `HYPERDB_DAEMON_PORT` / `port: 0` and its `Drop` waits for the lock to be free; `DoctorSandbox` drops `HYPERDB_DAEMON_PORT` and gets a state dir short enough for `sun_path` (today 93 bytes + `/daemon.sock` = 105 > 103). Delete the port CLI tests (`daemon_tests.rs` ~600-678, ~699-764, the base-port assertion ~887-889) and `doctor_cli_reports_live_from_scan_via_real_health_listener` (`doctor_tests.rs` ~2060-2114); replace `health_listener_second_bind_same_port_fails` with a lock-based single-instance test. New tests: state dir owned by someone else or group-writable → local mode (Unix; group-writable is testable without root); a client that finds the lock held and no record waits and does not spawn. +8. Docs and doc-contract tests in this same iteration: README daemon section and CLI reference (state dir, socket or pipe, no port, `--port` and `HYPERDB_DAEMON_PORT` gone, `HYPERDB_STATE_DIR` for isolation), DEVELOPMENT.md, the `readme_tests` / `doctor_tests` CLI pins rewritten to the new contract, `plan-to-release.js`. CHANGELOG `### Removed` (`--port`, `HYPERDB_DAEMON_PORT`, port scan) and `### Changed` (health channel transport, `daemon.json` field, status key, doctor fields, the state-dir ownership rule; old clients against a new daemon fall back to local mode). + +Commit: `feat(mcp)!: daemon health channel is a per-user Unix socket or named pipe`. + +## Iteration 5: retire legacy daemons + +**Files:** new `daemon/legacy.rs`, `daemon/discovery.rs` (the legacy-shape hook), `daemon/spawn.rs`, `daemon/mod.rs`, `daemon_tests.rs`, CHANGELOG.md. + +1. `legacy.rs` per spec section 3, Legacy daemons: parse the legacy shape; identified `PING`, `STATUS` pid must equal the record pid, else remove the stale record under the lock and send nothing; `STOP`; wait ≤ 15 s for the port to stop answering; on timeout, local mode without a spawn. +2. Tests with an in-test fake legacy peer on `127.0.0.1:0` and a legacy-shaped record: retired and record gone; pid mismatch → no `STOP` received and record removed; a peer that never closes → local mode, no spawn. +3. CHANGELOG `### Changed`: a running rc.5 daemon is stopped and replaced on first contact. + +Commit: `feat(mcp): retire a pre-1.0 TCP daemon on takeover`. + +## Iteration 6: takeover race + +**Files:** `daemon/run.rs`, `daemon/health.rs`, `daemon/spawn.rs`, `daemon/mod.rs`, `main.rs`, `engine.rs`, `daemon_tests.rs`, README.md / DEVELOPMENT.md (takeover wording), CHANGELOG.md. + +1. `DaemonState` gains `Notify`; `request_shutdown` calls `notify_one()`; `run_daemon`'s `select!` gains the branch. Shutdown order: record removed → listener joined → hyperd dropped → socket removed → lock dropped. +2. `maybe_take_over` and `daemon stop`: send `STOP`, wait until the lock is free (`STOP_WAIT` 15 s), then spawn / report; on timeout local mode / a clear CLI error. +3. `SPAWN_TIMEOUT` 20 s; `STARTUP_WAIT` 10 s (already used since iteration 4; confirm the value). +4. `main` logs a `run_daemon` error with `tracing::error!`; the engine's local-mode fallback logs at `warn`. +5. Fix the comments listed in spec section 4, point 6. +6. Tests per spec section 4, Tests: the takeover test (bound 30 s; red with `STARTUP_WAIT` = 0), the prompt-STOP unit test (record gone within 1 s; red with the `Notify` branch disabled), `daemon stop` returns only after the lock is free. +7. CHANGELOG `### Fixed`: version takeover starts the new daemon reliably; `daemon stop` waits for the daemon to exit; the stale-record race. + +Commit: `fix(mcp): version takeover starts the new daemon reliably`. + +## Phase 5 + +End-to-end: whole `cargo test -p hyperdb-mcp` and `make test`. Manual checks on this machine: a takeover (start a daemon with the previous build if available, otherwise two builds of this branch with an edited record version); `hyperdb-mcp daemon status` / `stop` / `doctor` against the new daemon; `kill -9` a daemon and confirm its hyperd exits and the next start succeeds (spec section 4 assumption). Then the parallel dual review (`feature-dev:code-reviewer` + `code-review`) with the reviewer brief, including the removed-surface-vs-changelog check. Update the ssteiner-ai STATUS.md (uncommitted) with the results and the deferred items (read-only attach, read-only functions, mc9-02, mc9-03, Windows integrity label, old client vs new daemon). diff --git a/docs/superpowers/specs/2026-10-10-mcp-security-design.md b/docs/superpowers/specs/2026-10-10-mcp-security-design.md new file mode 100644 index 00000000..1efafbd9 --- /dev/null +++ b/docs/superpowers/specs/2026-10-10-mcp-security-design.md @@ -0,0 +1,450 @@ +# hyperdb-mcp security fixes — design + +Status: approved direction (pre-1.0 audit decision 9), revised after the +Phase-2 dual review. Breaking changes are acceptable: this lands before 1.0.0. + +Findings covered: + +| Finding | Problem | +|---|---| +| `A1:X-sec-b-01` | `--read-only` is bypassed by `query_data`, `query_file` and `export` | +| `A1:X-sec-b-06`, `A2:mc11-03` | `export` overwrites by default and `remove_dir_all`s any directory | +| `A2:mc9-01` / `F1:mc9-03` | the daemon's health protocol is unauthenticated loopback TCP, reachable by every local user | +| `F1:mc9-01` | a version takeover usually fails: the new daemon dies on the old hyperd's socket | + +Decisions taken with the user: the health channel moves to a **per-user Unix +domain socket** (Unix) and a **named pipe with an owner-only DACL** (Windows), +replacing loopback TCP; the takeover race is fixed in the same change. + +## Measured Hyper behaviour + +Probed against the pinned hyperd (`0.0.26359`) on 2026-10-10. The design below +leans on every row, so a hyperd bump should re-check them (the iteration-1 +tests pin the ones that matter end to end). + +| Input | Result | +|---|---| +| Two statements through `prepare` or `execute_query` (Parse) | rejected, `42601` | +| Two statements through `execute_command` (simple query) | rejected, `0A000` "Multi-part queries are disabled" | +| One statement with a trailing `;`, or a trailing `--` comment, through `prepare` | accepted | +| An unbalanced `)` through `prepare` | rejected, `42601` | +| `WITH x AS (SELECT 1) DELETE FROM t ...` through `execute_query` | **runs the `DELETE`** | +| `WITH x AS (SELECT ...) INSERT INTO t ...` | **runs the `INSERT`** | +| `WITH d AS (DELETE ... RETURNING *) SELECT ...` | rejected, syntax error | +| `SELECT ... INTO t2` | rejected, "SELECT INTO is deprecated" | +| `EXPLAIN ANALYZE DELETE ...`, `EXPLAIN VERBOSE ...`, `EXPLAIN (VERBOSE) ...` | rejected, syntax error / unknown option | +| `EXPLAIN (ANALYZE) DELETE ...`, `EXPLAIN (ANALYZE) WITH ... DELETE ...` | **runs the `DELETE`** | +| `BEGIN READ ONLY` then `DELETE` | **deletes**: read-only transactions are not enforced | +| `SET default_transaction_read_only = on` | rejected, cannot change parameter | +| `/* a /* b */ c */` | the comment ends at the first `*/`: block comments do **not** nest | +| `E'a\'b'`, `$$a)b$$`, `$q$a)b$q$`, `'a\'` | all accepted; a plain string keeps `\` literal | + +## 1. Read-only SQL + +### Problem + +`query` and `chart` reject non-read-only SQL with `engine::is_read_only_sql` +(server.rs ~2697, ~2945). `query_data` and `query_file` (server.rs ~1945, +~2008) and `export` (server.rs ~3252) do not, and a qualified name +(`persistent.public.t`, `.public.t`) reaches the read-write persistent +attachment, so `DROP TABLE persistent.public.t` succeeds under `--read-only`. + +The classifier itself is also weaker than the tools assume. It reads only the +first keyword, so `WITH ... DELETE`, `WITH ... INSERT` and +`EXPLAIN (ANALYZE) DELETE` count as read-only and run (table above), through +`query` too. Its comment stripper nests block comments while Hyper does not, +so a comment Hyper closes early can hide a statement the stripper believes is +commented out. + +`export` has one more hole: it splices the caller's SQL into +`COPY ({sql}) TO WITH (...)` (export.rs ~257, ~392). A single statement +that starts with `SELECT` and closes the parenthesis itself can choose its own +`TO` target. That passes any keyword check and also defeats the overwrite and +protected-path checks of section 2, which look only at `params.path`. +(Hyper refuses multiple statements on both protocols, so it is the one-statement +form that matters.) + +### Design + +1. **A classifier that follows Hyper's grammar** (new module + `hyperdb-mcp/src/sql_classify.rs`, re-exported from `engine`). + `classify_statement` is rebuilt on a small tokenizer that matches Hyper's + lexer (`hyper/parser/sql/SQLLexer.cpp`): ASCII whitespace plus Hyper's + Unicode whitespace list (NBSP, U+2000-200F, U+2028/2029, U+3000, BOM); + `--` to `\r` or `\n`; `/* ... */` **without nesting** (an unterminated one + is a lex error); `'...'` with `''` (backslash literal); `E'...'` that also + skips the byte after a backslash; a string followed by whitespace holding + a newline (or a comment) and another `'` continues in the same escape mode + (`checkStringContinuation`); `"..."` with `""`; `$` plus a digit is a + parameter, any other `$` opens a dollar quote whose tag is **everything** + up to the next `$`, closed by that exact marker; `\x` is a client command; + `(`, `)`, `;`; words (ASCII letters, `_` or any non-whitespace non-ASCII + character to start, uppercased); everything else is opaque. `B'`, `N'`, + `X'`, `U&'` and `UESCAPE` do not move a string boundary, so they need no + case of their own. Classification works on the depth-0 tokens: + - A lex error, an unbalanced parenthesis, nesting deeper than 256 (the + item tree is recursive, so unbounded depth would overflow the stack), + or any token after a depth-0 `;` gives `Other` (not read-only). + - `SELECT`, `VALUES`, `TABLE`, `SHOW` give `ReadOnly`, except that a + depth-0 `INTO` (a reserved word, so only `SELECT ... INTO new_table`) + gives `Dml`; Hyper refuses it unless `enable_select_into` is set. The + same applies to the main query of a `WITH`. A statement that + starts with `(` can only be a query (`select_with_parens`), so the + group's contents are classified. + - `WITH`: the statement's kind is the kind of the first depth-0 word among + `SELECT VALUES TABLE INSERT UPDATE DELETE MERGE COPY` (a CTE cannot be + named `SELECT`, `VALUES` or `TABLE` unquoted). Depth-0 groups before it + (CTE bodies, column lists, a parenthesized main query) are classified + too: a group that is a write gives that kind (a writable CTE, which + Hyper rejects today anyway), a read-only group is remembered. With no + such word, a remembered read-only group gives `ReadOnly` + (`WITH x AS (...) (SELECT ...)`), otherwise `Other`. A CTE *named* + `insert` or `delete` makes a read-only query look like DML; that false + refusal is accepted. + - `EXPLAIN`: Hyper's option list accepts any option name, quoted ones + included (`EXPLAIN ("analyze") ...`), so **any** parenthesized option + list gives the kind of the statement after it, and so do the bare + PostgreSQL words `ANALYZE` / `ANALYSE` / `VERBOSE`. That also covers the + explainable `TRUNCATE` (DDL) and `EXECUTE` (`Other`). A plain `EXPLAIN` + only plans and stays `ReadOnly`; a bare `EXPLAIN` is `Other`. + - The DDL / DML / transaction-control keyword lists stay as they are. The + `execute` batch validator (server.rs ~2788) uses the same function and + gets the same, now correct, answers. + `strip_leading_sql_comments` gets the non-nesting rule too (it has other + callers). The old doc comments claiming "Hyper rejects data-modifying CTEs" + are replaced by the measured facts. +2. **Every SQL-taking read tool checks.** `query_data`, `query_file` and + SQL-mode `export` call `is_read_only_sql` **unconditionally**, exactly like + `query`: the rule belongs to the tool, not the flag. Rejection is + `ErrorCode::SqlError` with the `query` tool's wording, naming the tool + (`engine::require_read_only_sql`). `query_data` and `query_file` check the + caller's original SQL before ingest, and check again after + `replace_identifier`: `table_name` is caller-controlled, so substituting it + can change how the SQL tokenizes. `copy_query` and the saved-query re-run + already call `is_read_only_sql`; they inherit the stronger classifier. +3. **Export embeds only a statement Hyper parsed on its own.** Before + splicing, `export` sends the caller's SQL alone through + `Connection::prepare` (Parse/Describe, never executed, then dropped). Hyper + rejects unbalanced or multi-statement input there (table above). The + splice then puts the SQL on its own lines, `COPY (\n{sql}\n) TO ...`, so a + trailing `--` comment cannot swallow the suffix. A statement that parses + alone has balanced parentheses and no unterminated literal or comment, so + it cannot close the wrapper. This guards CSV, Parquet, Arrow IPC and + Iceberg (the two `COPY` sites); Hyper-format export copies whole tables and + takes no SQL. +4. The internal writes these tools make (scratch-table ingest and `DROP`, + `export_hyper`'s `CREATE DATABASE` / `ATTACH` / `DETACH` of its own + target) are separate statements and are untouched. `export` stays allowed + under `--read-only` (docs and `tool_schema_tests` pin that). + +### Out of scope (recorded for later) + +- Attaching the persistent database with `WITH (access_mode='readonly')` + under `--read-only` would make hyperd enforce it (`extractDatabaseAttachOptions` + in `hyper-db/hyper/cts/compiler/Compiler.cpp`), but the KV readers and the + saved-query store run `CREATE TABLE IF NOT EXISTS` on it today, by design. +- Read-only *functions* with side effects, if Hyper has any, are not + classified. + +## 2. Export and chart never destroy what they did not write + +### Problem + +`ExportParams.overwrite` defaults to `true` (server.rs ~3311). With it, +`export_iceberg` calls `remove_dir_all` on whatever directory `path` names +(export.rs ~373-389), and `export_hyper` unlinks whatever file is there +(~484-508), including the session's own persistent database or an attached +`.hyper` file. `chart` with an explicit `output_path` and `overwrite=true` +writes through `write_chart_to_disk` (chart.rs ~335-364) and can clobber the +same files. `validate_output_path` (attach.rs ~721) requires an absolute path, +canonicalizes, rejects `..` and creates missing parent directories, but the +export handler then discards its result and passes the raw `params.path` on +(server.rs ~3262 vs ~3309). + +### Design + +1. **`overwrite` defaults to `false`** for `export` and for `chart`. An + existing destination is then `PERMISSION_DENIED` with the existing "pass + overwrite=true to replace it" message. Auto-generated chart paths are + unique, so only an explicit `output_path` is affected. +2. **Use the validated path.** The path `validate_output_path` returns is the + one that is checked and the one handed to hyperd or the filesystem. +3. **Never write over a file the session uses**, for every export format and + for chart, even with `overwrite=true`. The server builds the protected set + once per call: the persistent database, the ephemeral scratch database, + and every `AttachSource::LocalFile` in the attach registry. When the + destination exists, it is refused with `INVALID_ARGUMENT` if it is the + *same file* as a protected one, compared by identity: `(dev, ino)` on Unix, + (volume serial number, file index) on Windows. Identity sees through + symlinks, hard links and case folding, which a path comparison does not. A + destination that does not exist cannot be one of them. The helper lives in + a self-contained `src/file_identity.rs` (std and `windows-sys` only). +4. **Iceberg replaces only an Iceberg table.** With `overwrite=true` and an + existing `dest`: + - it must be a directory that is empty, or whose top-level entries are a + subset of the layout Hyper's Iceberg export writes (at least + `metadata/`, holding a `*.metadata.json`; iteration 2 measures the full + set from a real export and pins it). Anything else, including a regular + file or a directory holding a `.hyper` database, is refused with + `INVALID_ARGUMENT`: "Refusing to replace '``': it is not an Iceberg + table directory". + - `dest` is renamed to a sibling backup `.hyperdb-mcp-old--` + and the export runs. On success the backup is removed. On failure, + whatever this call created at `dest` is removed first (it is ours: the + original was already moved away), then the backup is renamed back and + the export error is returned. If the restore itself fails, the error + names the backup path. A backup left behind by a crash is never deleted + automatically; the docs say so. +5. **Hyper replaces only a Hyper file, and never touches it until the new one + is complete.** With `overwrite=true` and an existing `path`, refuse unless + it is a regular file (not a symlink, not a directory) whose first five + bytes are `Hyper` (the file magic; verified against real files: + `48 79 70 65 72 08 00 00 ...`). The export then creates its database at a + sibling temp path `.hyperdb-mcp-tmp--`, fills and detaches + it, and renames it over `path`. On failure the temp file is removed and + `path` was never touched. The existing `is_file_in_use` → + `RESOURCE_BUSY` mapping applies to the final rename. +6. CSV, Parquet and Arrow IPC keep letting hyperd's `COPY ... TO` replace a + file when `overwrite=true`; points 1 to 3 cover them. + +## 3. Daemon health channel: per-user local socket + +### Problem + +The health listener binds `127.0.0.1:` (scanned from 7485). Every local +user can connect: `STOP` kills another user's daemon, `STATUS` leaks the +hyperd endpoint, `REPORT_HYPERD_ERROR` forces restarts. The module doc's +"at most one daemon per user" is really "per port". The bind is also the only +single-instance lock. + +### State directory + +Everything below lives in the state dir, and on Unix the state dir is what +keeps other users out, so it is checked rather than assumed. Today +`restrict_existing_dir` only warns when `chmod` fails (state_perms.rs +~106-122). New rule, Unix only, in both the daemon and the client: after +`ensure_owner_only_dir`, the dir must be owned by the effective uid and have +no group or other write bit. Otherwise the daemon refuses to start and the +client goes straight to local mode, each with a warning naming the path. (On +Windows the default `%LOCALAPPDATA%` location is per-user; the pipe DACL is the +boundary there.) + +The daemon creates and checks the state dir **before** it opens the lock file. + +### Transport + +New module `daemon/control.rs` hides the platform split behind one surface +the rest of the daemon and every test uses: + +```rust +/// Where a daemon's health channel listens. +pub struct HealthEndpoint(/* Unix: socket path; Windows: pipe name */); +impl HealthEndpoint { + pub fn for_new_daemon(state_dir: &Path) -> io::Result; + pub fn as_str(&self) -> &str; // what daemon.json carries + pub fn from_record(s: &str) -> Self; +} +pub struct ControlListener { .. } // bind, accept (bounded wait), wake +pub struct ControlStream { .. } // Read + Write, per-operation timeouts +pub fn connect(ep: &HealthEndpoint, connect_timeout: Duration) -> io::Result; +``` + +- **Unix:** `/daemon.sock`, a `std::os::unix::net::UnixListener`, + `chmod 0600` after `bind`. A path longer than `sun_path` (103 usable bytes + on macOS, 107 on Linux) fails `bind` with a message naming + `HYPERDB_STATE_DIR`; the client then falls back to local mode as it does for + any daemon failure. A leftover socket file is removed before `bind` only + while the daemon holds the lock and only if it is a socket (anything else + at the path is an error). The accept loop keeps today's design: + non-blocking listener, `libc::poll` with the 1 s cadence, accept then force + blocking, the self-connect wake on shutdown. Per-connection read and write + timeouts stay 5 s. +- **Windows:** `\\.\pipe\hyperdb-mcp--<16 hex digits>`, the digits from a + `RandomState` hash (seeded from OS randomness), so the name cannot be + guessed or pre-created. Raw `windows-sys` 0.61 (already in the lock file): + - every instance: `CreateNamedPipeW` with `PIPE_ACCESS_DUPLEX | + FILE_FLAG_OVERLAPPED`, `PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | + PIPE_REJECT_REMOTE_CLIENTS`, `PIPE_UNLIMITED_INSTANCES`, and the same + security descriptor from SDDL `D:P(A;;GA;;;)` (SID from + `GetTokenInformation(TokenUser)` + `ConvertSidToStringSidW`). The first + instance adds `FILE_FLAG_FIRST_PIPE_INSTANCE`. Pipe names are enumerable, + so the DACL, not the random name, keeps other users out; the random name + stops squatting. + - `bind` creates the first instance, so the pipe exists before + `daemon.json` names it. The accept loop waits in an overlapped + `ConnectNamedPipe` with a bounded `WaitForSingleObject` (1 s, rechecking + the shutdown flag; `ERROR_PIPE_CONNECTED` counts as connected), so a lost + wake cannot hang the listener join and with it the lock. It creates the + next instance **before** handing the connected one to a handler, so a + client never finds zero instances (which reads as `ERROR_FILE_NOT_FOUND`, + "no daemon"). + - `ControlStream` does overlapped `ReadFile` / `WriteFile` with a + per-operation timeout and `CancelIoEx` on expiry, on both the server and + the client (the client opens with `FILE_FLAG_OVERLAPPED` through + `OpenOptionsExt::custom_flags`). That gives Windows the same 5 s + per-connection bound as Unix with no helper threads. `ERROR_BROKEN_PIPE` + and `ERROR_NO_DATA` read as a closed connection. A handler calls + `FlushFileBuffers` before closing its instance. + - Clients retry `ERROR_PIPE_BUSY` until the connect timeout, like + `hyperdb-api-core`'s `connect_named_pipe`; `ERROR_FILE_NOT_FOUND` means + no daemon. + - Known limitation: a daemon started elevated may label its pipe with a + High integrity level that non-elevated clients cannot open. Not handled; + recorded. + +### Single-instance lock + +New module `daemon/lock.rs`: `DaemonLock::try_acquire(state_dir) -> +io::Result>` on `/daemon.lock` (`0600`). Unix: +`flock(LOCK_EX | LOCK_NB)` through `libc`, opened with `O_NOFOLLOW | +O_CLOEXEC` (std's `File::try_lock` needs Rust 1.89; MSRV is 1.88). Windows: +open with `share_mode(0)`; a sharing violation (error 32) means held. The OS +releases both on process exit, so a crashed daemon never leaves the lock +behind. Both conflict within one process too (separate opens), which the +in-process `TestDaemon` relies on. **`daemon.lock` is never unlinked**: a +process holding the lock on a deleted inode would no longer exclude a newcomer. + +The daemon holds the lock **for its whole life, until after hyperd has +exited**. That ordering is what fixes the takeover race (section 4), and the +lock state is also what clients read to tell "starting or stopping" from +"gone". + +### Discovery record and `discover()` + +`DaemonInfo.health_port: u16` becomes `health_endpoint: String` (the socket +path or pipe name). Everything that flows a port changes with it: +`Engine::daemon_health_port()` → `daemon_health_endpoint() -> Option<&str>`, +the status JSON key, `report_hyperd_error_to_daemon`, `send_command*`, +`ping_identified`, the doctor section and its warnings. The wake on shutdown +keeps the SeqCst pairing with the shutdown flag: `DaemonState` stores the +endpoint when it is built and an `AtomicBool` says whether the listener is +registered (a path cannot live in an atomic). + +Removed outright: the port scan (`PortScan`, `resolve_port_scan`, +`resolve_port`, `scan_for_daemon`, `probe_port`, `ProbeResult`, `ScanOutcome`, +the off-base cleanup in `ensure_daemon`), `DEFAULT_DAEMON_BASE_PORT`, +`DAEMON_PORT_SCAN_SPAN`, `ENV_DAEMON_PORT` / `HYPERDB_DAEMON_PORT`, +`DaemonConfig::port`, and the CLI's `daemon --port`, `daemon stop --port`, +`daemon status --port`. The doctor's `LiveFromScan` state goes too. Isolation +between daemons is by state dir (`HYPERDB_STATE_DIR`), which tests already use. + +`discover()` reads `daemon.json`, connects to its `health_endpoint` and +requires the identified `PONG`. That is the only way to find a daemon. + +`ensure_daemon()`: + +1. `discover()` finds a live daemon (and the version check passes): use it. +2. Otherwise probe the lock. **Held**: a daemon is starting (the record is + written only after hyperd is up, about 6-7 s in) or stopping, so wait for + it with `wait_for_daemon` instead of spawning a second one. **Free**: no + daemon is alive; remove a stale record while holding the lock (closing + today's read-then-delete race that could remove a newer daemon's record), + release, spawn. + +The doctor stays side-effect free: it reads the record and the lock state but +never cleans up. + +### Legacy daemons (rc.5 and earlier) + +A running pre-1.0 daemon publishes `health_port` and holds no lock. If the new +client simply spawned, the new daemon would get the lock and then fail its +hyperd-socket pre-flight against the old engine, and every new client would +sit in local mode until the old daemon exits, which by default is never. So +`daemon/legacy.rs` (marked for removal after 1.x), when `daemon.json` parses +only as the legacy shape: + +1. sends an identified `PING` to `127.0.0.1:` with the old + protocol, then `STATUS`, and continues only if the reported pid equals the + record's pid. Anything else is a stale record pointing at a port someone + else may now hold, so it is removed (under the lock) without sending + anything further. That keeps the new client from sending `STOP` to another + user's daemon, the very bug this change fixes. +2. sends `STOP` and waits, at most 15 s, until the port stops answering. The + rc.5 daemon removes `daemon.json` *before* joining its listener (run.rs + ~172-174), so once the port is quiet the record is gone too and cannot + later delete the new daemon's. The new daemon's startup wait (section 4) + covers the old hyperd still exiting. +3. on timeout, goes to local mode and never spawns. + +An old client meeting a new daemon cannot parse the record. It waits out its +10 s spawn timeout on every start and then runs in local mode; nothing is +destroyed. The CHANGELOG says so. + +## 4. Takeover race + +### Problem (timeline from the code) + +A newer client sends `STOP`. The old daemon's `request_shutdown` sets a flag +and the health listener closes within milliseconds, but nothing in +`run_daemon`'s `select!` observes the flag except `hyperd_monitor`, every 5 s. +The client sees the health port free and spawns the new daemon at once. The +new daemon's `build_params` pre-flight connects to `/sockets/hyper`, +which the old hyperd still serves, and exits with `AddrInUse` to a stderr +that is `/dev/null`. The old daemon then shuts down, the client waits out +`SPAWN_TIMEOUT` (10 s) and falls back to local mode, and no daemon runs. +Introduced by #304 (IPC engine at a fixed socket path); the comment at +run.rs ~251 claiming the pre-flight is reachable only with two daemons on +different ports is wrong. + +### Design + +1. **Act on STOP promptly.** `DaemonState` gains a `tokio::sync::Notify`; + `request_shutdown` calls `notify_one()` (which stores a permit, so a STOP + that lands before the branch is polled is not lost). `run_daemon`'s + `select!` gets a branch awaiting it. Shutdown then runs immediately: + remove the record, join the listener, drop hyperd (`HyperProcess::drop`, + up to about 5.1 s), remove the socket file, and only then drop the lock. +2. **Whoever sends STOP waits for the lock.** `maybe_take_over` and + `hyperdb-mcp daemon stop` send `STOP` and then wait until the lock is free + (the old daemon and its hyperd are gone), bounded by `STOP_WAIT` = 15 s. + Only then does the takeover spawn or the CLI report success. On timeout the + takeover goes to local mode, and `daemon stop` says the daemon did not exit. +3. **The new daemon also waits at its initial start**, for every other path + (a manual start right after a kill, a second client racing a spawn, a + client's brief lock probe). `run_daemon` retries the lock and the + hyperd-socket pre-flight within **one** `STARTUP_WAIT` budget of 10 s + (`HyperProcess::drop`'s ~5.1 s, the listener join's ~1 s, and slack for a + loaded CI runner). If the lock is held and the record's endpoint answers an + identified `PING` on two attempts 1 s apart, a live daemon is serving, so + fail with "already running" rather than wait the budget out (a daemon + still answering in its last milliseconds does not count). The retry exists + from the iteration that introduces the lock. `try_restart_hyperd` does not + retry: there the daemon has just reaped its own child. +4. **Timeouts.** `SPAWN_TIMEOUT` rises from 10 s to 20 s to cover a + `STARTUP_WAIT` plus hyperd's start. +5. **Make failures visible.** `main` logs a `run_daemon` error with + `tracing::error!` to the daemon log before exiting (stderr is `/dev/null` + when spawned); `engine.rs` logs the local-mode fallback at `warn`, not + `debug` (it is `debug` today, engine.rs ~611). +6. Fix the wrong comments: run.rs ~246-256 (pre-flight reachability), + ~237-242 (the obsolete `HyperProcess::drop` landmine; `HyperProcess` now + tracks `owns_socket_directory`), spawn.rs ~157-158 and ~211-220. + +Assumption, checked by hand in Phase 5: a hyperd orphaned by a `SIGKILL`ed +daemon exits on its own once its callback connection closes, so the +pre-flight retry eventually succeeds. If it does not, the new daemon fails +after `STARTUP_WAIT`, logs why, and clients use local mode; killing a stale +hyperd found through `hyper.pid` is out of scope. + +### Tests + +- **Takeover:** an in-process integration test, two daemons in sequence on + **one** state dir: start daemon A, send `STOP`, immediately start daemon B on + the same dir, and require B to become discoverable (new `started_at`), serve + a query through its engine, and A's thread to exit cleanly, within 30 s. + Red-before-green with `STARTUP_WAIT` set to zero (today's behaviour: B + fails on the lock or the pre-flight). +- **Prompt STOP:** a unit test that a `STOP` makes the daemon remove its + record within 1 s (the poll interval it replaces is 5 s), measured on the + record, not on `run_daemon`'s return, which includes the hyperd drop. + Red-before-green by disabling the `Notify` branch. +- **No pile-up:** a client that finds the lock held and no record waits and + does not spawn. + +## Non-goals + +- `mc9-02` (server-side line length and thread count are unbounded) and + `mc9-03` (the blocking restart inside `select!`). With the channel limited to + the owning user, mc9-02 is a self-DoS; both stay open in the audit. +- TLS for the MCP's own connections (decision 8 deferred it). +- Fixing an old client against a new daemon. diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 2631912a..788361c0 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -24,6 +24,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- **`query_data`, `query_file` and `export` now accept only read-only SQL**, + with or without `--read-only`. They used to run any statement, so + `query_data(sql: "DROP TABLE persistent.public.t")` dropped a persistent + table even on a read-only server. Writes are refused with `SQL_ERROR`, as + `query` already did. +- **The read-only SQL check now sees statements the way Hyper does.** + `WITH x AS (...) DELETE ...` and `EXPLAIN (ANALYZE) DELETE ...`, both of + which Hyper runs, passed as read-only through `query`, `chart`, + `save_query` and `copy_query`. Block comments are no longer treated as + nesting (Hyper's do not), string literals, quoted identifiers and dollar + quotes are skipped the way Hyper's lexer skips them, `SELECT ... INTO` + counts as a write, and input that is not exactly one statement (or nests + parentheses more than 256 deep) is refused. The `execute` batch validator uses the + same classifier, so it now counts those wrapped writes as DML. +- **`export` with `sql` can no longer break out of its `COPY (...) TO` + wrapper.** SQL that closed the parenthesis itself could name its own + target path and options; it is now refused, and a query that ends in a + `--` comment exports instead of failing with a syntax error. - **`query_data` / `query_file` no longer interpolate `table_name` unescaped into the scratch-table `DROP`.** A `table_name` containing a double quote could end the quoted identifier and append further SQL to the cleanup diff --git a/hyperdb-mcp/README.md b/hyperdb-mcp/README.md index d2c548d2..bd948023 100644 --- a/hyperdb-mcp/README.md +++ b/hyperdb-mcp/README.md @@ -362,7 +362,7 @@ query_data(data: '[{"region":"West","revenue":1200},...]', sql: 'SELECT region, | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `data` | string | yes | JSON array of objects, or CSV text | -| `sql` | string | yes | SQL query to run against the data | +| `sql` | string | yes | Read-only SQL query to run against the data | | `format` | string | no | `"json"` or `"csv"` — auto-detected if omitted | | `table_name` | string | no | Table name for use in SQL — defaults to `"data"` | | `schema` | object | no | Partial column-name → type map (see [Schema Overrides](#schema-overrides)) | @@ -378,7 +378,7 @@ query_file(path: '/tmp/sales.parquet', sql: 'SELECT TOP 10 * FROM data ORDER BY | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `path` | string | yes | Path to CSV / JSON / JSONL / Parquet / Arrow IPC file | -| `sql` | string | yes | SQL query to run | +| `sql` | string | yes | Read-only SQL query to run | | `table_name` | string | no | Table name for use in SQL — defaults to `"data"` | | `schema` | object | no | Partial column-name → type map (see [Schema Overrides](#schema-overrides)) | | `json_extract_path` | string | no | Dot-separated path to a nested array in a JSON wrapper file (numeric segments index arrays) | @@ -698,7 +698,7 @@ export(sql: 'SELECT ...', path: '~/Desktop/analysis.hyper', format: 'hyper') | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| `sql` | string | no | Query to export (if omitted, exports whole table) | +| `sql` | string | no | Read-only query to export (if omitted, exports whole table) | | `table` | string | no | Table name (used if `sql` omitted) | | `path` | string | yes | Output file path | | `format` | string | yes | `"csv"`, `"parquet"`, `"iceberg"`, `"arrow_ipc"`, or `"hyper"` | @@ -893,7 +893,7 @@ hyperdb-mcp --persistent-db ~/analytics.hyper --read-only - **Blocked:** `execute`, `load_data`, `load_file`, `load_files`, `load_iceberg`, `watch_directory`, `save_query`, `delete_query`, `set_table_metadata`, `copy_query`, `kv_set`, `kv_set_many`, `kv_delete`, `kv_pop`, and `kv_clear` — return `READ_ONLY_VIOLATION`. `attach_database` is also guarded when `writable: true` or `on_missing: "create"`; ordinary read-only attachment remains available. - **Resources, prompts, and resource subscriptions** work normally — read-only clients can still subscribe to `hyper://...` URIs and receive notifications when other (non-read-only) connections mutate state -The `query` tool also enforces read-only at the SQL level — only `SELECT`/`WITH`/`EXPLAIN`/`SHOW`/`VALUES` are accepted. +Independently of `--read-only`, every tool that takes a query (`query`, `query_data`, `query_file`, `export`, `chart`, `save_query`, `copy_query`) accepts only a single read-only statement — `SELECT`/`WITH`/`EXPLAIN`/`SHOW`/`VALUES` — and refuses anything else with `SQL_ERROR`. That includes a write behind `WITH` (`WITH x AS (...) DELETE ...`) or `EXPLAIN (ANALYZE)`, both of which Hyper runs. Hyper does not enforce read-only transactions, so this check is the only guard. --- diff --git a/hyperdb-mcp/src/engine.rs b/hyperdb-mcp/src/engine.rs index 4354e3f6..77605915 100644 --- a/hyperdb-mcp/src/engine.rs +++ b/hyperdb-mcp/src/engine.rs @@ -52,6 +52,8 @@ use crate::daemon; use crate::error::{ErrorCode, McpError}; use crate::schema::ColumnSchema; +pub(crate) use crate::sql_classify::strip_leading_sql_comments; +pub use crate::sql_classify::{StatementKind, classify_statement, is_read_only_sql}; use hyperdb_api::{ Catalog, Connection, CopyTableReport, CreateMode, HyperProcess, Parameters, SqlType, Transaction, escape_sql_path, @@ -998,6 +1000,25 @@ impl Engine { self.connection.execute_command(sql).map_err(McpError::from) } + /// Have Hyper parse `sql` on its own (Parse/Describe through + /// [`Connection::prepare`]), without running it, before a caller splices + /// it into a larger statement such as `COPY (...) TO`. + /// + /// Hyper refuses unbalanced parentheses, an unterminated literal or + /// comment, and more than one statement here, so SQL that passes cannot + /// close the surrounding parentheses itself. + /// + /// # Errors + /// + /// Returns the converted [`hyperdb_api::Error`] when Hyper rejects the + /// statement (typically [`ErrorCode::SqlError`]). + pub fn check_embeddable_query(&self, sql: &str) -> Result<(), McpError> { + self.connection + .prepare(sql) + .map(drop) + .map_err(McpError::from) + } + /// Execute an `ATTACH DATABASE` statement for a user-supplied `.hyper` /// file, mapping a lock conflict (SQLSTATE `55006`, or a legacy /// "already attached"/"file is locked" phrase from older hyperd) to @@ -2192,123 +2213,23 @@ fn translate_table_missing(err: McpError, table_name: &str) -> McpError { } } -/// Returns `true` if a SQL statement is read-only: `SELECT`, `WITH`, `EXPLAIN`, -/// `SHOW`, or `VALUES`. Anything else (`CREATE`, `INSERT`, `UPDATE`, `DELETE`, -/// `DROP`, `ALTER`, `COPY`, ...) is considered mutating. +/// Refuse `sql` unless [`is_read_only_sql`] accepts it, naming `tool` in +/// the error. /// -/// Checks whether the first SQL keyword indicates a read-only statement. -/// -/// Strips leading whitespace and SQL comments (line `--` and block `/* */`) -/// before inspecting the first alphabetic token. This prevents comment-based -/// bypass of the read-only guard (e.g. `/* harmless */ DROP TABLE ...`). -/// -/// Note: data-modifying CTEs (`WITH x AS (DELETE ...) SELECT ...`) still slip -/// past this check. Hyper itself rejects such CTEs, so this is defense-in-depth -/// rather than the sole security boundary. -#[must_use] -pub fn is_read_only_sql(sql: &str) -> bool { - matches!(classify_statement(sql), StatementKind::ReadOnly) -} - -/// Coarse classification of a single SQL statement, comment-aware. -/// -/// Used by the atomic-batch `execute` tool to enforce the rule "a batch -/// must be either all-DDL singletons or all-DML; mixing the two aborts -/// the transaction with SQLSTATE 0A000". The first-keyword heuristic -/// matches what `is_read_only_sql` already trusts elsewhere in the -/// codebase. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum StatementKind { - /// `SELECT` / `WITH` / `EXPLAIN` / `SHOW` / `VALUES`. - ReadOnly, - /// `CREATE` / `DROP` / `ALTER` / `TRUNCATE` / `RENAME` — Hyper auto-commits. - Ddl, - /// `INSERT` / `UPDATE` / `DELETE` / `COPY` / `MERGE` — transactional. - Dml, - /// `BEGIN` / `START` / `COMMIT` / `END` / `ROLLBACK` / `ABORT` / - /// `SAVEPOINT` / `RELEASE`. Rejected inside a batch because the - /// `execute` tool already manages the transaction; an explicit - /// COMMIT mid-batch would defeat atomicity. - TransactionControl, - /// Empty/comment-only input or an unrecognized first keyword. Treated - /// as opaque by the batch validator (passed through to Hyper). - Other, -} - -/// Coarse-classify the first SQL statement in `sql` after stripping -/// leading whitespace and line/block comments. +/// # Errors /// -/// First-keyword only: a `WITH x AS (DELETE …) SELECT …` CTE is -/// classified as `ReadOnly` even though it mutates. Hyper itself -/// rejects data-modifying CTEs, so this is a defense-in-depth heuristic -/// rather than the only barrier. -#[must_use] -pub fn classify_statement(sql: &str) -> StatementKind { - let stripped = strip_leading_sql_comments(sql); - let first_token: String = stripped - .chars() - .take_while(|c| c.is_alphabetic()) - .flat_map(char::to_uppercase) - .collect(); - match first_token.as_str() { - "SELECT" | "WITH" | "EXPLAIN" | "SHOW" | "VALUES" => StatementKind::ReadOnly, - "CREATE" | "DROP" | "ALTER" | "TRUNCATE" | "RENAME" => StatementKind::Ddl, - "INSERT" | "UPDATE" | "DELETE" | "COPY" | "MERGE" => StatementKind::Dml, - "BEGIN" | "START" | "COMMIT" | "END" | "ROLLBACK" | "ABORT" | "SAVEPOINT" | "RELEASE" => { - StatementKind::TransactionControl - } - _ => StatementKind::Other, - } -} - -/// Strips leading whitespace, line comments (`--`), and block comments (`/* */`) -/// from SQL text. Handles nested block comments. -pub(crate) fn strip_leading_sql_comments(sql: &str) -> &str { - let mut s = sql; - loop { - s = s.trim_start(); - if s.starts_with("--") { - // Line comment — skip to end of line (handles LF, CRLF, and CR) - match s.find(&['\n', '\r'][..]) { - Some(pos) => { - let mut next = pos + 1; - // Handle CRLF: skip both characters - if s.as_bytes().get(pos) == Some(&b'\r') - && s.as_bytes().get(pos + 1) == Some(&b'\n') - { - next = pos + 2; - } - s = &s[next..]; - } - None => return "", - } - } else if s.starts_with("/*") { - // Block comment — find matching close, handling nesting - let mut depth = 0u32; - let mut chars = s.char_indices().peekable(); - let mut end = None; - while let Some((i, c)) = chars.next() { - if c == '/' && chars.peek().map(|(_, c2)| *c2) == Some('*') { - chars.next(); - depth += 1; - } else if c == '*' && chars.peek().map(|(_, c2)| *c2) == Some('/') { - chars.next(); - depth -= 1; - if depth == 0 { - end = Some(i + 2); - break; - } - } - } - match end { - Some(pos) => s = &s[pos..], - None => return "", // Unclosed comment — no valid SQL - } - } else { - break; - } - } - s +/// Returns [`ErrorCode::SqlError`] for SQL that is not a single read-only +/// statement. +pub(crate) fn require_read_only_sql(tool: &str, sql: &str) -> Result<(), McpError> { + if is_read_only_sql(sql) { + return Ok(()); + } + Err(McpError::new( + ErrorCode::SqlError, + format!( + "The {tool} tool only accepts read-only SQL (SELECT, WITH, EXPLAIN, SHOW, VALUES). Use the execute tool for DDL/DML." + ), + )) } impl Drop for Engine { @@ -2571,67 +2492,6 @@ fn bootstrap_public_schema(connection: &Connection) -> Result<(), McpError> { }) } -#[cfg(test)] -mod statement_helper_tests { - use super::*; - - #[test] - fn classify_statement_recognizes_each_kind() { - assert_eq!(classify_statement("SELECT 1"), StatementKind::ReadOnly); - assert_eq!( - classify_statement("with x as (..) select * from x"), - StatementKind::ReadOnly - ); - assert_eq!( - classify_statement("CREATE TABLE t (i INT)"), - StatementKind::Ddl - ); - assert_eq!(classify_statement("drop table t"), StatementKind::Ddl); - assert_eq!( - classify_statement("INSERT INTO t VALUES (1)"), - StatementKind::Dml - ); - assert_eq!(classify_statement("update t set i = 2"), StatementKind::Dml); - assert_eq!(classify_statement("delete from t"), StatementKind::Dml); - assert_eq!(classify_statement(""), StatementKind::Other); - } - - #[test] - fn classify_statement_recognizes_transaction_control() { - for kw in [ - "BEGIN", - "Begin transaction", - "START TRANSACTION", - "COMMIT", - "Commit work", - "END", - "ROLLBACK", - "Rollback to savepoint sp1", - "ABORT", - "SAVEPOINT sp1", - "RELEASE SAVEPOINT sp1", - ] { - assert_eq!( - classify_statement(kw), - StatementKind::TransactionControl, - "expected TransactionControl for `{kw}`" - ); - } - } - - #[test] - fn classify_statement_strips_comments() { - assert_eq!( - classify_statement("/* harmless */ DROP TABLE t"), - StatementKind::Ddl - ); - assert_eq!( - classify_statement("-- pretend to be readonly\nINSERT INTO t VALUES (1)"), - StatementKind::Dml - ); - } -} - #[cfg(test)] mod endpoint_description_tests { use super::*; diff --git a/hyperdb-mcp/src/export.rs b/hyperdb-mcp/src/export.rs index 5495f403..94916b1f 100644 --- a/hyperdb-mcp/src/export.rs +++ b/hyperdb-mcp/src/export.rs @@ -25,7 +25,7 @@ //! directly in Tableau Desktop. (Cannot use plain `std::fs::copy` because //! on Windows `hyperd` holds an exclusive lock on the workspace file.) -use crate::engine::Engine; +use crate::engine::{Engine, require_read_only_sql}; use crate::error::{ErrorCode, McpError}; use crate::stats::{ExportStats, StatsTimer}; use hyperdb_api::{CopyTableReport, escape_sql_path, escape_string_literal}; @@ -87,7 +87,9 @@ pub struct ExportResult { /// - Returns [`ErrorCode::PermissionDenied`] if `opts.path` already /// exists and `opts.overwrite` is `false`. /// - Returns [`ErrorCode::SqlError`] when neither `opts.sql` nor -/// `opts.table` is provided (for row-oriented formats). +/// `opts.table` is provided (for row-oriented formats), when `opts.sql` +/// is not a single read-only statement, or when Hyper cannot parse it on +/// its own. /// - Returns [`ErrorCode::UnsupportedFormat`] when `opts.format` is /// not one of `hyper`, `csv`, `parquet`, `arrow_ipc`, or `iceberg`. /// - Propagates any format-specific error from the delegated exporter @@ -146,6 +148,12 @@ pub fn export_to_file(engine: &Engine, opts: &ExportOptions) -> Result`. + // It must be read-only, and Hyper must parse it on its own first, so + // it cannot close the parenthesis and name a `TO` target of its own. + require_read_only_sql("export", &select_sql)?; + engine.check_embeddable_query(&select_sql)?; + let extra = opts.format_options.as_ref(); match opts.format.as_str() { "csv" => export_csv(engine, &select_sql, &opts.path, extra, &timer), @@ -234,6 +242,13 @@ fn render_copy_with_clause( Ok(clause) } +/// `COPY () TO WITH ()`, with `sql` on lines of its +/// own so a trailing `--` comment in it ends before the `)`. `sql` must +/// have passed [`Engine::check_embeddable_query`]. +fn copy_to_sql(sql: &str, quoted_path: &str, with_clause: &str) -> String { + format!("COPY (\n{sql}\n) TO {quoted_path} WITH ({with_clause})") +} + /// Shared helper: issue a single `COPY (query) TO 'path' WITH (...)` to /// hyperd and return the reported row count + on-disk file size. All /// row-oriented exports funnel through this — the format-specific logic @@ -254,7 +269,7 @@ fn run_copy_to( ) -> Result { let with_clause = render_copy_with_clause(base_with, extra_options)?; let quoted_path = escape_string_literal(path); - let copy_sql = format!("COPY ({sql}) TO {quoted_path} WITH ({with_clause})"); + let copy_sql = copy_to_sql(sql, "ed_path, &with_clause); let row_count = engine.execute_command(©_sql)?; let file_size = std::fs::metadata(path).map_or(0, |m| m.len()); @@ -389,7 +404,7 @@ fn export_iceberg( let with_clause = render_copy_with_clause("format => 'iceberg'", format_options)?; let quoted_path = escape_string_literal(path); - let copy_sql = format!("COPY ({sql}) TO {quoted_path} WITH ({with_clause})"); + let copy_sql = copy_to_sql(sql, "ed_path, &with_clause); let row_count = engine.execute_command(©_sql)?; diff --git a/hyperdb-mcp/src/lib.rs b/hyperdb-mcp/src/lib.rs index cbc7a805..77005f14 100644 --- a/hyperdb-mcp/src/lib.rs +++ b/hyperdb-mcp/src/lib.rs @@ -96,6 +96,7 @@ pub mod saved_queries; pub mod schema; #[doc(hidden)] pub mod server; +mod sql_classify; #[doc(hidden)] pub mod stats; #[doc(hidden)] diff --git a/hyperdb-mcp/src/readme.rs b/hyperdb-mcp/src/readme.rs index 32161737..1241b659 100644 --- a/hyperdb-mcp/src/readme.rs +++ b/hyperdb-mcp/src/readme.rs @@ -290,8 +290,10 @@ scale 0 and truncates decimal places. Example: `41.54178215::numeric` - **File paths must be absolute.** Relative paths are rejected. - **Identifiers fold to lowercase** unless double-quoted. `SELECT * FROM Sales` reads `sales`. Use `\"Sales\"` to preserve case. -- **`query` is read-only.** SELECT / WITH / EXPLAIN / SHOW / VALUES - only. For DDL / DML use `execute`. +- **Query SQL is read-only** in every tool that takes it (`query`, + `query_data`, `query_file`, `export`, `chart`, ...): one SELECT / WITH / + EXPLAIN / SHOW / VALUES, even without `--read-only`. A write behind + `WITH` or `EXPLAIN (ANALYZE)` is refused. For DDL / DML use `execute`. - **Read-only mode** (`--read-only` flag on the server) guards exactly: `execute`, `load_data`, `load_file`, `load_files`, `load_iceberg`, `watch_directory`, `save_query`, `delete_query`, `set_table_metadata`, diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index ae3ae394..d6b73133 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -18,6 +18,7 @@ use crate::chart::{ }; use crate::engine::{ Engine, StatementKind, classify_statement, is_read_only_sql, lock_engine_recovering_poison, + require_read_only_sql, }; use crate::error::{ErrorCode, McpError}; use crate::export::{ExportOptions, export_to_file}; @@ -152,7 +153,7 @@ present in multiple stores (row multiplication). pub struct QueryDataParams { /// JSON array of objects or CSV text. pub data: String, - /// SQL query to run against the data. Reference the table by + /// Read-only SQL query to run against the data. Reference the table by /// `table_name` (default `data`). pub sql: String, /// Data format: `"json"` or `"csv"`. Auto-detected from the first byte @@ -172,7 +173,7 @@ pub struct QueryDataParams { pub struct QueryFileParams { /// Absolute path to a CSV / JSON / JSONL / Parquet / Arrow IPC file. pub path: String, - /// SQL query to run. Reference the file's rows by `table_name` + /// Read-only SQL query to run. Reference the file's rows by `table_name` /// (default `data`), e.g. `SELECT * FROM data`. pub sql: String, /// Table name the SQL references the file's rows as. Default `data`. @@ -639,7 +640,7 @@ pub struct InspectFileParams { /// Parameters for the `export` tool. #[derive(Debug, Deserialize, JsonSchema)] pub struct ExportParams { - /// SQL query to export (if omitted, exports whole table) + /// Read-only SQL query to export (if omitted, exports whole table) pub sql: Option, /// Table name (used if sql omitted) pub table: Option, @@ -1917,6 +1918,7 @@ impl HyperMcpServer { Parameters(params): Parameters, ) -> Result { let result = self.with_engine(|engine| { + require_read_only_sql("query_data", ¶ms.sql)?; let tname = params.table_name.unwrap_or_else(|| "data".into()); let temp_table = format!("_tmp_{}_{}", tname, rand_suffix()); let fmt = params.format.unwrap_or_else(|| detect_format(¶ms.data)); @@ -1937,8 +1939,11 @@ impl HyperMcpServer { let query_sql = replace_identifier(¶ms.sql, &tname, &temp_table); // Drop the scratch table whether the query succeeds or fails — // propagating the query error with `?` before the drop would - // orphan the temp table (it then surfaces in `describe`). - let query_result = engine.execute_query_to_json(&query_sql); + // orphan the temp table (it then surfaces in `describe`). The + // rewritten SQL is checked again: `table_name` is the caller's, + // and substituting it can change how the statement tokenizes. + let query_result = require_read_only_sql("query_data", &query_sql) + .and_then(|()| engine.execute_query_to_json(&query_sql)); let _ = engine.execute_command(&drop_scratch_table_sql(&temp_table)); let rows = query_result?; @@ -1965,6 +1970,7 @@ impl HyperMcpServer { ) -> Result { let result = self.with_engine(|engine| { crate::attach::validate_input_path(¶ms.path, "data file")?; + require_read_only_sql("query_file", ¶ms.sql)?; let tname = params.table_name.unwrap_or_else(|| "data".into()); let temp_table = format!("_tmp_{}_{}", tname, rand_suffix()); let schema_override = crate::schema::normalize_schema_param(params.schema.as_ref())?; @@ -2000,8 +2006,11 @@ impl HyperMcpServer { let query_sql = replace_identifier(¶ms.sql, &tname, &temp_table); // Drop the scratch table whether the query succeeds or fails — // propagating the query error with `?` before the drop would - // orphan the temp table (it then surfaces in `describe`). - let query_result = engine.execute_query_to_json(&query_sql); + // orphan the temp table (it then surfaces in `describe`). The + // rewritten SQL is checked again: `table_name` is the caller's, + // and substituting it can change how the statement tokenizes. + let query_result = require_read_only_sql("query_file", &query_sql) + .and_then(|()| engine.execute_query_to_json(&query_sql)); let _ = engine.execute_command(&drop_scratch_table_sql(&temp_table)); let rows = query_result?; @@ -2690,12 +2699,7 @@ impl HyperMcpServer { Parameters(params): Parameters, ) -> Result { let result = self.with_engine(|engine| { - if !is_read_only_sql(¶ms.sql) { - return Err(McpError::new( - ErrorCode::SqlError, - "The query tool only accepts read-only SQL (SELECT, WITH, EXPLAIN, SHOW, VALUES). Use the execute tool for DDL/DML.", - )); - } + require_read_only_sql("query", ¶ms.sql)?; // Optional database routing — temporarily redirect search_path // for the duration of this call. Restored on guard drop. let target_db = self.resolve_db(engine, params.database.as_deref(), None, false)?; @@ -5273,13 +5277,10 @@ fn validate_execute_batch(stmts: &[String]) -> Result<(), McpError> { ) .with_suggestion("Remove the empty element or replace it with a real statement.")); } - // Classification is comment-aware and only inspects the leading - // keyword, so it returns a meaningful answer even for input - // that happens to be multi-statement. We don't try to detect - // multi-statement input client-side — Hyper's own - // "Multi-part queries" / SQLSTATE 0A000 error is mapped at - // [error.rs:130-134] to a clear "split into separate array - // elements" suggestion, so the LLM gets the same actionable + // Multi-statement input classifies as `Other` and passes through: + // Hyper's own "Multi-part queries" / SQLSTATE 0A000 error is + // mapped at [error.rs:130-134] to a clear "split into separate + // array elements" suggestion, so the LLM gets the same actionable // hint after one round-trip. match classify_statement(stmt) { StatementKind::ReadOnly => { diff --git a/hyperdb-mcp/src/sql_classify.rs b/hyperdb-mcp/src/sql_classify.rs new file mode 100644 index 00000000..72a58e31 --- /dev/null +++ b/hyperdb-mcp/src/sql_classify.rs @@ -0,0 +1,546 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Comment- and literal-aware classification of SQL text. +//! +//! The read-only tools (`query`, `query_data`, `query_file`, `chart`, +//! `export`, ...) refuse SQL that [`is_read_only_sql`] does not accept, and +//! nothing on the server backs that up: Hyper does not enforce +//! `BEGIN READ ONLY`, and `default_transaction_read_only` cannot be set. So +//! the classifier must see the statement exactly the way Hyper does. +//! +//! The tokenizer below mirrors Hyper's lexer +//! (`hyper/parser/sql/SQLLexer.cpp`) for everything that decides where a +//! token starts and ends: +//! +//! - Block comments do **not** nest: `/* a /* b */` is one comment, and the +//! text after the first `*/` is live SQL. An unterminated one is an error. +//! - `--` comments end at `\r` or `\n`. +//! - `'...'` doubles `''` and keeps `\` literal; `E'...'` also skips the +//! byte after a `\`. A string followed by whitespace holding a newline +//! (comments count as newlines) and another `'` continues the same +//! string, in the same escape mode. (`B'`, `N'`, `X'`, `U&'` and a +//! `UESCAPE 'c'` clause end where a word plus plain strings would, so +//! they need no case of their own.) +//! - `"..."` doubles `""`. +//! - `$` followed by a digit is a parameter. Any other `$` opens a dollar +//! quote whose tag is *everything* up to the next `$` (not just an +//! identifier), closed by the next occurrence of that whole `$tag$`. +//! - Identifiers are ASCII letters, digits and `_` plus any non-ASCII +//! character except Hyper's Unicode whitespace list. +//! +//! On top of the tokens, statements are classified by Hyper's grammar +//! (`hyper/parser/sql/sql.ypp`), measured against hyperd `0.0.26359`: +//! +//! - `WITH ... DELETE` / `INSERT` / `UPDATE` / `MERGE` run the write, so a +//! `WITH` statement takes the kind of its main statement. A CTE cannot be +//! named `SELECT`, `VALUES` or `TABLE` unquoted, so the first of those +//! words at depth 0 is the main statement. A CTE body that is itself a +//! write is refused too, although Hyper rejects writable CTEs today. +//! - `EXPLAIN (ANALYZE) DELETE ...` runs the `DELETE`. The option list +//! accepts any (even quoted) option name, so an `EXPLAIN` with an option +//! list takes the kind of the statement it explains. A plain `EXPLAIN` +//! only plans. +//! - Hyper refuses more than one statement per call on both protocols, but +//! anything after a top-level `;` is classified as [`StatementKind::Other`] +//! anyway rather than relying on that. + +/// Coarse classification of a single SQL statement. +/// +/// Used by the read-only tools through [`is_read_only_sql`], and by the +/// atomic-batch `execute` tool to enforce "a batch must be either all-DDL +/// singletons or all-DML; mixing the two aborts the transaction with +/// SQLSTATE 0A000". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StatementKind { + /// `SELECT` / `VALUES` / `TABLE` (without `INTO`) / `SHOW`, a + /// parenthesized query, a + /// `WITH` whose main statement is one of those, or an `EXPLAIN` that + /// does not run its statement. + ReadOnly, + /// `CREATE` / `DROP` / `ALTER` / `TRUNCATE` / `RENAME` — Hyper auto-commits. + Ddl, + /// `INSERT` / `UPDATE` / `DELETE` / `COPY` / `MERGE` / `SELECT ... INTO`, + /// including behind a `WITH` or an `EXPLAIN (ANALYZE)` — transactional. + Dml, + /// `BEGIN` / `START` / `COMMIT` / `END` / `ROLLBACK` / `ABORT` / + /// `SAVEPOINT` / `RELEASE`. Rejected inside a batch because the + /// `execute` tool already manages the transaction; an explicit + /// COMMIT mid-batch would defeat atomicity. + TransactionControl, + /// Empty or comment-only input, an unrecognized first keyword, an + /// unterminated literal or comment, unbalanced parentheses, or more + /// than one statement. Never read-only; the batch validator passes it + /// through to Hyper, which is the final authority on syntax. + Other, +} + +/// Returns `true` if `sql` is a single read-only statement +/// ([`StatementKind::ReadOnly`]). +/// +/// Comments, string literals and quoted identifiers are tokenized the way +/// Hyper does (see the module docs), so neither `/* x */ DROP TABLE t` +/// nor `WITH x AS (SELECT 1) DELETE FROM t` nor +/// `EXPLAIN (ANALYZE) DELETE FROM t` passes. +#[must_use] +pub fn is_read_only_sql(sql: &str) -> bool { + classify_statement(sql) == StatementKind::ReadOnly +} + +/// Classify `sql`, which should hold one statement (a trailing `;` is +/// fine). See [`StatementKind`] for the kinds and the module docs for the +/// rules. +#[must_use] +pub fn classify_statement(sql: &str) -> StatementKind { + parse(sql).map_or(StatementKind::Other, |items| classify_items(&items)) +} + +/// Strips leading whitespace, line comments (`--`), and block comments +/// (`/* */`) from SQL text. Block comments do not nest, as in Hyper; an +/// unterminated one leaves nothing. +pub(crate) fn strip_leading_sql_comments(sql: &str) -> &str { + let mut s = sql; + loop { + s = s.trim_start(); + if let Some(rest) = s.strip_prefix("--") { + match rest.find(['\n', '\r']) { + Some(pos) => s = &rest[pos..], + None => return "", + } + } else if let Some(rest) = s.strip_prefix("/*") { + match rest.find("*/") { + Some(pos) => s = &rest[pos + 2..], + None => return "", + } + } else { + return s; + } + } +} + +/// A token, with parenthesized groups already nested. +#[derive(Debug)] +enum Item { + /// An unquoted identifier or keyword, ASCII-uppercased. + Word(String), + /// The items between a `(` and its `)`. + Group(Vec), + Semicolon, + /// Anything else: literals, quoted identifiers, operators, numbers. + Other, +} + +const READ_ONLY_KEYWORDS: [&str; 3] = ["SELECT", "VALUES", "TABLE"]; +const DML_KEYWORDS: [&str; 5] = ["INSERT", "UPDATE", "DELETE", "MERGE", "COPY"]; + +/// Deepest parenthesis nesting accepted. Parsing, classifying and dropping +/// the item tree each recurse once per level, so unbounded nesting would +/// overflow the stack and abort the server; deeper input is `Other`. +const MAX_GROUP_DEPTH: usize = 256; +fn classify_items(items: &[Item]) -> StatementKind { + let (statement, rest) = match items.iter().position(|i| matches!(i, Item::Semicolon)) { + Some(pos) => (&items[..pos], &items[pos + 1..]), + None => (items, &[][..]), + }; + if !rest.is_empty() { + return StatementKind::Other; + } + classify_single(statement) +} + +fn classify_single(items: &[Item]) -> StatementKind { + let rest = items.get(1..).unwrap_or_default(); + match items.first() { + // Only a query can start with `(` (`select_with_parens`). + Some(Item::Group(inner)) => classify_items(inner), + Some(Item::Word(word)) => match word.as_str() { + "SELECT" | "VALUES" | "TABLE" => classify_query(rest), + "SHOW" => StatementKind::ReadOnly, + "WITH" => classify_with(rest), + "EXPLAIN" => classify_explain(rest), + "CREATE" | "DROP" | "ALTER" | "TRUNCATE" | "RENAME" => StatementKind::Ddl, + "INSERT" | "UPDATE" | "DELETE" | "COPY" | "MERGE" => StatementKind::Dml, + "BEGIN" | "START" | "COMMIT" | "END" | "ROLLBACK" | "ABORT" | "SAVEPOINT" + | "RELEASE" => StatementKind::TransactionControl, + _ => StatementKind::Other, + }, + _ => StatementKind::Other, + } +} + +/// The items after a query's first keyword. `INTO` is a reserved word, so +/// at depth 0 it can only be `SELECT ... INTO new_table`, which creates a +/// table (Hyper refuses it unless `enable_select_into` is set). +fn classify_query(items: &[Item]) -> StatementKind { + if items + .iter() + .any(|i| matches!(i, Item::Word(w) if w == "INTO")) + { + StatementKind::Dml + } else { + StatementKind::ReadOnly + } +} + +/// The items after `WITH`: the CTE list, then the main statement. +fn classify_with(items: &[Item]) -> StatementKind { + let mut saw_read_only_group = false; + for (index, item) in items.iter().enumerate() { + match item { + Item::Word(word) if READ_ONLY_KEYWORDS.contains(&word.as_str()) => { + return classify_query(&items[index + 1..]); + } + Item::Word(word) if DML_KEYWORDS.contains(&word.as_str()) => { + return StatementKind::Dml; + } + // CTE bodies, column lists and options before the main + // statement, or a parenthesized main query. + Item::Group(inner) => match classify_items(inner) { + StatementKind::ReadOnly => saw_read_only_group = true, + StatementKind::Other => {} + kind => return kind, + }, + _ => {} + } + } + if saw_read_only_group { + StatementKind::ReadOnly + } else { + StatementKind::Other + } +} + +/// The items after `EXPLAIN`. +fn classify_explain(items: &[Item]) -> StatementKind { + match items.first() { + None => StatementKind::Other, + // `EXPLAIN (ANALYZE) ...` executes; any option list is treated so. + Some(Item::Group(_)) => classify_single(&items[1..]), + // Not Hyper syntax, but PostgreSQL's executing form. + Some(first) if is_bare_explain_option(first) => { + let options = items + .iter() + .take_while(|i| is_bare_explain_option(i)) + .count(); + classify_single(&items[options..]) + } + Some(_) => StatementKind::ReadOnly, + } +} + +fn is_bare_explain_option(item: &Item) -> bool { + matches!(item, Item::Word(w) if matches!(w.as_str(), "ANALYZE" | "ANALYSE" | "VERBOSE")) +} + +/// Tokenize `sql` and nest its parentheses. `None` on an unterminated +/// literal, quoted identifier or comment, unbalanced parentheses, or +/// nesting deeper than [`MAX_GROUP_DEPTH`]. +fn parse(sql: &str) -> Option> { + let bytes = sql.as_bytes(); + let mut stack: Vec> = vec![Vec::new()]; + let mut i = 0; + while let Some(&b) = bytes.get(i) { + let item = match b { + b'\t' | b'\n' | 0x0B | 0x0C | b'\r' | b' ' => { + i += 1; + continue; + } + b'-' if bytes.get(i + 1) == Some(&b'-') => { + i = bytes[i + 2..] + .iter() + .position(|&c| c == b'\n' || c == b'\r') + .map_or(bytes.len(), |pos| i + 2 + pos); + continue; + } + b'/' if bytes.get(i + 1) == Some(&b'*') => { + i = i + 2 + find(&bytes[i + 2..], b"*/")? + 2; + continue; + } + b'(' => { + if stack.len() > MAX_GROUP_DEPTH { + return None; + } + stack.push(Vec::new()); + i += 1; + continue; + } + b')' => { + let group = stack.pop()?; + stack.last_mut()?.push(Item::Group(group)); + i += 1; + continue; + } + b';' => { + i += 1; + Item::Semicolon + } + b'\'' => { + i = skip_string(bytes, i + 1, false)?; + Item::Other + } + b'"' => { + i = skip_quoted_identifier(bytes, i + 1)?; + Item::Other + } + b'$' if bytes.get(i + 1).is_some_and(u8::is_ascii_digit) => { + i += 1; + while bytes.get(i).is_some_and(u8::is_ascii_digit) { + i += 1; + } + Item::Other + } + b'$' => { + i = skip_dollar_quote(bytes, i)?; + Item::Other + } + // A client command (`\x`) is one token with its letters. + b'\\' => { + i += 1; + if bytes + .get(i) + .is_some_and(|&c| c.is_ascii_alphanumeric() || c == b'?') + { + i += 1; + while bytes.get(i).is_some_and(u8::is_ascii_alphanumeric) { + i += 1; + } + } + Item::Other + } + b'e' | b'E' if bytes.get(i + 1) == Some(&b'\'') => { + i = skip_string(bytes, i + 2, true)?; + Item::Other + } + _ if b.is_ascii_alphabetic() || b == b'_' => { + let start = i; + i = skip_identifier(sql, i); + Item::Word(sql[start..i].to_ascii_uppercase()) + } + _ if b.is_ascii() => { + i += 1; + Item::Other + } + _ => { + // Every token boundary above is an ASCII byte, so `i` is + // on a char boundary here. + let c = sql[i..].chars().next()?; + if is_unicode_whitespace(c) { + i += c.len_utf8(); + continue; + } + let start = i; + i = skip_identifier(sql, i); + Item::Word(sql[start..i].to_ascii_uppercase()) + } + }; + stack.last_mut()?.push(item); + } + let root = stack.pop()?; + stack.is_empty().then_some(root) +} + +/// Hyper's non-ASCII whitespace (`SQLLexer::getNextSingle`). +fn is_unicode_whitespace(c: char) -> bool { + matches!( + c, + '\u{A0}' | '\u{2000}'..='\u{200F}' | '\u{2028}' | '\u{2029}' | '\u{3000}' | '\u{FEFF}' + ) +} + +/// End of the identifier starting at byte `i`. +fn skip_identifier(sql: &str, mut i: usize) -> usize { + let bytes = sql.as_bytes(); + while let Some(&b) = bytes.get(i) { + if b.is_ascii_alphanumeric() || b == b'_' { + i += 1; + } else if b.is_ascii() { + break; + } else { + match sql[i..].chars().next() { + Some(c) if !is_unicode_whitespace(c) => i += c.len_utf8(), + _ => break, + } + } + } + i +} + +/// Offset of the first `needle` in `haystack`. +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +/// End of a string literal whose body starts at `i` (after the opening +/// `'`), including any continuation. `escapes` is `E'...'`. +fn skip_string(bytes: &[u8], mut i: usize, escapes: bool) -> Option { + loop { + let b = *bytes.get(i)?; + i += 1; + if b == b'\'' { + if bytes.get(i) == Some(&b'\'') { + i += 1; + } else if let Some(next) = string_continuation(bytes, i) { + i = next; + } else { + return Some(i); + } + } else if escapes && b == b'\\' && i < bytes.len() { + i += 1; + } + } +} + +/// `SQLLexer.cpp`'s `checkStringContinuation` (pedantic): if the bytes +/// from `i` are whitespace and comments holding a newline and then `'`, +/// the offset after that `'`. +fn string_continuation(bytes: &[u8], mut i: usize) -> Option { + let mut had_newline = false; + while let Some(&b) = bytes.get(i) { + i += 1; + match b { + b' ' | b'\t' | 0x0C => {} + b'\n' | b'\r' => had_newline = true, + b'\'' => return had_newline.then_some(i), + b'-' if bytes.get(i) == Some(&b'-') => { + i += 1; + while bytes.get(i).is_some_and(|&c| c != b'\n' && c != b'\r') { + i += 1; + } + had_newline = true; + } + b'/' if bytes.get(i) == Some(&b'*') => { + i = i + 1 + find(bytes.get(i + 1..)?, b"*/")? + 2; + had_newline = true; + } + _ => return None, + } + } + None +} + +/// End of a quoted identifier whose body starts at `i`. +fn skip_quoted_identifier(bytes: &[u8], mut i: usize) -> Option { + loop { + let b = *bytes.get(i)?; + i += 1; + if b == b'"' { + if bytes.get(i) == Some(&b'"') { + i += 1; + } else { + return Some(i); + } + } + } +} + +/// End of a dollar quote opening at `i` (on its first `$`). +fn skip_dollar_quote(bytes: &[u8], i: usize) -> Option { + let tag_end = i + 1 + bytes[i + 1..].iter().position(|&c| c == b'$')?; + let marker = &bytes[i..=tag_end]; + let body = tag_end + 1; + Some(body + find(&bytes[body..], marker)? + marker.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_statement_recognizes_each_kind() { + assert_eq!(classify_statement("SELECT 1"), StatementKind::ReadOnly); + assert_eq!( + classify_statement("with x as (..) select * from x"), + StatementKind::ReadOnly + ); + assert_eq!( + classify_statement("CREATE TABLE t (i INT)"), + StatementKind::Ddl + ); + assert_eq!(classify_statement("drop table t"), StatementKind::Ddl); + assert_eq!( + classify_statement("INSERT INTO t VALUES (1)"), + StatementKind::Dml + ); + assert_eq!(classify_statement("update t set i = 2"), StatementKind::Dml); + assert_eq!(classify_statement("delete from t"), StatementKind::Dml); + assert_eq!(classify_statement(""), StatementKind::Other); + } + + #[test] + fn classify_statement_recognizes_transaction_control() { + for kw in [ + "BEGIN", + "Begin transaction", + "START TRANSACTION", + "COMMIT", + "Commit work", + "END", + "ROLLBACK", + "Rollback to savepoint sp1", + "ABORT", + "SAVEPOINT sp1", + "RELEASE SAVEPOINT sp1", + ] { + assert_eq!( + classify_statement(kw), + StatementKind::TransactionControl, + "expected TransactionControl for `{kw}`" + ); + } + } + + #[test] + fn classify_statement_strips_comments() { + assert_eq!( + classify_statement("/* harmless */ DROP TABLE t"), + StatementKind::Ddl + ); + assert_eq!( + classify_statement("-- pretend to be readonly\nINSERT INTO t VALUES (1)"), + StatementKind::Dml + ); + } + + /// The `execute` batch validator now sees the write behind a `WITH` or + /// an `EXPLAIN (ANALYZE)`. + #[test] + fn classify_statement_sees_wrapped_writes_as_dml() { + for sql in [ + "WITH x AS (SELECT 1) DELETE FROM t", + "EXPLAIN (ANALYZE) DELETE FROM t", + "EXPLAIN (ANALYZE) WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x", + ] { + assert_eq!(classify_statement(sql), StatementKind::Dml, "{sql}"); + } + assert_eq!( + classify_statement("EXPLAIN (ANALYZE) TRUNCATE t"), + StatementKind::Ddl + ); + } + + #[test] + fn strip_leading_sql_comments_does_not_nest() { + assert_eq!(strip_leading_sql_comments("/* a /* b */ c */"), "c */"); + assert_eq!(strip_leading_sql_comments("-- x\r\nSELECT"), "SELECT"); + assert_eq!(strip_leading_sql_comments("/* open"), ""); + } + + #[test] + fn deep_nesting_is_other_not_a_stack_overflow() { + let deep = format!("SELECT {}1{}", "(".repeat(100_000), ")".repeat(100_000)); + assert_eq!(classify_statement(&deep), StatementKind::Other); + let depth = MAX_GROUP_DEPTH - 1; + let fine = format!("SELECT {}1{}", "(".repeat(depth), ")".repeat(depth)); + assert_eq!(classify_statement(&fine), StatementKind::ReadOnly); + } + + #[test] + fn dollar_quote_tag_is_everything_up_to_the_next_dollar() { + // Hyper's tag here is `a b`, so the body runs to the next `$a b$`. + assert_eq!( + classify_statement("SELECT $a b$ ) DELETE $a b$"), + StatementKind::ReadOnly + ); + assert_eq!(classify_statement("SELECT $a b$ x"), StatementKind::Other); + } +} diff --git a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs index 7b4cd086..6b47f545 100644 --- a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs +++ b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs @@ -139,15 +139,17 @@ impl TestHarness { } /// Same in-memory MCP harness, but attaches a caller-owned persistent - /// workspace. The parent of the self-child fixture owns that path's RAII - /// directory, so it remains valid for the complete contention scenario. + /// workspace, so a test can seed it with one server and reopen it with + /// another (or, in the self-child fixture, keep it valid for the + /// complete contention scenario). async fn start_at_persistent( persistent_path: PathBuf, + read_only: bool, ) -> Result> { let temp_dir = Arc::new(TempDir::new()?); let (server_io, client_io) = tokio::io::duplex(64 * 1024); let workspace = Some(persistent_path.to_string_lossy().to_string()); - let server = HyperMcpServer::with_no_daemon(workspace, false, true); + let server = HyperMcpServer::with_no_daemon(workspace, read_only, true); let engine_handle = server.engine_handle(); let server_handle = tokio::spawn(async move { @@ -1393,7 +1395,7 @@ async fn run_persistent_lock_mcp_child(workspace: PathBuf) -> TestResult { .canonicalize() .unwrap_or_else(|_| workspace.clone()); let owner = Engine::new_no_daemon(Some(workspace.to_string_lossy().into_owned()))?; - let h = TestHarness::start_at_persistent(workspace).await?; + let h = TestHarness::start_at_persistent(workspace, false).await?; let status = tokio::time::timeout( Duration::from_secs(2), @@ -4006,3 +4008,118 @@ async fn copy_query_preserves_target_and_resolved_database() -> TestResult { ); Ok(()) } + +const GUARDED_TABLE: &str = "\"persistent\".\"public\".\"guarded\""; + +/// Create `persistent.public.guarded` holding two rows in a new workspace +/// under `dir`, using a writable server that is shut down again. +async fn seed_guarded_workspace( + dir: &TempDir, +) -> Result> { + let workspace = dir.path().join("workspace.hyper"); + let h = TestHarness::start_at_persistent(workspace.clone(), false).await?; + for sql in [ + format!("CREATE TABLE {GUARDED_TABLE} (id INT)"), + format!("INSERT INTO {GUARDED_TABLE} VALUES (1), (2)"), + ] { + let r = call_tool(&h.client, "execute", serde_json::json!({ "sql": [sql] })).await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + } + h.shutdown().await?; + Ok(workspace) +} + +/// Every SQL-taking read tool refuses a write, with and without +/// `--read-only`: `query_data`, `query_file` and `export` used to run any +/// statement, and `query` let a write through behind `WITH` or +/// `EXPLAIN (ANALYZE)`. Hyper enforces nothing here, so the table must +/// still hold its rows afterwards. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sql_read_tools_refuse_writes_in_both_modes() -> TestResult { + let dir = TempDir::new()?; + let workspace = seed_guarded_workspace(&dir).await?; + let input = dir.path().join("input.csv"); + std::fs::write(&input, b"a\n1\n")?; + let input = input.to_string_lossy().into_owned(); + let count = format!("SELECT COUNT(*) AS n FROM {GUARDED_TABLE}"); + + let mut failures = Vec::new(); + let mut export_index = 0; + for read_only in [false, true] { + let h = TestHarness::start_at_persistent(workspace.clone(), read_only).await?; + let mut cases = Vec::new(); + for sql in [ + format!("DELETE FROM {GUARDED_TABLE}"), + format!("DROP TABLE {GUARDED_TABLE}"), + ] { + export_index += 1; + let out = dir.path().join(format!("refused-{export_index}.csv")); + cases.push(( + "query_data", + serde_json::json!({ "data": "a\n1\n", "format": "csv", "sql": sql }), + )); + cases.push(( + "query_file", + serde_json::json!({ "path": input, "sql": sql }), + )); + cases.push(( + "export", + serde_json::json!({ "sql": sql, "path": out.to_string_lossy(), "format": "csv" }), + )); + } + for sql in [ + format!("WITH x AS (SELECT 1) DELETE FROM {GUARDED_TABLE}"), + format!("EXPLAIN (ANALYZE) DELETE FROM {GUARDED_TABLE}"), + ] { + cases.push(("query", serde_json::json!({ "sql": sql }))); + } + for (tool, args) in cases { + let case = format!("read_only={read_only} {tool} {}", args["sql"]); + let r = call_tool(&h.client, tool, args).await?; + record_error_contract(&mut failures, &case, &r, "SQL_ERROR"); + } + + // Plain reads keep working, and a plain EXPLAIN only plans. + export_index += 1; + let out = dir.path().join(format!("allowed-{export_index}.csv")); + for (tool, args) in [ + ( + "query_data", + serde_json::json!({ "data": "a\n1\n", "format": "csv", "sql": count }), + ), + ( + "query_file", + serde_json::json!({ "path": input, "sql": count }), + ), + ( + "export", + serde_json::json!({ "sql": count, "path": out.to_string_lossy(), "format": "csv" }), + ), + ( + "query", + serde_json::json!({ "sql": format!("EXPLAIN DELETE FROM {GUARDED_TABLE}") }), + ), + ] { + let case = format!("read_only={read_only} {tool} {}", args["sql"]); + let r = call_tool(&h.client, tool, args).await?; + if is_error(&r) { + failures.push(format!("{case}: must succeed; got {}", all_text(&r))); + } + } + + let rows = + all_text(&call_tool(&h.client, "query", serde_json::json!({ "sql": count })).await?); + if !rows.contains("\"n\":2") && !rows.contains("\"n\": 2") { + failures.push(format!( + "read_only={read_only}: the table must keep its two rows; got {rows}" + )); + } + h.shutdown().await?; + } + assert!( + failures.is_empty(), + "read tools let a write through:\n- {}", + failures.join("\n- ") + ); + Ok(()) +} diff --git a/hyperdb-mcp/tests/export_tests.rs b/hyperdb-mcp/tests/export_tests.rs index 2a58fbb1..0f2855a7 100644 --- a/hyperdb-mcp/tests/export_tests.rs +++ b/hyperdb-mcp/tests/export_tests.rs @@ -1124,3 +1124,86 @@ fn export_hyper_preserves_constraints() { .execute_command("DETACH DATABASE \"verify\"") .unwrap(); } + +/// Export options for `sql` written to `path` as `format`. +fn sql_export(sql: &str, path: &std::path::Path, format: &str) -> ExportOptions { + ExportOptions { + sql: Some(sql.into()), + table: None, + path: path.to_str().unwrap().into(), + format: format.into(), + overwrite: true, + format_options: None, + source_db: None, + } +} + +/// Query mode splices the SQL into `COPY (...) TO `, so SQL that +/// closes the parenthesis itself could name its own target and options. +/// It is refused, and neither target is written. +#[test] +fn export_refuses_sql_that_escapes_the_copy_wrapper() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + for format in ["csv", "iceberg"] { + let path = dir.path().join(format!("export-{format}")); + let escaped = dir.path().join(format!("escaped-{format}.csv")); + let sql = format!( + "SELECT * FROM test_export) TO '{}' WITH (format => 'csv') --", + escaped.display() + ); + let err = export_to_file(&te.engine, &sql_export(&sql, &path, format)) + .err() + .unwrap_or_else(|| panic!("{format}: the escaping export must be refused")); + assert_eq!(err.code, ErrorCode::SqlError, "{format}: {err:?}"); + assert!( + !escaped.exists(), + "{format}: the SQL's own target was written" + ); + assert!(!path.exists(), "{format}: the export target was written"); + } +} + +/// Query mode refuses a write, which Hyper would otherwise run (or reject +/// only by accident of the wrapper). +#[test] +fn export_refuses_write_sql() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("refused.csv"); + for sql in [ + "DELETE FROM test_export", + "WITH x AS (SELECT 1) DELETE FROM test_export", + ] { + let err = export_to_file(&te.engine, &sql_export(sql, &path, "csv")) + .err() + .unwrap_or_else(|| panic!("`{sql}` must be refused")); + assert_eq!(err.code, ErrorCode::SqlError, "{sql}: {err:?}"); + } + let rows = te + .engine + .execute_query_to_json("SELECT COUNT(*) AS n FROM test_export") + .unwrap(); + assert_eq!(rows[0]["n"], 2, "the table must keep its rows"); + assert!(!path.exists()); +} + +/// A query that ends in a `--` comment still exports: the wrapper puts the +/// closing parenthesis on its own line. +#[test] +fn export_query_ending_in_a_line_comment() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + for format in ["csv", "iceberg"] { + let path = dir.path().join(format!("commented-{format}")); + let result = export_to_file( + &te.engine, + &sql_export("SELECT * FROM test_export -- both rows", &path, format), + ) + .unwrap_or_else(|err| panic!("{format}: {err:?}")); + assert_eq!(result.rows, 2, "{format}"); + } +} diff --git a/hyperdb-mcp/tests/read_only_tests.rs b/hyperdb-mcp/tests/read_only_tests.rs index 0e11577a..e8e80f44 100644 --- a/hyperdb-mcp/tests/read_only_tests.rs +++ b/hyperdb-mcp/tests/read_only_tests.rs @@ -3,7 +3,7 @@ //! Tests for the read-only SQL classifier and the read-only server flag. -use hyperdb_mcp::engine::is_read_only_sql; +use hyperdb_mcp::engine::{StatementKind, classify_statement, is_read_only_sql}; use hyperdb_mcp::server::HyperMcpServer; /// Classifier accepts the canonical read-only statement kinds. @@ -57,8 +57,9 @@ fn strips_leading_comments() { // Block comments assert!(is_read_only_sql("/* comment */ SELECT 1")); assert!(is_read_only_sql("/* multi\nline */ SELECT 1")); - // Nested block comments - assert!(is_read_only_sql( + // Block comments do not nest in Hyper: the comment ends at the first + // `*/`, so `still outer */ SELECT 1` is live (and invalid) SQL. + assert!(!is_read_only_sql( "/* outer /* inner */ still outer */ SELECT 1" )); // Mixed @@ -69,6 +70,135 @@ fn strips_leading_comments() { assert!(!is_read_only_sql("-- sneaky\rDROP TABLE t")); } +/// `WITH` takes the kind of its main statement: Hyper runs +/// `WITH ... DELETE` / `INSERT` (measured against hyperd 0.0.26359). +#[test] +fn with_takes_the_kind_of_its_main_statement() { + for sql in [ + "WITH x AS (SELECT 1) DELETE FROM t", + "with x as (select 1) insert into t select * from x", + "WITH x AS (SELECT 1) UPDATE t SET a = 1", + "WITH x AS (SELECT 1) MERGE INTO t USING x ON true WHEN MATCHED THEN DELETE", + "WITH RECURSIVE x(n) AS (SELECT 1) DELETE FROM t", + // `SELECT ... INTO` creates a table (Hyper refuses it by default). + "SELECT * INTO t2 FROM t", + "WITH x AS (SELECT 1) SELECT * INTO t2 FROM x", + "(SELECT 1 INTO t2)", + "SELECT 1 UNION SELECT 2 INTO t2", + // A writable CTE body (Hyper rejects these today). + "WITH d AS (DELETE FROM t RETURNING *) SELECT * FROM d", + "WITH x AS (SELECT 1) (DELETE FROM t)", + ] { + assert!(!is_read_only_sql(sql), "{sql}"); + } + for sql in [ + "WITH x AS (SELECT 1), y AS (SELECT 2) SELECT * FROM x, y", + "WITH x AS (SELECT 1) (SELECT * FROM x)", + "WITH x AS (SELECT 1) (SELECT * FROM x) ORDER BY 1", + "WITH x AS (SELECT 1) VALUES (1)", + "WITH x AS (SELECT 1) TABLE x", + // A depth-0 `INSERT` after the main `SELECT` is not the main statement. + "WITH x AS (SELECT 1) SELECT 'INSERT', \"delete\" FROM x", + // A quoted `"into"` is an identifier, not the keyword. + "SELECT * FROM (SELECT 1) AS \"into\"", + ] { + assert!(is_read_only_sql(sql), "{sql}"); + } +} + +/// `EXPLAIN (ANALYZE) ...` runs its statement in Hyper; a plain `EXPLAIN` +/// only plans. +#[test] +fn explain_with_options_takes_the_kind_of_its_statement() { + for sql in [ + "EXPLAIN (ANALYZE) DELETE FROM t", + "explain (analyze) with x as (select 1) delete from t", + "EXPLAIN (\"analyze\") DELETE FROM t", + "EXPLAIN (FORMAT JSON, ANALYZE) INSERT INTO t VALUES (1)", + "EXPLAIN (ANALYZE) TRUNCATE t", + "EXPLAIN ANALYZE DELETE FROM t", + "EXPLAIN ANALYZE VERBOSE DELETE FROM t", + "EXPLAIN", + ] { + assert!(!is_read_only_sql(sql), "{sql}"); + } + for sql in [ + "EXPLAIN SELECT 1", + "EXPLAIN DELETE FROM t", + "EXPLAIN (ANALYZE) SELECT 1", + "EXPLAIN (ANALYZE) WITH x AS (SELECT 1) SELECT * FROM x", + ] { + assert!(is_read_only_sql(sql), "{sql}"); + } +} + +/// Literals and quoted identifiers hide their contents, tokenized the way +/// Hyper's lexer does. +#[test] +fn literals_hide_their_contents() { + for sql in [ + "SELECT ')' AS a", + "SELECT 'a;b' AS a", + "SELECT 'DELETE' AS a", + "SELECT 'it''s' AS a", + "SELECT 'a\\' AS a", + "SELECT E'a\\'b)' AS a", + "SELECT $$a)b; DELETE$$ AS a", + "SELECT $q$a)b$q$ AS a", + "SELECT \"a)\"\"b\" FROM t", + "SELECT 'a'\n')b' AS a", + "SELECT 'a' -- c\n')b' AS a", + "TABLE t", + "(SELECT 1)", + "(SELECT 1) UNION (SELECT 2)", + "SHOW search_path", + "SELECT 1;", + "SELECT 1; -- done", + "SELECT $1", + "SELECT\u{a0}1", + "\u{feff}SELECT 1", + ] { + assert!(is_read_only_sql(sql), "{sql:?}"); + } + // A plain string keeps `\` literal, so this string ends at the second + // quote and the `)` that follows is live, unbalancing the statement. + assert!(!is_read_only_sql("SELECT 'a\\') AS a")); + // Dollar-quote tags are case-sensitive: `$Q$` does not close `$q$`. + assert!(!is_read_only_sql("SELECT $q$a$Q$ AS a")); + // A string continuation stays in escape mode, so `\'` does not end it. + assert!(is_read_only_sql("SELECT E'a'\n'\\') DELETE' AS a")); + // Without a newline the second `'...'` is a separate (plain) string, + // which ends at `\'`, leaving `) DELETE'` unterminated. + assert!(!is_read_only_sql("SELECT E'a' '\\') DELETE' AS a")); +} + +/// A comment Hyper closes early cannot hide a statement. +#[test] +fn non_nesting_comments_cannot_hide_a_statement() { + assert!(!is_read_only_sql("/* /* */ DELETE FROM t; /* */ SELECT 1")); + assert!(!is_read_only_sql("/* /* */ DELETE FROM t -- */ SELECT 1")); + assert!(is_read_only_sql("/* a /* b */ SELECT 1")); +} + +/// Input Hyper would not parse as one statement is never read-only. +#[test] +fn malformed_or_multiple_statements_are_other() { + for sql in [ + "SELECT 'unterminated", + "SELECT \"unterminated", + "SELECT $$unterminated", + "SELECT $tag", + "SELECT 1 /* unterminated", + "SELECT 1)", + "SELECT (1", + "SELECT 1) TO '/tmp/x' WITH (format => 'csv') --", + "SELECT 1; DELETE FROM t", + "SELECT 1;;", + ] { + assert_eq!(classify_statement(sql), StatementKind::Other, "{sql}"); + } +} + /// Verify a read-only server reports itself as read-only and a writable /// server does not. #[test] From 1659af829e6bf342875e56f2b8d3d8459306fd67 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 14:18:15 -0700 Subject: [PATCH 31/61] fix(mcp)!: export and chart refuse to overwrite by default and never replace what they did not write export defaulted overwrite to true, and with it an iceberg export ran remove_dir_all on whatever directory `path` named while a hyper export deleted and recreated whatever file was there, including the session's persistent database or an attached .hyper file. chart also overwrote by default. BREAKING CHANGE: `overwrite` now defaults to false for export and chart; an existing destination is refused with PERMISSION_DENIED unless the call passes overwrite=true. Even with overwrite, neither tool writes to a database the session has open (persistent, local, or any attached file) or to a directory holding one. Files are compared by identity ((dev, ino) on Unix, volume serial and file index on Windows, in the new file_identity.rs), so symlinks, hard links and `.` paths are caught. iceberg replaces only an empty directory or an Iceberg table, moving the old one aside and putting it back if the export fails; hyper replaces only a Hyper database file and always builds at a sibling temp path that is renamed into place once complete. Export now writes the validated path rather than the caller's raw one, handing hyperd the same `\\?\`-free form ingest uses on Windows, and still reports the caller's path in the response. --- Cargo.lock | 1 + .../specs/2026-10-10-mcp-security-design.md | 38 +- hyperdb-mcp/CHANGELOG.md | 14 + hyperdb-mcp/Cargo.toml | 3 + hyperdb-mcp/README.md | 26 +- hyperdb-mcp/src/attach.rs | 66 ++- hyperdb-mcp/src/chart.rs | 9 +- hyperdb-mcp/src/export.rs | 398 ++++++++++++++---- hyperdb-mcp/src/file_identity.rs | 113 +++++ hyperdb-mcp/src/ingest.rs | 8 +- hyperdb-mcp/src/lib.rs | 1 + hyperdb-mcp/src/readme.rs | 16 +- hyperdb-mcp/src/server.rs | 65 ++- hyperdb-mcp/tests/chart_tests.rs | 5 +- hyperdb-mcp/tests/end_to_end_mcp_tests.rs | 143 +++++++ hyperdb-mcp/tests/export_tests.rs | 374 +++++++++++++--- hyperdb-mcp/tests/integration_tests.rs | 1 + 17 files changed, 1086 insertions(+), 195 deletions(-) create mode 100644 hyperdb-mcp/src/file_identity.rs diff --git a/Cargo.lock b/Cargo.lock index 9ede9d73..51c87ac8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1967,6 +1967,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "windows-sys 0.61.2", ] [[package]] diff --git a/docs/superpowers/specs/2026-10-10-mcp-security-design.md b/docs/superpowers/specs/2026-10-10-mcp-security-design.md index 1efafbd9..ab780817 100644 --- a/docs/superpowers/specs/2026-10-10-mcp-security-design.md +++ b/docs/superpowers/specs/2026-10-10-mcp-security-design.md @@ -168,7 +168,11 @@ export handler then discards its result and passes the raw `params.path` on overwrite=true to replace it" message. Auto-generated chart paths are unique, so only an explicit `output_path` is affected. 2. **Use the validated path.** The path `validate_output_path` returns is the - one that is checked and the one handed to hyperd or the filesystem. + one that is checked and the one handed to hyperd or the filesystem; on + Windows the export gives both the `\\?\`-free, forward-slash form + ingest already uses, since hyperd cannot open an extended-length path. + The response still reports the caller's `path` / `output_path`, not the + canonical form. 3. **Never write over a file the session uses**, for every export format and for chart, even with `overwrite=true`. The server builds the protected set once per call: the persistent database, the ephemeral scratch database, @@ -177,14 +181,20 @@ export handler then discards its result and passes the raw `params.path` on *same file* as a protected one, compared by identity: `(dev, ino)` on Unix, (volume serial number, file index) on Windows. Identity sees through symlinks, hard links and case folding, which a path comparison does not. A - destination that does not exist cannot be one of them. The helper lives in - a self-contained `src/file_identity.rs` (std and `windows-sys` only). + destination that does not exist cannot be one of them. A directory + destination that contains a protected file is refused the same way, and + a protected file whose identity cannot be read (other than `NotFound`) + fails the call with `INTERNAL_ERROR` rather than being skipped. The + helper lives in a self-contained `src/file_identity.rs` (std and + `windows-sys` only). The set is per MCP session: a file another client + attached through the same daemon is not in it (accepted; documented). 4. **Iceberg replaces only an Iceberg table.** With `overwrite=true` and an existing `dest`: - - it must be a directory that is empty, or whose top-level entries are a - subset of the layout Hyper's Iceberg export writes (at least - `metadata/`, holding a `*.metadata.json`; iteration 2 measures the full - set from a real export and pins it). Anything else, including a regular + - it must be a directory that is empty, or whose top-level entries are + only directories named `data/` and `metadata/`, with a + `*.metadata.json` in `metadata/` (measured against hyperd 0.0.26359: an + export writes exactly those two, with and without rows). Anything else, + including a regular file or a directory holding a `.hyper` database, is refused with `INVALID_ARGUMENT`: "Refusing to replace '``': it is not an Iceberg table directory". @@ -199,11 +209,15 @@ export handler then discards its result and passes the raw `params.path` on is complete.** With `overwrite=true` and an existing `path`, refuse unless it is a regular file (not a symlink, not a directory) whose first five bytes are `Hyper` (the file magic; verified against real files: - `48 79 70 65 72 08 00 00 ...`). The export then creates its database at a - sibling temp path `.hyperdb-mcp-tmp--`, fills and detaches - it, and renames it over `path`. On failure the temp file is removed and - `path` was never touched. The existing `is_file_in_use` → - `RESOURCE_BUSY` mapping applies to the final rename. + `48 79 70 65 72 08 00 00 ...`). Every Hyper export, replacing or not, + creates its database at a sibling temp path + `.hyperdb-mcp-tmp--` (hyperd accepts a database path without + a `.hyper` extension), fills and detaches it, and renames it over `path`. + On failure the temp file is removed and `path` was never touched. The + existing `is_file_in_use` → `RESOURCE_BUSY` mapping applies to the final + rename; on Windows a held file may instead fail it with + `ERROR_ACCESS_DENIED`, which stays `PERMISSION_DENIED` (unverified + locally; Windows CI decides). 6. CSV, Parquet and Arrow IPC keep letting hyperd's `COPY ... TO` replace a file when `overwrite=true`; points 1 to 3 cover them. diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 788361c0..353ffd74 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **Trimmed the always-in-context tool descriptions (~26% smaller)** — moved the format-selection and edge-case detail into `get_readme` while keeping the actionable rules inline, cutting the tokens the tool catalog costs per load. +- **BREAKING: `export` and `chart` no longer overwrite by default.** An + existing destination is refused with `PERMISSION_DENIED` unless the call + passes `overwrite: true`. `export` used to replace an existing file or + directory silently; `chart` defaulted `overwrite` to true. ### Fixed @@ -42,6 +46,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/). wrapper.** SQL that closed the parenthesis itself could name its own target path and options; it is now refused, and a query that ends in a `--` comment exports instead of failing with a syntax error. +- **`export` and `chart` no longer destroy the databases the session has + open.** Pointing either at the persistent database, the local database, an + attached `.hyper` file (by any path, including a symlink) or a directory + holding one is refused with `INVALID_ARGUMENT`. `export` with `format: "hyper"` used to delete and + recreate an attached file. +- **`export` with `overwrite: true` replaces only what it could have written.** + `iceberg` used to `remove_dir_all` whatever directory `path` named; it now + replaces only an empty directory or an Iceberg table, and `hyper` only a + Hyper database file. Both build the new export before discarding the old one, so a + failed export leaves the previous table or file in place. - **`query_data` / `query_file` no longer interpolate `table_name` unescaped into the scratch-table `DROP`.** A `table_name` containing a double quote could end the quoted identifier and append further SQL to the cleanup diff --git a/hyperdb-mcp/Cargo.toml b/hyperdb-mcp/Cargo.toml index 4adda1ca..11d536ac 100644 --- a/hyperdb-mcp/Cargo.toml +++ b/hyperdb-mcp/Cargo.toml @@ -49,6 +49,9 @@ dirs = "7" [target.'cfg(unix)'.dependencies] libc = "0.2" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } + [lints] workspace = true diff --git a/hyperdb-mcp/README.md b/hyperdb-mcp/README.md index bd948023..c06c3847 100644 --- a/hyperdb-mcp/README.md +++ b/hyperdb-mcp/README.md @@ -700,13 +700,33 @@ export(sql: 'SELECT ...', path: '~/Desktop/analysis.hyper', format: 'hyper') |-----------|------|----------|-------------| | `sql` | string | no | Read-only query to export (if omitted, exports whole table) | | `table` | string | no | Table name (used if `sql` omitted) | -| `path` | string | yes | Output file path | +| `path` | string | yes | Output file path (a directory for `iceberg`) | | `format` | string | yes | `"csv"`, `"parquet"`, `"iceberg"`, `"arrow_ipc"`, or `"hyper"` | +| `database` | string | no | Source database: `local` (default), `persistent`, or an attached alias | +| `format_options` | object | no | Options passed to hyperd's `COPY ... TO ... WITH (...)`; ignored for `hyper` | +| `overwrite` | bool | no | Replace an existing destination (default `false`) | The `"hyper"` format produces a `.hyper` file that opens directly in **Tableau Desktop**. It does not mutate the source database; it creates or replaces the destination and materializes every user table from the selected source into it. +An existing destination is refused with `PERMISSION_DENIED` unless `overwrite` +is `true`. Even then, `iceberg` replaces only an empty directory or one shaped +like an Iceberg table, and `hyper` only a Hyper database file; anything else is +refused with `INVALID_ARGUMENT`. A failed export leaves the previous contents in +place: the old Iceberg table is set aside while the export runs and put back on +failure, and a Hyper export is built beside the old file and renamed over it +only once complete. If an old Iceberg table cannot be put back, the error names +the sibling `.hyperdb-mcp-old-*` directory that holds it. A server that +stops mid-export leaves that directory, or a Hyper export's +`.hyperdb-mcp-tmp-*` file, behind; neither is deleted automatically. + +`export` and `chart` never write to a database the session has open (the +persistent database, the local database, or an attached file) or to a directory +containing one, whatever path reaches it; that is refused with +`INVALID_ARGUMENT`. "The session" is this MCP connection: a file another client +attached through the same daemon is not covered. + ### Visualization #### `chart` @@ -735,7 +755,7 @@ chart(sql: 'SELECT product, SUM(revenue) as total FROM sales GROUP BY product', | `bins` | int | no | Histogram bins (default 20, clamped 1..500) | | `output_path` | string | no | Destination file; parent directories are created | | `inline` | bool | no | Return image bytes inline (default `true`) | -| `overwrite` | bool | no | Permit replacing `output_path` (default `true`) | +| `overwrite` | bool | no | Permit replacing `output_path` (default `false`) | | `bar_orientation` | string | no | `vertical` (default) or `horizontal`; bars only | | `label_values` | bool | no | Draw each original y scalar beside its bar | | `show_legend` | bool | no | Show series legend (default `true`) | @@ -1080,7 +1100,7 @@ Errors include a machine-readable code and a suggestion: | `CONNECTION_LOST` | `hyperd` crashed or wire protocol desynchronized | Retry — the server tears down the engine and reconnects on the next call | | `INVALID_ARGUMENT` | Bad parameter (relative path, duplicate saved-query name, invalid alias) | Fix the argument per the message | | `EMPTY_DATA` | Input has zero rows or columns | Check the source data | -| `PERMISSION_DENIED` | Filesystem permission denied, or `export` with `overwrite: false` hit an existing file | Choose another path or fix permissions | +| `PERMISSION_DENIED` | Filesystem permission denied, or `export` / `chart` hit an existing file without `overwrite: true` | Choose another path, pass `overwrite: true`, or fix permissions | | `INTERNAL_ERROR` | Unexpected server failure | Retry; report with `hyperdb-mcp doctor` output | Server-returned errors include a machine-readable `code`, a `message`, and a diff --git a/hyperdb-mcp/src/attach.rs b/hyperdb-mcp/src/attach.rs index f119d3bd..ea242c7a 100644 --- a/hyperdb-mcp/src/attach.rs +++ b/hyperdb-mcp/src/attach.rs @@ -33,7 +33,7 @@ use crate::engine::Engine; use crate::error::{ErrorCode, McpError}; use hyperdb_api::escape_sql_path; use serde_json::{Value, json}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Mutex; use std::time::SystemTime; @@ -712,12 +712,15 @@ pub fn validate_input_path(path: &str, kind: &str) -> Result /// Validate a user-supplied output path that may not yet exist. /// /// Must be absolute. If the file exists, behaves like [`validate_input_path`]. -/// Otherwise the parent directory must exist and canonicalize cleanly. +/// Otherwise missing parent directories are created, and the result is the +/// canonical parent joined with the file name. Write to the returned path, +/// not to `path`. /// /// # Errors /// /// Same shape as [`validate_input_path`]; additionally returns -/// [`ErrorCode::InvalidArgument`] if the path has no parent or no file-name. +/// [`ErrorCode::InvalidArgument`] if the path has no parent or no file-name, +/// and [`ErrorCode::InternalError`] if a parent directory cannot be created. pub fn validate_output_path(path: &str, kind: &str) -> Result { let pb = PathBuf::from(path); if !pb.is_absolute() { @@ -767,6 +770,63 @@ pub fn validate_output_path(path: &str, kind: &str) -> Result Ok(canonical_parent.join(file_name)) } +/// Refuse to write to `dest` when it is one of the `protected` files the +/// session has open (its persistent and ephemeral databases and every +/// attached file), whatever path reaches it, or a directory holding one: +/// replacing the directory would take the file with it. +/// +/// A `dest` that does not exist is not one of them, and a protected path +/// that no longer exists protects nothing. +/// +/// # Errors +/// +/// - Returns [`ErrorCode::InvalidArgument`] if `dest` is the same file as a +/// protected one, or a directory containing one. +/// - Returns [`ErrorCode::InternalError`] if the identity of `dest` or of a +/// protected file cannot be read for a reason other than its absence. +pub fn refuse_protected_destination( + dest: &Path, + protected: &[PathBuf], + kind: &str, +) -> Result<(), McpError> { + let unidentified = |path: &Path, e: std::io::Error| { + McpError::new( + ErrorCode::InternalError, + format!("Cannot identify {kind} path '{}': {e}", path.display()), + ) + }; + let dest_id = match crate::file_identity::FileId::of(dest) { + Ok(id) => id, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(unidentified(dest, e)), + }; + let dest_dir = dest.canonicalize().ok().filter(|d| d.is_dir()); + for path in protected { + let same = match crate::file_identity::FileId::of(path) { + Ok(id) => id == dest_id, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(unidentified(path, e)), + }; + let inside = dest_dir + .as_ref() + .is_some_and(|dir| path.canonicalize().is_ok_and(|p| p.starts_with(dir))); + if same || inside { + let what = if same { "is" } else { "contains" }; + return Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "Refusing to write {kind} to '{}': it {what} a database this session has open", + dest.display() + ), + ) + .with_suggestion( + "Choose a path that is not the persistent database or an attached file.", + )); + } + } + Ok(()) +} + /// Validate a `LocalFile` path. Must be absolute, must exist, must /// canonicalize cleanly with no `..` components in the result. Returns /// the canonical path on success. diff --git a/hyperdb-mcp/src/chart.rs b/hyperdb-mcp/src/chart.rs index 49e3dec8..d52f94ad 100644 --- a/hyperdb-mcp/src/chart.rs +++ b/hyperdb-mcp/src/chart.rs @@ -275,16 +275,16 @@ impl ChartDisposition { #[must_use] pub fn resolve_chart_disposition( inline: bool, - output_path: Option<&str>, + output_path: Option<&std::path::Path>, format: ChartFormat, ) -> ChartDisposition { match (inline, output_path) { (true, None) => ChartDisposition::InlineOnly, (true, Some(p)) => ChartDisposition::WriteAndInline { - path: std::path::PathBuf::from(p), + path: p.to_path_buf(), }, (false, Some(p)) => ChartDisposition::WriteOnly { - path: std::path::PathBuf::from(p), + path: p.to_path_buf(), }, (false, None) => ChartDisposition::WriteOnly { path: auto_generated_chart_path(format), @@ -353,7 +353,8 @@ pub fn write_chart_to_disk( )); } - if !overwrite && path.exists() { + // A dangling symlink counts as existing. + if !overwrite && std::fs::symlink_metadata(path).is_ok() { return Err(McpError::new( ErrorCode::PermissionDenied, format!( diff --git a/hyperdb-mcp/src/export.rs b/hyperdb-mcp/src/export.rs index 94916b1f..ff815433 100644 --- a/hyperdb-mcp/src/export.rs +++ b/hyperdb-mcp/src/export.rs @@ -30,6 +30,8 @@ use crate::error::{ErrorCode, McpError}; use crate::stats::{ExportStats, StatsTimer}; use hyperdb_api::{CopyTableReport, escape_sql_path, escape_string_literal}; use serde_json::{Map, Value}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; /// Specifies what to export and where. /// @@ -45,15 +47,20 @@ pub struct ExportOptions { /// Table name — converted to `SELECT * FROM "
"` when `sql` is /// None. Ignored when `format = "hyper"`. pub table: Option, - /// Destination file path. + /// Destination file path. Must be absolute. pub path: String, /// One of `"csv"`, `"parquet"`, `"arrow_ipc"`, `"iceberg"`, or /// `"hyper"`. pub format: String, - /// Whether to overwrite an existing file at `path`. When `false` and + /// Whether to replace an existing destination. When `false` and /// `path` already exists, [`export_to_file`] returns a - /// [`ErrorCode::PermissionDenied`] error without touching the file. + /// [`ErrorCode::PermissionDenied`] error without touching it. When + /// `true`, `iceberg` replaces only an Iceberg table directory and + /// `hyper` only a Hyper database file. pub overwrite: bool, + /// Files the export must never write to, compared by identity: the + /// session's own databases and its attached files. + pub protected_paths: Vec, /// Extra options passed through verbatim into the `WITH (...)` /// clause of hyperd's `COPY TO`. Keys must match hyperd's own /// option names (e.g. `codec`, `rows_per_row_group`, @@ -79,11 +86,25 @@ pub struct ExportResult { pub schema_fidelity: Option, } +impl ExportResult { + /// Report the destination as the caller named it: the export wrote to + /// its validated form, which is the same file. + fn reported_at(mut self, path: &str) -> Self { + path.clone_into(&mut self.stats.output_path); + self + } +} + /// Top-level export dispatcher. Resolves the source SQL, then delegates to /// the format-specific exporter. /// /// # Errors /// +/// - Returns [`ErrorCode::InvalidArgument`] if `opts.path` is relative, +/// has `..` components or is not UTF-8, if it is one of +/// `opts.protected_paths`, or if `opts.overwrite` would replace +/// something other than an Iceberg table (`iceberg`) or a Hyper database +/// file (`hyper`). /// - Returns [`ErrorCode::PermissionDenied`] if `opts.path` already /// exists and `opts.overwrite` is `false`. /// - Returns [`ErrorCode::SqlError`] when neither `opts.sql` nor @@ -112,16 +133,31 @@ pub fn export_to_file(engine: &Engine, opts: &ExportOptions) -> Result Result Result export_csv(engine, &select_sql, &opts.path, extra, &timer), - "parquet" => export_parquet(engine, &select_sql, &opts.path, extra, &timer), - "arrow_ipc" => export_arrow_ipc(engine, &select_sql, &opts.path, extra, &timer), - "iceberg" => export_iceberg( - engine, - &select_sql, - &opts.path, - opts.overwrite, - extra, - &timer, - ), + let result = match opts.format.as_str() { + "csv" => export_csv(engine, &select_sql, path, extra, &timer), + "parquet" => export_parquet(engine, &select_sql, path, extra, &timer), + "arrow_ipc" => export_arrow_ipc(engine, &select_sql, path, extra, &timer), + "iceberg" => export_iceberg(engine, &select_sql, path, extra, &timer), other => Err(McpError::new( ErrorCode::UnsupportedFormat, format!("Unsupported export format: {other}"), )), - } + }; + result.map(|result| result.reported_at(&opts.path)) } /// Validate that an option key like `codec` is a safe identifier. @@ -367,46 +398,41 @@ fn export_arrow_ipc( /// The produced layout round-trips cleanly back through `load_iceberg`. /// /// Caller semantics: -/// - `path` is a *directory* path (not a single file). If a directory -/// or file already exists there and `overwrite` is true, we remove -/// it first — hyperd's `COPY TO` refuses to write into an existing -/// non-empty Iceberg location. +/// - `path` is a *directory* path (not a single file). hyperd's `COPY TO` +/// refuses to write into an existing non-empty Iceberg location, so an +/// existing `path` (which `export_to_file` lets through only with +/// `overwrite`) must be an Iceberg table or empty, and is moved aside +/// while the export runs: a failed export puts it back. fn export_iceberg( engine: &Engine, sql: &str, path: &str, - overwrite: bool, format_options: Option<&Map>, timer: &StatsTimer, ) -> Result { - // Clear the destination if it exists. The outer overwrite guard in - // `export_to_file` has already rejected the call if `overwrite` is - // false and the path exists, so by the time we get here either the - // path is empty or we've been told to replace it. - let dest = std::path::Path::new(path); - if dest.exists() && overwrite { - if dest.is_dir() { - std::fs::remove_dir_all(dest).map_err(|e| { - McpError::new( - ErrorCode::PermissionDenied, - format!("Cannot remove existing Iceberg directory '{path}': {e}"), - ) - })?; - } else { - std::fs::remove_file(dest).map_err(|e| { - McpError::new( - ErrorCode::PermissionDenied, - format!("Cannot remove existing file at '{path}': {e}"), - ) - })?; - } - } - + let dest = Path::new(path); let with_clause = render_copy_with_clause("format => 'iceberg'", format_options)?; let quoted_path = escape_string_literal(path); let copy_sql = copy_to_sql(sql, "ed_path, &with_clause); - let row_count = engine.execute_command(©_sql)?; + let previous = if std::fs::symlink_metadata(dest).is_ok() { + require_iceberg_table_dir(dest)?; + Some(ReplaceGuard::move_aside(dest)?) + } else { + None + }; + let row_count = match engine.execute_command(©_sql) { + Ok(rows) => rows, + Err(e) => { + return Err(match previous { + Some(previous) => previous.restore(e), + None => e, + }); + } + }; + if let Some(previous) = previous { + previous.commit(); + } // Directory size = sum of all file sizes under `path`. Not strictly // required by callers, but useful for telemetry. @@ -446,6 +472,161 @@ fn walk_dir_size(dir: &std::path::Path) -> std::io::Result { Ok(total) } +/// Refuse to replace `dest` unless it is an empty directory or one shaped +/// like hyperd's Iceberg export: top-level entries only `data/` and +/// `metadata/`, with a `*.metadata.json` in `metadata/` (measured against +/// hyperd 0.0.26359, with and without rows). +fn require_iceberg_table_dir(dest: &Path) -> Result<(), McpError> { + if is_iceberg_table_dir(dest).unwrap_or(false) { + return Ok(()); + } + Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "Refusing to replace '{}': it is not an Iceberg table directory", + dest.display() + ), + ) + .with_suggestion("Export to a new directory, or to an existing Iceberg table to replace it.")) +} + +fn is_iceberg_table_dir(dest: &Path) -> std::io::Result { + if !std::fs::symlink_metadata(dest)?.is_dir() { + return Ok(false); + } + let mut has_metadata = false; + for entry in std::fs::read_dir(dest)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + return Ok(false); + } + match entry.file_name().to_str() { + Some("data") => {} + Some("metadata") => { + for file in std::fs::read_dir(entry.path())? { + let file = file?; + if file.file_type()?.is_file() + && file + .file_name() + .to_str() + .is_some_and(|name| name.ends_with(".metadata.json")) + { + has_metadata = true; + } + } + if !has_metadata { + return Ok(false); + } + } + _ => return Ok(false), + } + } + // An empty directory has neither entry, and holds nothing to lose. + Ok(has_metadata || std::fs::read_dir(dest)?.next().is_none()) +} + +/// A sibling of `dest` named `.hyperdb-mcp---`, unique +/// within this process. +fn sibling_path(dest: &Path, tag: &str) -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + let mut name = dest.file_name().unwrap_or_default().to_os_string(); + name.push(format!(".hyperdb-mcp-{tag}-{}-{n}", std::process::id())); + dest.with_file_name(name) +} + +/// The previous contents of an export destination, moved to a sibling +/// backup while the export writes a new one. +/// +/// [`Self::commit`] deletes the backup. [`Self::restore`], or dropping the +/// guard without either, removes whatever the export wrote and moves the +/// backup back. A backup that cannot be restored is left in place and +/// named, never deleted. +struct ReplaceGuard { + dest: PathBuf, + backup: Option, +} + +impl ReplaceGuard { + fn move_aside(dest: &Path) -> Result { + let backup = sibling_path(dest, "old"); + std::fs::rename(dest, &backup).map_err(|e| { + McpError::new( + if is_file_in_use(&e) { + ErrorCode::ResourceBusy + } else { + ErrorCode::PermissionDenied + }, + format!("Cannot move existing '{}' aside: {e}", dest.display()), + ) + })?; + Ok(Self { + dest: dest.to_path_buf(), + backup: Some(backup), + }) + } + + /// The export succeeded: delete the previous contents. + fn commit(mut self) { + if let Some(backup) = self.backup.take() + && let Err(e) = std::fs::remove_dir_all(&backup) + { + tracing::warn!( + backup = %backup.display(), + err = %e, + "failed to delete the replaced export destination", + ); + } + } + + /// The export failed with `err`: put the previous contents back and + /// return `err`, naming the backup if it could not be restored. + fn restore(mut self, err: McpError) -> McpError { + let Some(backup) = self.backup.take() else { + return err; + }; + match restore_backup(&self.dest, &backup) { + Ok(()) => err, + Err(e) => McpError { + message: format!( + "{} (the previous contents could not be restored and are at '{}': {e})", + err.message, + backup.display() + ), + ..err + }, + } + } +} + +impl Drop for ReplaceGuard { + fn drop(&mut self) { + if let Some(backup) = self.backup.take() + && let Err(e) = restore_backup(&self.dest, &backup) + { + tracing::warn!( + backup = %backup.display(), + err = %e, + "failed to restore the previous export destination", + ); + } + } +} + +/// Remove what is at `dest` now (written by the failed export, since the +/// original was moved away) and move `backup` back. Something another +/// process created at `dest` while the export ran would be removed too; +/// that race is accepted. +fn restore_backup(dest: &Path, backup: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(dest) { + Ok(meta) if meta.is_dir() => std::fs::remove_dir_all(dest)?, + Ok(_) => std::fs::remove_file(dest)?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + std::fs::rename(backup, dest) +} + /// True when a filesystem error means "another process is holding this file /// open" rather than "you lack permission to touch it". /// @@ -491,21 +672,35 @@ fn export_hyper( source_db: Option<&str>, timer: &StatsTimer, ) -> Result { - // The target path is a separate file from the primary workspace, - // so OS-level copy/delete on it is fine — the lock conflict only - // affects the workspace hyperd has open. Pre-delete on overwrite - // because `CREATE DATABASE IF NOT EXISTS` would otherwise silently - // attach to the stale contents. - if std::path::Path::new(path).exists() { - std::fs::remove_file(path).map_err(|e| { + // `export_to_file` lets an existing `path` through only with + // `overwrite`; it must then be a Hyper database, and it is not touched + // until the new one is complete. The new database is built at a + // sibling temp path (`CREATE DATABASE` refuses an existing file) and + // renamed over `path`, so a failed export leaves `path` as it was. + let dest = Path::new(path); + if std::fs::symlink_metadata(dest).is_ok() { + require_hyper_file(dest)?; + } + let temp = sibling_path(dest, "tmp"); + let temp_str = temp + .to_str() + .expect("a sibling of a UTF-8 path with an ASCII suffix is UTF-8"); + + let result = build_hyper_export(engine, temp_str, source_db, timer).and_then(|report| { + std::fs::rename(&temp, dest).map_err(|e| { // A file another process is holding open is not a permissions // problem, and "Check file permissions on the source or target // path" — `PermissionDenied`'s guidance — sends the caller // somewhere there is nothing to find. `ResourceBusy` carries // the guidance that actually resolves it (close the holder, or - // copy the file), and this is the *only* place a contended - // export target can surface: on Windows the unlink below is - // what fails, and on Unix it succeeds regardless of holders. + // copy the file). On Unix the rename succeeds regardless of + // holders. On Windows, replacing a file held without + // `FILE_SHARE_DELETE` may instead fail with + // `ERROR_ACCESS_DENIED`, which cannot be told apart from a real + // permission error and so stays `PermissionDenied`. + // + // The rename also replaces a file created at `dest` after + // `export_to_file` found nothing there; that race is accepted. if is_file_in_use(&e) { McpError::new( ErrorCode::ResourceBusy, @@ -516,12 +711,75 @@ fn export_hyper( } else { McpError::new( ErrorCode::PermissionDenied, - format!("Cannot remove existing target '{path}': {e}"), + format!("Cannot move the export into place at '{path}': {e}"), ) } })?; + Ok(report) + }); + let report = match result { + Ok(report) => report, + Err(e) => { + if let Err(remove_err) = std::fs::remove_file(&temp) + && remove_err.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!( + temp = %temp.display(), + err = %remove_err, + "failed to remove the temporary hyper export", + ); + } + return Err(e); + } + }; + let rows = report.rows_copied; + + let file_size = std::fs::metadata(path).map_or(0, |m| m.len()); + + Ok(ExportResult { + rows, + stats: ExportStats { + operation: "export".into(), + rows, + elapsed_ms: timer.elapsed_ms(), + file_size_bytes: file_size, + format: "hyper".into(), + output_path: path.into(), + }, + schema_fidelity: Some(report), + }) +} + +/// Refuse to replace `dest` unless it is a regular file (not a symlink or +/// a directory) that starts with Hyper's file magic. +fn require_hyper_file(dest: &Path) -> Result<(), McpError> { + let is_hyper = std::fs::symlink_metadata(dest).is_ok_and(|m| m.is_file()) && { + let mut magic = [0_u8; 5]; + std::fs::File::open(dest) + .and_then(|mut f| std::io::Read::read_exact(&mut f, &mut magic)) + .is_ok_and(|()| &magic == b"Hyper") + }; + if is_hyper { + return Ok(()); } + Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "Refusing to replace '{}': it is not a Hyper database file", + dest.display() + ), + ) + .with_suggestion("Export to a new path, or to an existing .hyper file to replace it.")) +} +/// Create a database at `path`, fill it with the user tables of +/// `source_db`, and detach it. +fn build_hyper_export( + engine: &Engine, + path: &str, + source_db: Option<&str>, + timer: &StatsTimer, +) -> Result { // Unique alias so we don't collide with a user-issued attach. The // `__export_target_` prefix + PID + nanos makes accidental overlap // exceedingly unlikely and stays within the 63-char identifier cap. @@ -566,23 +824,7 @@ fn export_hyper( ); } - let report = result?; - let rows = report.rows_copied; - - let file_size = std::fs::metadata(path).map_or(0, |m| m.len()); - - Ok(ExportResult { - rows, - stats: ExportStats { - operation: "export".into(), - rows, - elapsed_ms: timer.elapsed_ms(), - file_size_bytes: file_size, - format: "hyper".into(), - output_path: path.into(), - }, - schema_fidelity: Some(report), - }) + result } /// Copy every user table from `source_db` (None → primary) into the diff --git a/hyperdb-mcp/src/file_identity.rs b/hyperdb-mcp/src/file_identity.rs new file mode 100644 index 00000000..5fdbcf51 --- /dev/null +++ b/hyperdb-mcp/src/file_identity.rs @@ -0,0 +1,113 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! File identity: whether two paths name the same file on disk. +//! +//! Comparing paths misses symlinks, hard links and case-folding file +//! systems; comparing identities does not. Unix uses `(st_dev, st_ino)`. +//! Windows uses the volume serial number and file index from +//! `GetFileInformationByHandle`, opening the file with no access rights so +//! that a file `hyperd` holds open exclusively can still be identified. +//! +//! This module depends only on `std` and `windows-sys`, so it can be +//! compiled for Windows on its own. + +use std::io; +use std::path::Path; + +/// The identity of an existing file or directory. Equal identities are the +/// same file, whatever paths reached them. Symlinks are followed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FileId { + volume: u64, + index: u64, +} + +impl FileId { + /// The identity of the file `path` names. + /// + /// # Errors + /// + /// Returns the I/O error from looking the file up, `NotFound` when it + /// does not exist. + #[cfg(unix)] + pub(crate) fn of(path: &Path) -> io::Result { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(path)?; + Ok(Self { + volume: metadata.dev(), + index: metadata.ino(), + }) + } + + /// The identity of the file `path` names. + /// + /// # Errors + /// + /// Returns the I/O error from opening the file, `NotFound` when it does + /// not exist. + #[cfg(windows)] + pub(crate) fn of(path: &Path) -> io::Result { + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, FILE_FLAG_BACKUP_SEMANTICS, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, GetFileInformationByHandle, + }; + + // No access rights: reading metadata then does not conflict with + // another process's share mode. `FILE_FLAG_BACKUP_SEMANTICS` is + // required to open a directory. + let file = std::fs::OpenOptions::new() + .access_mode(0) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path)?; + let mut info = std::mem::MaybeUninit::::uninit(); + // SAFETY: `file` is open for the duration of the call, and `info` + // points to writable storage of the type the call fills. + let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle(), info.as_mut_ptr()) }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the call succeeded, so it initialized `info`. + let info = unsafe { info.assume_init() }; + Ok(Self { + volume: u64::from(info.dwVolumeSerialNumber), + index: (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow), + }) + } +} + +#[cfg(test)] +mod tests { + use super::FileId; + + #[test] + fn same_file_through_another_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let a = dir.path().join("a"); + let b = dir.path().join("b"); + std::fs::write(&a, b"a").expect("write a"); + std::fs::write(&b, b"b").expect("write b"); + let id = FileId::of(&a).expect("id of a"); + assert_eq!(id, FileId::of(&dir.path().join(".").join("a")).expect("id")); + assert_ne!(id, FileId::of(&b).expect("id of b")); + #[cfg(unix)] + { + let link = dir.path().join("link"); + std::os::unix::fs::symlink(&a, &link).expect("symlink"); + assert_eq!(id, FileId::of(&link).expect("id of link")); + let hard = dir.path().join("hard"); + std::fs::hard_link(&a, &hard).expect("hard link"); + assert_eq!(id, FileId::of(&hard).expect("id of hard link")); + } + } + + #[test] + fn missing_file_is_not_found() { + let dir = tempfile::tempdir().expect("tempdir"); + let err = FileId::of(&dir.path().join("missing")).expect_err("missing"); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); + } +} diff --git a/hyperdb-mcp/src/ingest.rs b/hyperdb-mcp/src/ingest.rs index 9a98b22c..05f4466d 100644 --- a/hyperdb-mcp/src/ingest.rs +++ b/hyperdb-mcp/src/ingest.rs @@ -58,7 +58,13 @@ use std::path::{Path, PathBuf}; /// Any canonicalize failure falls back to the original path — the /// common case (absolute, existing file on any platform) still works. fn canonicalize_for_copy(path: &Path) -> PathBuf { - let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); + path_for_hyperd(path.canonicalize().unwrap_or_else(|_| path.to_path_buf())) +} + +/// Steps 1 and 2 of [`canonicalize_for_copy`] for a path that is already +/// canonical: on Windows, strip the extended-length prefix and use forward +/// slashes. Any other path is returned unchanged. +pub(crate) fn path_for_hyperd(canonical: PathBuf) -> PathBuf { #[cfg(windows)] { if let Some(s) = canonical.to_str() { diff --git a/hyperdb-mcp/src/lib.rs b/hyperdb-mcp/src/lib.rs index 77005f14..db6987e2 100644 --- a/hyperdb-mcp/src/lib.rs +++ b/hyperdb-mcp/src/lib.rs @@ -78,6 +78,7 @@ pub mod engine; pub mod error; #[doc(hidden)] pub mod export; +mod file_identity; #[doc(hidden)] pub mod ingest; #[doc(hidden)] diff --git a/hyperdb-mcp/src/readme.rs b/hyperdb-mcp/src/readme.rs index 1241b659..d8e56c42 100644 --- a/hyperdb-mcp/src/readme.rs +++ b/hyperdb-mcp/src/readme.rs @@ -91,8 +91,8 @@ Log bars start at the positive lower bound, not zero. `output_path`, its PNG (default) or SVG is returned `inline` and no file is written. Supplying `output_path` writes the file and still returns it inline; set `inline=false` for disk-only output (an omitted path then gets -an auto-generated temp file). Set `overwrite=false` to refuse an existing -destination. The path extension and explicit `format`, when both supplied, +an auto-generated temp file). An existing file needs `overwrite=true`. +The path extension and explicit `format`, when both supplied, must agree. Use `database` to route the SQL to local, persistent, or an attached database. @@ -194,12 +194,12 @@ interop; `.hyper` export snapshots every table for Tableau. ### Export - `export` — write a table or query result to a file (Parquet, Iceberg, - Arrow IPC, CSV, .hyper). A `.hyper` export leaves the source unchanged - but creates/replaces the destination file and materializes every user - table into it — a faithful backup: NOT NULL, DEFAULT, COLLATE, ASSUMED - PRIMARY KEY, and ASSUMED UNIQUE all survive (Hyper never accepts - enforced PRIMARY KEY / UNIQUE / FOREIGN KEY / CHECK at CREATE TABLE, so - no source table carries those). The response carries a `schema_fidelity` + Arrow IPC, CSV, .hyper). Replaces a file only with `overwrite: true`, + never an open database. A `.hyper` export leaves the source unchanged + and writes every user table to the destination — a faithful backup: + NOT NULL, DEFAULT, COLLATE, ASSUMED PRIMARY KEY, and ASSUMED UNIQUE all + survive (Hyper never accepts enforced PRIMARY KEY / UNIQUE / FOREIGN + KEY / CHECK at CREATE TABLE, so no source table carries those). The response carries a `schema_fidelity` object (`fully_preserved` plus per-class counts and an `unpreserved` list naming each `table` + `column`) — check it before trusting an export as a backup. diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index d6b73133..e02dc71b 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -55,6 +55,7 @@ use serde::Deserialize; use serde_json::{Value, json}; use sqlformat::{FormatOptions, Indent, QueryParams as SqlQueryParams}; use std::fmt::Write as _; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; /// Number of rows returned by the `hyper://tables/{name}/sample` JSON @@ -571,9 +572,10 @@ pub struct ChartParams { /// image is returned inline. Defaults to true so MCP clients can /// display the chart without a separate file-read round-trip. pub inline: Option, - /// When false, refuse to overwrite an existing file at `output_path` - /// and return `PERMISSION_DENIED` without touching it. Defaults to - /// true (overwrite silently), matching the `export` tool. + /// Whether to replace an existing file at `output_path`. Defaults to + /// false: an existing file is refused with `PERMISSION_DENIED`; pass + /// true to replace it. A database this session has open is never + /// replaced. pub overwrite: Option, /// Target database alias for unqualified name resolution in the /// chart's SQL. Omit to query the local database. Pass @@ -651,9 +653,11 @@ pub struct ExportParams { /// root with a `metadata/` and `data/` subdir); for all other /// formats it is a single file. pub format: String, - /// If false, refuse to overwrite an existing file at `path` and return - /// a `PERMISSION_DENIED` error instead. Defaults to true (overwrite - /// silently) to match pre-flag behavior. + /// Whether to replace an existing destination. Defaults to false: an + /// existing destination is refused with `PERMISSION_DENIED`; pass true + /// to replace it. `iceberg` replaces only an Iceberg table directory, + /// `hyper` only a `.hyper` file, and a database this session has open + /// is never replaced. pub overwrite: Option, /// Optional per-format options passed through into hyperd's `COPY /// (query) TO '…' WITH (…)` clause. Keys must match hyperd's own @@ -1226,6 +1230,18 @@ impl HyperMcpServer { Arc::clone(&self.attachments) } + /// The files `export` and `chart` must never write to: the ephemeral + /// and persistent databases and every attached file. + fn protected_paths(&self, engine: &Engine) -> Vec { + let mut paths = vec![engine.ephemeral_path().to_path_buf()]; + paths.extend(engine.persistent_path().map(Path::to_path_buf)); + paths.extend(self.attachments.list().into_iter().map(|entry| { + let AttachSource::LocalFile { path } = entry.source; + path + })); + paths + } + /// Whether the server is running in read-only mode. #[must_use] pub fn is_read_only(&self) -> bool { @@ -2948,11 +2964,21 @@ impl HyperMcpServer { )); } - // If the caller passed an explicit output path, validate it. - // Auto-generated paths land in a temp dir and don't need this gate. - if let Some(out) = params.output_path.as_deref() { - crate::attach::validate_output_path(out, "chart output")?; - } + // If the caller passed an explicit output path, validate it and + // write to the validated path. Auto-generated paths land in a + // temp dir and don't need this gate. + let output_path = match params.output_path.as_deref() { + Some(out) => { + let path = crate::attach::validate_output_path(out, "chart output")?; + crate::attach::refuse_protected_destination( + &path, + &self.protected_paths(engine), + "chart", + )?; + Some(path) + } + None => None, + }; // Resolve format up front — the path extension may imply it, // and we need the format before we can auto-generate a path. let format = crate::chart::resolve_chart_format( @@ -3040,10 +3066,10 @@ impl HyperMcpServer { // tool error instead of a half-delivered response. let disposition = crate::chart::resolve_chart_disposition( params.inline.unwrap_or(true), - params.output_path.as_deref(), + output_path.as_deref(), opts.format, ); - let overwrite = params.overwrite.unwrap_or(true); + let overwrite = params.overwrite.unwrap_or(false); if let Some(path) = disposition.path() { crate::chart::write_chart_to_disk(path, &chart.bytes, overwrite)?; } @@ -3059,7 +3085,12 @@ impl HyperMcpServer { ChartFormat::Svg => "svg", }; let wants_inline = disposition.wants_inline(); - let output_path_str = disposition.path().map(|p| p.to_string_lossy().into_owned()); + // Report the path as the caller gave it; the write went to + // its validated form, the same file. + let output_path_str = params + .output_path + .clone() + .or_else(|| disposition.path().map(|p| p.to_string_lossy().into_owned())); let mut stats = serde_json::Map::new(); stats.insert("operation".into(), json!("chart")); @@ -3261,9 +3292,6 @@ impl HyperMcpServer { Parameters(params): Parameters, ) -> Result { let result = self.with_engine(|engine| { - // Validate output path: must be absolute, no `..` components. - // (Iceberg "exports" to a directory; the same rules apply.) - crate::attach::validate_output_path(¶ms.path, "export")?; // `format_options` must be a JSON object if supplied. Anything // else (array, string, number, null) is a caller error — reject // with a clear message rather than silently dropping it. @@ -3312,7 +3340,8 @@ impl HyperMcpServer { table: effective_table, path: params.path, format: params.format, - overwrite: params.overwrite.unwrap_or(true), + overwrite: params.overwrite.unwrap_or(false), + protected_paths: self.protected_paths(engine), format_options, source_db: target_db.clone(), }; diff --git a/hyperdb-mcp/tests/chart_tests.rs b/hyperdb-mcp/tests/chart_tests.rs index b62c19f1..62ea2e3b 100644 --- a/hyperdb-mcp/tests/chart_tests.rs +++ b/hyperdb-mcp/tests/chart_tests.rs @@ -12,6 +12,7 @@ use hyperdb_mcp::chart::{ }; use hyperdb_mcp::error::ErrorCode; use serde_json::json; +use std::path::Path; fn bar_opts() -> ChartOptions { ChartOptions { @@ -773,7 +774,7 @@ fn disposition_inline_only() { /// `inline=true` + `output_path` → both. #[test] fn disposition_inline_and_write() { - let d = resolve_chart_disposition(true, Some("/tmp/x.png"), ChartFormat::Png); + let d = resolve_chart_disposition(true, Some(Path::new("/tmp/x.png")), ChartFormat::Png); assert!(matches!(d, ChartDisposition::WriteAndInline { .. })); assert!(d.wants_inline()); assert_eq!(d.path().unwrap(), std::path::Path::new("/tmp/x.png")); @@ -782,7 +783,7 @@ fn disposition_inline_and_write() { /// `inline=false` + `output_path` → write only. #[test] fn disposition_write_only_explicit_path() { - let d = resolve_chart_disposition(false, Some("/tmp/x.svg"), ChartFormat::Svg); + let d = resolve_chart_disposition(false, Some(Path::new("/tmp/x.svg")), ChartFormat::Svg); assert!(matches!(d, ChartDisposition::WriteOnly { .. })); assert!(!d.wants_inline()); assert_eq!(d.path().unwrap(), std::path::Path::new("/tmp/x.svg")); diff --git a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs index 6b47f545..012d38d1 100644 --- a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs +++ b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs @@ -4123,3 +4123,146 @@ async fn sql_read_tools_refuse_writes_in_both_modes() -> TestResult { ); Ok(()) } + +/// `export` and `chart` never write over a database the session has open, +/// even with `overwrite=true` and through a symlink: the persistent +/// database and an attached file are refused with `INVALID_ARGUMENT`, and +/// both keep their rows. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn export_and_chart_never_replace_session_databases() -> TestResult { + let dir = TempDir::new()?; + let workspace = seed_guarded_workspace(&dir).await?; + let h = TestHarness::start_at_persistent(workspace.clone(), false).await?; + let attached = dir.path().join("attached.hyper"); + let r = call_tool( + &h.client, + "attach_database", + serde_json::json!({ + "alias": "user_db", + "kind": "local_file", + "path": attached.to_string_lossy(), + "writable": true, + "on_missing": "create", + }), + ) + .await?; + assert!(!is_error(&r), "attach failed: {:?}", first_text(&r)); + for sql in [ + "CREATE TABLE \"user_db\".\"public\".\"kept\" (id INT)", + "INSERT INTO \"user_db\".\"public\".\"kept\" VALUES (1), (2)", + ] { + let r = call_tool(&h.client, "execute", serde_json::json!({ "sql": [sql] })).await?; + assert!(!is_error(&r), "seed attached failed: {:?}", first_text(&r)); + } + + let mut targets = vec![workspace.clone(), attached.clone()]; + #[cfg(unix)] + { + let link = dir.path().join("link.hyper"); + std::os::unix::fs::symlink(&workspace, &link)?; + targets.push(link); + } + + let mut failures = Vec::new(); + for target in &targets { + let path = target.to_string_lossy(); + for (tool, args) in [ + ( + "export", + serde_json::json!({ "table": "guarded", "database": "persistent", + "path": path, "format": "csv", "overwrite": true }), + ), + ( + "export", + serde_json::json!({ "path": path, "format": "hyper", "overwrite": true }), + ), + ( + "export", + serde_json::json!({ "path": path, "format": "hyper" }), + ), + ( + "chart", + serde_json::json!({ "sql": "SELECT 'a' AS x, 1 AS y", "chart_type": "bar", + "x": "x", "y": "y", "output_path": path, "overwrite": true }), + ), + ] { + let case = format!("{tool} {args}"); + let r = call_tool(&h.client, tool, args).await?; + record_error_contract(&mut failures, &case, &r, "INVALID_ARGUMENT"); + if !all_text(&r).contains("has open") { + failures.push(format!( + "{case}: must name the open database; got {}", + all_text(&r) + )); + } + } + } + + for (sql, table) in [ + ( + format!("SELECT COUNT(*) AS n FROM {GUARDED_TABLE}"), + "persistent", + ), + ( + "SELECT COUNT(*) AS n FROM \"user_db\".\"public\".\"kept\"".to_owned(), + "attached", + ), + ] { + let rows = + all_text(&call_tool(&h.client, "query", serde_json::json!({ "sql": sql })).await?); + if !rows.contains("\"n\":2") && !rows.contains("\"n\": 2") { + failures.push(format!( + "the {table} table must keep its two rows; got {rows}" + )); + } + } + h.shutdown().await?; + assert!( + failures.is_empty(), + "a session database was not protected:\n- {}", + failures.join("\n- ") + ); + Ok(()) +} + +/// `overwrite` defaults to false for `export` and `chart`: an existing file +/// is refused with `PERMISSION_DENIED` and left as it was. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn export_and_chart_refuse_an_existing_file_by_default() -> TestResult { + let h = TestHarness::start(false, false).await?; + let dir = TempDir::new()?; + let csv = dir.path().join("existing.csv"); + let png = dir.path().join("existing.png"); + std::fs::write(&csv, b"keep")?; + std::fs::write(&png, b"keep")?; + + let mut failures = Vec::new(); + for (tool, args) in [ + ( + "export", + serde_json::json!({ "sql": "SELECT 1 AS a", "path": csv.to_string_lossy(), + "format": "csv" }), + ), + ( + "chart", + serde_json::json!({ "sql": "SELECT 'a' AS x, 1 AS y", "chart_type": "bar", + "x": "x", "y": "y", "output_path": png.to_string_lossy() }), + ), + ] { + let case = format!("{tool} {args}"); + let r = call_tool(&h.client, tool, args).await?; + record_error_contract(&mut failures, &case, &r, "PERMISSION_DENIED"); + } + for path in [&csv, &png] { + if std::fs::read(path)? != b"keep" { + failures.push(format!("{} was modified", path.display())); + } + } + h.shutdown().await?; + assert!( + failures.is_empty(), + "an existing file was not refused:\n- {}", + failures.join("\n- ") + ); + Ok(()) +} diff --git a/hyperdb-mcp/tests/export_tests.rs b/hyperdb-mcp/tests/export_tests.rs index 0f2855a7..ba7dfe26 100644 --- a/hyperdb-mcp/tests/export_tests.rs +++ b/hyperdb-mcp/tests/export_tests.rs @@ -42,6 +42,7 @@ fn export_csv_from_table() { path: path_str.into(), format: "csv".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -68,6 +69,7 @@ fn export_csv_from_query() { path: path_str.into(), format: "csv".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -90,6 +92,7 @@ fn export_parquet_from_table() { path: path_str.into(), format: "parquet".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -113,6 +116,7 @@ fn export_hyper_copies_workspace() { path: path_str.into(), format: "hyper".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -154,6 +158,7 @@ fn export_hyper_with_source_db_snapshots_persistent() { path: path_str.into(), format: "hyper".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: Some("persistent".into()), }; @@ -205,6 +210,7 @@ fn export_hyper_requires_no_sql_or_table() { path: path_str.into(), format: "hyper".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -236,6 +242,7 @@ fn export_csv_without_sql_or_table_errors() { path: path_str.into(), format: "csv".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -272,6 +279,7 @@ fn export_overwrite_false_rejects_existing_file() { path: csv_path.to_str().unwrap().into(), format: "csv".into(), overwrite: false, + protected_paths: Vec::new(), format_options: None, source_db: None, }, @@ -303,6 +311,7 @@ fn export_overwrite_false_rejects_existing_file() { path: hyper_path.to_str().unwrap().into(), format: "hyper".into(), overwrite: false, + protected_paths: Vec::new(), format_options: None, source_db: None, }, @@ -331,6 +340,7 @@ fn export_overwrite_false_allows_new_path() { path: path.to_str().unwrap().into(), format: "csv".into(), overwrite: false, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -340,8 +350,7 @@ fn export_overwrite_false_allows_new_path() { assert!(std::fs::metadata(&path).unwrap().len() > 0); } -/// `overwrite = true` replaces an existing file — this is both the default -/// and the long-standing behavior from before the flag existed. +/// `overwrite = true` lets hyperd's `COPY ... TO` replace an existing file. #[test] fn export_overwrite_true_replaces_existing_file() { let te = TestEngine::new_ephemeral(); @@ -356,6 +365,7 @@ fn export_overwrite_true_replaces_existing_file() { path: path.to_str().unwrap().into(), format: "csv".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -370,23 +380,11 @@ fn export_overwrite_true_replaces_existing_file() { assert!(contents.contains("Alice") && contents.contains("Bob")); } -/// Regression test for #277: exporting `format: "hyper"` routes its -/// `CREATE DATABASE` / `ATTACH DATABASE` pair through -/// `Engine::execute_attach_command`, so a lock conflict on the target -/// surfaces as `RESOURCE_BUSY` rather than a generic `SqlError`. -/// -/// `export_hyper` deletes any pre-existing target before `CREATE DATABASE`, -/// so no external lock can be forced through the public API on Unix. The -/// deterministic case is a target already attached under another alias in -/// the same session, and the result differs per platform: -/// -/// - **Unix:** the pre-delete unlinks the open file, and `CREATE DATABASE` -/// fails with SQLSTATE `42P04` ("database already exists"). That does not -/// match `is_resource_busy`, so it stays `SqlError`. -/// - **Windows:** `hyperd` holds the file open, the pre-delete fails with -/// `ERROR_SHARING_VIOLATION`, and the error classifies as `RESOURCE_BUSY`. +/// A `hyper` export never replaces a database the session has attached, +/// even with `overwrite`. The file is compared by identity, so another path +/// to it is refused too, and nothing is left beside it. #[test] -fn export_hyper_over_same_session_attached_target_is_classified_per_platform() { +fn export_hyper_over_an_attached_target_is_refused() { let te = TestEngine::new_ephemeral(); setup_test_table(&te); @@ -394,9 +392,8 @@ fn export_hyper_over_same_session_attached_target_is_classified_per_platform() { let path = dir.path().join("already_attached.hyper"); let path_str = path.to_str().unwrap(); - // Attach the (fresh) target path under an unrelated alias first, - // exactly like a user calling `attach_database` on a file they - // then separately ask to export a hyper snapshot over. + // Attach the target under an unrelated alias first, like a user calling + // `attach_database` on a file they then ask to export a snapshot over. te.engine .execute_command(&format!( "CREATE DATABASE {}", @@ -409,53 +406,31 @@ fn export_hyper_over_same_session_attached_target_is_classified_per_platform() { hyperdb_api::escape_sql_path(path_str) )) .unwrap(); - - let opts = ExportOptions { - sql: None, - table: Some("test_export".into()), - path: path_str.into(), - format: "hyper".into(), - overwrite: true, - format_options: None, - source_db: None, + // Metadata, not contents: on Windows hyperd holds the attached file + // exclusively, so it cannot be read. + let stamp = |p: &std::path::Path| { + let meta = std::fs::metadata(p).unwrap(); + (meta.len(), meta.modified().unwrap()) }; - let err = export_to_file(&te.engine, &opts) - .expect_err("exporting over a path already attached in this session must fail"); - - if cfg!(windows) { - // The pre-delete loses to hyperd's own handle on the attached - // file. That is a holder conflict, not a permissions problem, so - // it must carry the "close the other process / copy the file" - // guidance rather than "check file permissions". - assert_eq!( - err.code, - ErrorCode::ResourceBusy, - "a target held open by another process must classify as \ - RESOURCE_BUSY, not PERMISSION_DENIED: {err:?}" - ); - let guidance = err - .suggestion - .as_deref() - .expect("RESOURCE_BUSY must carry recovery guidance"); - assert!( - guidance.to_lowercase().contains("another process"), - "guidance must point at the holder, not at file permissions: {guidance}" - ); - } else { - // Unix unlinks the open file, so the export reaches its own - // `CREATE DATABASE` and collides with the same-session alias. - assert_eq!( - err.code, - ErrorCode::SqlError, - "a same-session 'database already exists' conflict (42P04) is not a \ - lock conflict and must not be reclassified as RESOURCE_BUSY: {err:?}" - ); - assert!( - err.message.contains("42P04") || err.message.to_lowercase().contains("already exists"), - "expected a duplicate-database error, got: {}", - err.message - ); + let before = stamp(&path); + + for target in [ + path.clone(), + dir.path().join(".").join("already_attached.hyper"), + ] { + let opts = ExportOptions { + format: "hyper".into(), + path: target.to_str().unwrap().into(), + overwrite: true, + protected_paths: vec![path.clone()], + ..ExportOptions::default() + }; + let err = export_to_file(&te.engine, &opts) + .expect_err("exporting over an attached database must fail"); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{err:?}"); } + assert_eq!(stamp(&path), before); + assert_eq!(dir_entries(dir.path()), ["already_attached.hyper"]); } /// Iceberg round-trip: export a table to an Iceberg directory, then read @@ -481,6 +456,7 @@ fn iceberg_export_round_trips_through_load_iceberg() { path: iceberg_str.into(), format: "iceberg".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -576,6 +552,7 @@ fn iceberg_export_overwrite_replaces_directory() { path: iceberg_str.into(), format: "iceberg".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -589,6 +566,7 @@ fn iceberg_export_overwrite_replaces_directory() { path: iceberg_str.into(), format: "iceberg".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -631,6 +609,7 @@ fn iceberg_export_refuses_overwrite_when_disabled() { path: iceberg_str.into(), format: "iceberg".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -643,6 +622,7 @@ fn iceberg_export_refuses_overwrite_when_disabled() { path: iceberg_str.into(), format: "iceberg".into(), overwrite: false, + protected_paths: Vec::new(), format_options: None, source_db: None, }; @@ -697,6 +677,7 @@ fn parquet_export_round_trips_through_load_file() { path: path_str.into(), format: "parquet".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }, @@ -792,6 +773,7 @@ fn arrow_ipc_export_round_trips_through_load_file() { path: path_str.into(), format: "arrow_ipc".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }, @@ -871,6 +853,7 @@ fn parquet_export_honors_compression_override() { path: path.to_str().unwrap().into(), format: "parquet".into(), overwrite: true, + protected_paths: Vec::new(), format_options: Some(opts), source_db: None, }, @@ -926,6 +909,7 @@ fn csv_export_honors_delimiter_override() { path: path_str.into(), format: "csv".into(), overwrite: true, + protected_paths: Vec::new(), format_options: Some(opts), source_db: None, }, @@ -971,6 +955,7 @@ fn format_options_invalid_shapes_reject_cleanly() { path: path_str.clone(), format: "parquet".into(), overwrite: true, + protected_paths: Vec::new(), format_options: Some(null_val), source_db: None, }, @@ -993,6 +978,7 @@ fn format_options_invalid_shapes_reject_cleanly() { path: path_str.clone(), format: "parquet".into(), overwrite: true, + protected_paths: Vec::new(), format_options: Some(bad_key), source_db: None, }, @@ -1042,6 +1028,7 @@ fn export_hyper_preserves_constraints() { path: path_str.into(), format: "hyper".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }, @@ -1133,6 +1120,7 @@ fn sql_export(sql: &str, path: &std::path::Path, format: &str) -> ExportOptions path: path.to_str().unwrap().into(), format: format.into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, } @@ -1207,3 +1195,257 @@ fn export_query_ending_in_a_line_comment() { assert_eq!(result.rows, 2, "{format}"); } } + +/// The sorted names in `dir`. +fn dir_entries(dir: &std::path::Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +/// Every file under `dir` with its contents, sorted, for byte-identity +/// checks across a failed export. +fn tree_snapshot(dir: &std::path::Path) -> Vec<(String, Vec)> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(p) = stack.pop() { + for entry in std::fs::read_dir(&p).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + stack.push(entry.path()); + } else { + let rel = entry + .path() + .strip_prefix(dir) + .unwrap() + .to_string_lossy() + .into_owned(); + out.push((rel, std::fs::read(entry.path()).unwrap())); + } + } + } + out.sort(); + out +} + +/// `overwrite=true` replaces only an Iceberg table: a directory holding +/// anything else, a `.hyper` file among it, or a regular file, is refused +/// and left as it was. +#[test] +fn iceberg_export_refuses_to_replace_what_is_not_an_iceberg_table() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + + let sentinel = dir.path().join("documents"); + std::fs::create_dir(&sentinel).unwrap(); + std::fs::write(sentinel.join("notes.txt"), b"keep me").unwrap(); + let with_hyper = dir.path().join("databases"); + std::fs::create_dir_all(with_hyper.join("metadata")).unwrap(); + std::fs::write(with_hyper.join("metadata/v1.metadata.json"), b"{}").unwrap(); + std::fs::write(with_hyper.join("work.hyper"), b"Hyper\x08\0\0").unwrap(); + let file = dir.path().join("plain-file"); + std::fs::write(&file, b"keep me").unwrap(); + + for target in [&sentinel, &with_hyper] { + let before = tree_snapshot(target); + let err = export_to_file( + &te.engine, + &sql_export("SELECT * FROM test_export", target, "iceberg"), + ) + .expect_err("a non-Iceberg directory must not be replaced"); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{err:?}"); + assert!( + err.message.contains("not an Iceberg table"), + "{}", + err.message + ); + assert_eq!(tree_snapshot(target), before); + } + let err = export_to_file( + &te.engine, + &sql_export("SELECT * FROM test_export", &file, "iceberg"), + ) + .expect_err("a regular file must not be replaced by an Iceberg export"); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{err:?}"); + assert_eq!(std::fs::read(&file).unwrap(), b"keep me"); + assert_eq!( + dir_entries(dir.path()), + ["databases", "documents", "plain-file"] + ); +} + +/// An Iceberg replace that fails puts the previous table back byte for +/// byte and leaves no backup beside it. The cast fails while `COPY` runs, +/// after the query passed its parse check. +#[test] +fn failed_iceberg_replace_restores_the_previous_table() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("table"); + export_to_file( + &te.engine, + &sql_export("SELECT * FROM test_export", &target, "iceberg"), + ) + .unwrap(); + let before = tree_snapshot(&target); + + let err = export_to_file( + &te.engine, + &sql_export( + "SELECT CAST(name AS INT) AS n FROM test_export", + &target, + "iceberg", + ), + ) + .expect_err("the cast must fail at execution"); + // 22P02 (invalid integer text), hyperd's error rather than a refusal. + assert_eq!(err.code, ErrorCode::SchemaMismatch, "{err:?}"); + assert_eq!(tree_snapshot(&target), before); + assert_eq!(dir_entries(dir.path()), ["table"]); +} + +/// `overwrite=true` replaces only a Hyper database file: anything else is +/// refused and untouched, while a real `.hyper` file is replaced. +#[test] +fn hyper_export_replaces_only_a_hyper_file() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + let not_hyper = dir.path().join("report.hyper"); + std::fs::write(¬_hyper, b"Hype? not a database").unwrap(); + let a_dir = dir.path().join("dir.hyper"); + std::fs::create_dir(&a_dir).unwrap(); + + let hyper_export = |path: &std::path::Path| ExportOptions { + path: path.to_str().unwrap().into(), + format: "hyper".into(), + overwrite: true, + ..ExportOptions::default() + }; + for target in [¬_hyper, &a_dir] { + let err = export_to_file(&te.engine, &hyper_export(target)) + .expect_err("only a Hyper file may be replaced"); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{err:?}"); + assert!( + err.message.contains("not a Hyper database"), + "{}", + err.message + ); + } + assert_eq!(std::fs::read(¬_hyper).unwrap(), b"Hype? not a database"); + + let real = dir.path().join("real.hyper"); + export_to_file(&te.engine, &hyper_export(&real)).unwrap(); + let first = std::fs::read(&real).unwrap(); + te.engine + .execute_command("INSERT INTO test_export VALUES (3, 'Carol', 1.0)") + .unwrap(); + let result = export_to_file(&te.engine, &hyper_export(&real)).unwrap(); + assert_eq!(result.rows, 3); + assert_ne!(std::fs::read(&real).unwrap(), first); + assert_eq!( + dir_entries(dir.path()), + ["dir.hyper", "real.hyper", "report.hyper"] + ); +} + +/// A `hyper` export that fails after creating its database (an unknown +/// `source_db` fails once the target is attached) leaves the previous +/// file byte-identical and no temporary file beside it. +#[test] +fn failed_hyper_replace_leaves_the_previous_file() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("backup.hyper"); + let opts = |source_db: Option<&str>| ExportOptions { + path: target.to_str().unwrap().into(), + format: "hyper".into(), + overwrite: true, + source_db: source_db.map(Into::into), + ..ExportOptions::default() + }; + export_to_file(&te.engine, &opts(None)).unwrap(); + let before = std::fs::read(&target).unwrap(); + + export_to_file(&te.engine, &opts(Some("no_such_db"))) + .expect_err("an unknown source database must fail"); + assert_eq!(std::fs::read(&target).unwrap(), before); + assert_eq!(dir_entries(dir.path()), ["backup.hyper"]); +} + +/// Every format refuses a protected file, even with `overwrite`, whichever +/// path reaches it: the file itself, a path through `.`, or a symlink. +#[test] +fn export_refuses_a_protected_file_by_identity() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + let protected = dir.path().join("session.hyper"); + std::fs::write(&protected, b"Hyper\x08\0\0 session").unwrap(); + let mut targets = vec![ + protected.clone(), + dir.path().join(".").join("session.hyper"), + ]; + #[cfg(unix)] + { + let link = dir.path().join("link.hyper"); + std::os::unix::fs::symlink(&protected, &link).unwrap(); + targets.push(link); + } + + for target in &targets { + for format in ["csv", "parquet", "arrow_ipc", "iceberg", "hyper"] { + for overwrite in [true, false] { + let opts = ExportOptions { + table: Some("test_export".into()), + path: target.to_str().unwrap().into(), + format: format.into(), + overwrite, + protected_paths: vec![protected.clone()], + ..ExportOptions::default() + }; + let err = export_to_file(&te.engine, &opts) + .expect_err("a protected file must never be written"); + assert_eq!( + err.code, + ErrorCode::InvalidArgument, + "{format} overwrite={overwrite} {}: {err:?}", + target.display() + ); + } + } + } + assert_eq!(std::fs::read(&protected).unwrap(), b"Hyper\x08\0\0 session"); +} + +/// A directory holding a protected file is refused too: replacing an +/// Iceberg-shaped directory would delete the file inside it. +#[test] +fn export_refuses_a_directory_holding_a_protected_file() { + let te = TestEngine::new_ephemeral(); + setup_test_table(&te); + let dir = tempfile::tempdir().unwrap(); + let table = dir.path().join("table"); + std::fs::create_dir_all(table.join("metadata")).unwrap(); + std::fs::create_dir_all(table.join("data")).unwrap(); + std::fs::write(table.join("metadata").join("v1.metadata.json"), b"{}").unwrap(); + let protected = table.join("data").join("session.hyper"); + std::fs::write(&protected, b"Hyper\x08\0\0 session").unwrap(); + let before = tree_snapshot(&table); + + let opts = ExportOptions { + protected_paths: vec![protected], + ..sql_export("SELECT * FROM test_export", &table, "iceberg") + }; + let err = export_to_file(&te.engine, &opts) + .expect_err("a directory holding a protected file must not be replaced"); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{err:?}"); + assert_eq!(tree_snapshot(&table), before); + assert_eq!(dir_entries(dir.path()), ["table"]); +} diff --git a/hyperdb-mcp/tests/integration_tests.rs b/hyperdb-mcp/tests/integration_tests.rs index 2d0ed08e..b7379369 100644 --- a/hyperdb-mcp/tests/integration_tests.rs +++ b/hyperdb-mcp/tests/integration_tests.rs @@ -79,6 +79,7 @@ fn full_pipeline_csv_ingest_and_export() { path: export_path_str.into(), format: "csv".into(), overwrite: true, + protected_paths: Vec::new(), format_options: None, source_db: None, }; From 82c6328e4897113238284df1b42bbbe8f2c1cb54 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 14:37:28 -0700 Subject: [PATCH 32/61] fix(core): scope the AsyncWriteExt import to the Unix cancel path It is only used by the Unix-domain-socket branch of AsyncConnection::cancel, so the Windows build warned about an unused import. --- hyperdb-api-core/src/client/async_client.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hyperdb-api-core/src/client/async_client.rs b/hyperdb-api-core/src/client/async_client.rs index be2d4422..75570f8c 100644 --- a/hyperdb-api-core/src/client/async_client.rs +++ b/hyperdb-api-core/src/client/async_client.rs @@ -456,7 +456,6 @@ impl AsyncClient { pub async fn cancel(&self) -> Result<()> { use crate::protocol::message::frontend; use bytes::BytesMut; - use tokio::io::AsyncWriteExt; info!( target: "hyperdb_api", @@ -478,6 +477,8 @@ impl AsyncClient { } #[cfg(unix)] ConnectionEndpoint::DomainSocket { directory, name } => { + use tokio::io::AsyncWriteExt; + let socket_path = directory.join(name); let mut stream = UnixStream::connect(&socket_path).await.map_err(|e| { warn!( From 9748e7744d02f136e9e29400c87474aca02cdc9f Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 14:37:28 -0700 Subject: [PATCH 33/61] feat(mcp): per-user control transport and daemon lock Add the building blocks that replace the unauthenticated loopback health port: a control endpoint only the current user can open (Unix socket 0600 / Windows named pipe with a user-SID DACL and first-instance flag) and a single-instance daemon lock. Not wired into the daemon yet. --- hyperdb-mcp/Cargo.toml | 2 +- hyperdb-mcp/src/daemon/control.rs | 1207 +++++++++++++++++++++++++++++ hyperdb-mcp/src/daemon/lock.rs | 184 +++++ hyperdb-mcp/src/daemon/mod.rs | 2 + 4 files changed, 1394 insertions(+), 1 deletion(-) create mode 100644 hyperdb-mcp/src/daemon/control.rs create mode 100644 hyperdb-mcp/src/daemon/lock.rs diff --git a/hyperdb-mcp/Cargo.toml b/hyperdb-mcp/Cargo.toml index 11d536ac..de10e30e 100644 --- a/hyperdb-mcp/Cargo.toml +++ b/hyperdb-mcp/Cargo.toml @@ -50,7 +50,7 @@ dirs = "7" libc = "0.2" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_IO", "Win32_System_Memory", "Win32_System_Pipes", "Win32_System_Threading"] } [lints] workspace = true diff --git a/hyperdb-mcp/src/daemon/control.rs b/hyperdb-mcp/src/daemon/control.rs new file mode 100644 index 00000000..f42e13dd --- /dev/null +++ b/hyperdb-mcp/src/daemon/control.rs @@ -0,0 +1,1207 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! The per-user control transport of the daemon. +//! +//! The daemon's health protocol (`PING`, `HEARTBEAT`, `STOP`, `STATUS`, ...) +//! used to listen on a loopback TCP port, which any local process of any user +//! could reach. This module replaces the port with an endpoint only the +//! current user can open: +//! +//! - **Unix:** a Unix domain socket at `/daemon.sock`, mode `0600`, +//! inside the already-private state directory. +//! - **Windows:** a named pipe `\\.\pipe\hyperdb-mcp--` whose +//! DACL grants access to the current user's SID only, that rejects remote +//! clients, and whose first instance is created with +//! `FILE_FLAG_FIRST_PIPE_INSTANCE` so nothing can squat on the name. +//! +//! The endpoint is recorded in the discovery record as a plain string +//! ([`HealthEndpoint`]). Both transports expose the same blocking +//! [`ControlStream`] (`Read + Write` with a per-operation timeout) and the +//! same bounded-wait accept ([`ControlListener::accept_timeout`]), so the +//! accept loop in the health module keeps its shape. +//! +//! A known limitation on Windows: a daemon started from an elevated prompt +//! may create a pipe that clients running without elevation cannot open. +//! +//! This module depends only on `std`, `libc` and `windows-sys`, so it can be +//! compiled for Windows on its own. + +use std::fmt; +use std::io::{self, Read, Write}; +use std::path::Path; +use std::time::Duration; + +/// How long a handler waits for one read or write on a connection. +pub const CONNECTION_IO_TIMEOUT: Duration = Duration::from_secs(5); + +/// Where a daemon's control endpoint lives: a socket path on Unix, a pipe name +/// on Windows. This is the string stored in the discovery record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HealthEndpoint(String); + +impl HealthEndpoint { + /// The endpoint a new daemon should bind, for the given state directory. + /// + /// # Errors + /// + /// Fails when the endpoint cannot be represented: on Unix, a state + /// directory path that is not valid UTF-8 or leaves a socket path longer + /// than the platform allows (the message names `HYPERDB_STATE_DIR`). + pub fn for_new_daemon(state_dir: &Path) -> io::Result { + platform::endpoint_for_new_daemon(state_dir).map(Self) + } + + /// Parse the endpoint string of a discovery record found in `state_dir`. + /// `None` when it is not exactly an endpoint a daemon of this state + /// directory could have produced, so a tampered or foreign record (say, a + /// path to some other program's socket) is never connected to. + #[must_use] + pub fn from_record(record: &str, state_dir: &Path) -> Option { + platform::is_valid_endpoint(record, state_dir).then(|| Self(record.to_owned())) + } + + /// The string form, for the discovery record. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for HealthEndpoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// The server side of the control transport. +#[derive(Debug)] +pub struct ControlListener { + inner: platform::Listener, +} + +impl ControlListener { + /// Bind the endpoint. Fails when something live already serves it. + /// + /// On Unix a leftover socket file from a dead daemon is removed first, but + /// only after a connect to it is refused and only if it really is a + /// socket; the caller must hold the [`super::lock::DaemonLock`]. + /// + /// # Errors + /// + /// Fails if the endpoint is in use, is not a socket, or cannot be created + /// or restricted to the current user. + pub fn bind(endpoint: &HealthEndpoint) -> io::Result { + platform::Listener::bind(endpoint.as_str()).map(|inner| Self { inner }) + } + + /// Wait up to `timeout` for a client. `Ok(None)` on timeout (or a + /// spurious wakeup), so the caller can recheck its shutdown flag. + /// + /// # Errors + /// + /// Fails when the listener itself is broken. + pub fn accept_timeout(&mut self, timeout: Duration) -> io::Result> { + self.inner + .accept_timeout(timeout) + .map(|stream| stream.map(|inner| ControlStream { inner })) + } + + /// Nudge a listener blocked in [`Self::accept_timeout`] by connecting to + /// it and hanging up. Best effort: the accept loop also wakes on its own + /// cadence. + pub fn wake(endpoint: &HealthEndpoint) { + drop(connect(endpoint, Duration::from_millis(250))); + } +} + +/// One accepted or connected control connection. +#[derive(Debug)] +pub struct ControlStream { + inner: platform::Stream, +} + +impl ControlStream { + /// Bound every later read and write on this connection by `timeout`. + /// + /// # Errors + /// + /// Fails if the timeout cannot be applied or is zero. + pub fn set_io_timeout(&mut self, timeout: Duration) -> io::Result<()> { + self.inner.set_io_timeout(timeout) + } + + /// End the conversation: deliver everything written so far, then close. + /// + /// # Errors + /// + /// Fails if the final flush fails; a peer that already hung up is fine. + pub fn finish(self) -> io::Result<()> { + self.inner.finish() + } +} + +impl Read for ControlStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } +} + +impl Write for ControlStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// Connect to a daemon's control endpoint as a client. +/// +/// `connect_timeout` bounds the wait for a busy Windows pipe; a Unix socket +/// connect completes or fails at once. The returned stream has +/// [`CONNECTION_IO_TIMEOUT`] applied. +/// +/// # Errors +/// +/// `NotFound` / `ConnectionRefused` mean no daemon is serving the endpoint; +/// `TimedOut` means a Windows pipe stayed busy for the whole timeout. +pub fn connect(endpoint: &HealthEndpoint, connect_timeout: Duration) -> io::Result { + let mut stream = ControlStream { + inner: platform::connect(endpoint.as_str(), connect_timeout)?, + }; + stream.set_io_timeout(CONNECTION_IO_TIMEOUT)?; + Ok(stream) +} + +#[cfg(unix)] +mod platform { + use std::io::{self, Read, Write}; + use std::os::fd::AsRawFd; + use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; + use std::os::unix::net::{UnixListener, UnixStream}; + use std::path::{Path, PathBuf}; + use std::time::Duration; + + /// File name of the socket inside the state directory. + const SOCKET_FILE_NAME: &str = "daemon.sock"; + + /// Size of `sockaddr_un.sun_path` on this platform, NUL included. + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + const SUN_PATH_CAPACITY: usize = 104; + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + )))] + const SUN_PATH_CAPACITY: usize = 108; + + pub(super) fn endpoint_for_new_daemon(state_dir: &Path) -> io::Result { + let path = state_dir.join(SOCKET_FILE_NAME); + let path = path.to_str().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "the state directory path {} is not valid UTF-8, so it cannot hold the \ + daemon control socket; set HYPERDB_STATE_DIR to another directory", + state_dir.display() + ), + ) + })?; + check_path_length(path)?; + Ok(path.to_owned()) + } + + pub(super) fn is_valid_endpoint(record: &str, state_dir: &Path) -> bool { + Path::new(record) == state_dir.join(SOCKET_FILE_NAME) && record.len() < SUN_PATH_CAPACITY + } + + fn check_path_length(path: &str) -> io::Result<()> { + let usable = SUN_PATH_CAPACITY - 1; + if path.len() > usable { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "the daemon control socket path {path} is {} bytes, longer than the {usable} \ + a Unix socket path can hold; set HYPERDB_STATE_DIR to a shorter directory", + path.len() + ), + )); + } + Ok(()) + } + + #[derive(Debug)] + pub(super) struct Listener { + socket: UnixListener, + path: PathBuf, + /// `(device, inode)` of the socket file this listener created. + identity: (u64, u64), + } + + impl Listener { + pub(super) fn bind(endpoint: &str) -> io::Result { + check_path_length(endpoint)?; + let path = PathBuf::from(endpoint); + reclaim_stale_socket(&path)?; + let listener = UnixListener::bind(&path)?; + let created = std::fs::symlink_metadata(&path)?; + // Constructed first so a failure below still removes the socket. + let this = Self { + socket: listener, + path, + identity: (created.dev(), created.ino()), + }; + // The state directory is private (0700), which is the real + // protection during the window between bind and this chmod. + std::fs::set_permissions(&this.path, std::fs::Permissions::from_mode(0o600))?; + // Non-blocking so a client that hangs up between the poll and the + // accept cannot park the accept loop. + this.socket.set_nonblocking(true)?; + Ok(this) + } + + pub(super) fn accept_timeout(&mut self, timeout: Duration) -> io::Result> { + let mut fds = libc::pollfd { + fd: self.socket.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // The floor keeps a sub-millisecond timeout from spinning. + let millis = i32::try_from(timeout.as_millis().max(1)).unwrap_or(i32::MAX); + // SAFETY: `fds` is one valid, initialised `pollfd` that outlives + // the call, and the count passed is 1. + let ready = unsafe { libc::poll(&raw mut fds, 1, millis) }; + if ready < 0 { + let error = io::Error::last_os_error(); + return if error.kind() == io::ErrorKind::Interrupted { + Ok(None) + } else { + Err(error) + }; + } + if ready == 0 { + return Ok(None); + } + match self.socket.accept() { + Ok((stream, _addr)) => { + // An accepted socket can inherit the listener's + // non-blocking mode on some platforms. + stream.set_nonblocking(false)?; + Ok(Some(Stream { inner: stream })) + } + // A client that gave up between the poll and the accept, or a + // signal, is not a broken listener. + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock + | io::ErrorKind::Interrupted + | io::ErrorKind::ConnectionAborted + ) => + { + Ok(None) + } + Err(error) => Err(error), + } + } + } + + impl Drop for Listener { + fn drop(&mut self) { + // Best effort: a daemon killed outright leaves the file, and the + // next bind reclaims it. Only unlink the file this listener + // created, so a successor that already reclaimed the path (the + // lock released before this drop) keeps its live socket. + let still_ours = std::fs::symlink_metadata(&self.path) + .is_ok_and(|now| (now.dev(), now.ino()) == self.identity); + if still_ours { + drop(std::fs::remove_file(&self.path)); + } + } + } + + /// Remove a leftover socket file so the path can be bound again. + /// + /// Refuses to touch anything that is not a socket, and refuses to unlink a + /// socket something still accepts on. + fn reclaim_stale_socket(path: &Path) -> io::Result<()> { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + if !metadata.file_type().is_socket() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!( + "{} exists and is not a socket; refusing to replace it", + path.display() + ), + )); + } + match UnixStream::connect(path) { + Ok(_live) => Err(io::Error::new( + io::ErrorKind::AddrInUse, + format!("another daemon is serving {}", path.display()), + )), + Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => { + std::fs::remove_file(path) + } + Err(error) => Err(error), + } + } + + pub(super) fn connect(endpoint: &str, _connect_timeout: Duration) -> io::Result { + UnixStream::connect(endpoint).map(|inner| Stream { inner }) + } + + #[derive(Debug)] + pub(super) struct Stream { + inner: UnixStream, + } + + impl Stream { + pub(super) fn set_io_timeout(&mut self, timeout: Duration) -> io::Result<()> { + self.inner.set_read_timeout(Some(timeout))?; + self.inner.set_write_timeout(Some(timeout)) + } + + pub(super) fn finish(mut self) -> io::Result<()> { + self.inner.flush()?; + match self.inner.shutdown(std::net::Shutdown::Write) { + // The peer read its answer and hung up first: nothing is lost. + Err(error) if error.kind() == io::ErrorKind::NotConnected => Ok(()), + other => other, + } + } + + pub(super) fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } + + pub(super) fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.write(buf) + } + + pub(super) fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } + } +} + +#[cfg(windows)] +mod platform { + use std::fs::OpenOptions; + use std::hash::{BuildHasher, RandomState}; + use std::io; + use std::mem; + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; + use std::path::Path; + use std::ptr; + use std::time::{Duration, Instant}; + + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_FILE_NOT_FOUND, ERROR_INSUFFICIENT_BUFFER, ERROR_IO_PENDING, + ERROR_NO_DATA, ERROR_PIPE_BUSY, ERROR_PIPE_CONNECTED, ERROR_PIPE_NOT_CONNECTED, + GetLastError, LocalFree, WAIT_OBJECT_0, WAIT_TIMEOUT, + }; + use windows_sys::Win32::Security::Authorization::{ + ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, + SDDL_REVISION_1, + }; + use windows_sys::Win32::Security::{ + GetTokenInformation, SECURITY_ATTRIBUTES, TOKEN_QUERY, TOKEN_USER, TokenUser, + }; + use windows_sys::Win32::Storage::FileSystem::{ + FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, FlushFileBuffers, PIPE_ACCESS_DUPLEX, + ReadFile, SECURITY_IDENTIFICATION, WriteFile, + }; + use windows_sys::Win32::System::IO::{CancelIoEx, GetOverlappedResult, OVERLAPPED}; + use windows_sys::Win32::System::Pipes::{ + ConnectNamedPipe, CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, + PIPE_TYPE_BYTE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, + }; + use windows_sys::Win32::System::Threading::{ + CreateEventW, GetCurrentProcess, OpenProcessToken, WaitForSingleObject, + }; + + /// Prefix every endpoint of ours carries; anything else in a record is + /// refused. + const PIPE_PREFIX: &str = r"\\.\pipe\hyperdb-mcp-"; + + /// Size of the in/out buffers of a pipe instance. + const PIPE_BUFFER_BYTES: u32 = 64 * 1024; + + /// How often a busy pipe is retried while connecting. + const BUSY_RETRY: Duration = Duration::from_millis(10); + + #[expect( + clippy::unnecessary_wraps, + reason = "the Unix twin can fail, and both share one signature" + )] + pub(super) fn endpoint_for_new_daemon(_state_dir: &Path) -> io::Result { + let random = RandomState::new().hash_one(std::process::id()); + Ok(format!("{PIPE_PREFIX}{}-{random:016x}", std::process::id())) + } + + pub(super) fn is_valid_endpoint(record: &str, _state_dir: &Path) -> bool { + record.strip_prefix(PIPE_PREFIX).is_some_and(|rest| { + !rest.is_empty() + && rest + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + }) + } + + fn wide(text: &str) -> Vec { + text.encode_utf16().chain(std::iter::once(0)).collect() + } + + /// Milliseconds for the Win32 wait calls, never `INFINITE`. + fn wait_millis(timeout: Duration) -> u32 { + u32::try_from(timeout.as_millis()).map_or(u32::MAX - 1, |millis| millis.min(u32::MAX - 1)) + } + + fn zeroed_overlapped(event: &OwnedHandle) -> OVERLAPPED { + // SAFETY: `OVERLAPPED` is a plain-old-data struct of integers and + // pointers; all-zero is its documented initial state. + let mut overlapped: OVERLAPPED = unsafe { mem::zeroed() }; + overlapped.hEvent = event.as_raw_handle(); + overlapped + } + + fn new_event() -> io::Result { + // SAFETY: null attributes and name are allowed; the returned handle is + // checked for null before it is wrapped. + let handle = unsafe { CreateEventW(ptr::null(), 1, 0, ptr::null()) }; + if handle.is_null() { + return Err(io::Error::last_os_error()); + } + // SAFETY: `handle` is a valid, owned event handle returned just above. + Ok(unsafe { OwnedHandle::from_raw_handle(handle) }) + } + + /// A security descriptor allowing only the current user, freed on drop. + struct UserOnlyDescriptor { + descriptor: *mut core::ffi::c_void, + } + + // SAFETY: the descriptor is an immutable, self-contained `LocalAlloc` + // block; nothing ties it to the creating thread. + unsafe impl Send for UserOnlyDescriptor {} + + impl UserOnlyDescriptor { + fn new() -> io::Result { + let sddl = wide(&format!("D:P(A;;GA;;;{})", current_user_sid()?)); + let mut descriptor = ptr::null_mut(); + // SAFETY: `sddl` is NUL-terminated and outlives the call; + // `descriptor` is a valid out-pointer; the size out-parameter is + // optional. + let ok = unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + SDDL_REVISION_1, + &raw mut descriptor, + ptr::null_mut(), + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + Ok(Self { descriptor }) + } + + fn attributes(&self) -> SECURITY_ATTRIBUTES { + SECURITY_ATTRIBUTES { + nLength: u32::try_from(mem::size_of::()) + .expect("SECURITY_ATTRIBUTES is a few bytes"), + lpSecurityDescriptor: self.descriptor, + bInheritHandle: 0, + } + } + } + + impl Drop for UserOnlyDescriptor { + fn drop(&mut self) { + // SAFETY: `descriptor` was allocated by the conversion call with + // `LocalAlloc` and is freed exactly once, here. + unsafe { LocalFree(self.descriptor) }; + } + } + + /// The current user's SID in string form (`S-1-5-21-...`). + fn current_user_sid() -> io::Result { + let mut token = ptr::null_mut(); + // SAFETY: the pseudo-handle of the current process needs no closing; + // `token` is a valid out-pointer. + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &raw mut token) } == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: `token` is a valid handle returned just above, owned here. + let token = unsafe { OwnedHandle::from_raw_handle(token) }; + + let mut needed = 0_u32; + // SAFETY: a null buffer of length 0 is the documented way to ask for + // the required size. + unsafe { + GetTokenInformation( + token.as_raw_handle(), + TokenUser, + ptr::null_mut(), + 0, + &raw mut needed, + ); + } + // SAFETY: reads the thread's last-error value only. + if unsafe { GetLastError() } != ERROR_INSUFFICIENT_BUFFER { + return Err(io::Error::last_os_error()); + } + // `u64` elements keep the buffer aligned for the pointers in TOKEN_USER. + let words = (usize::try_from(needed).expect("u32 fits usize")).div_ceil(8); + let mut buffer = vec![0_u64; words]; + // SAFETY: `buffer` holds at least `needed` writable bytes. + let ok = unsafe { + GetTokenInformation( + token.as_raw_handle(), + TokenUser, + buffer.as_mut_ptr().cast(), + needed, + &raw mut needed, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the call above filled `buffer` with a TOKEN_USER, and the + // buffer is 8-byte aligned. + let sid = unsafe { (*buffer.as_ptr().cast::()).User.Sid }; + + let mut text = ptr::null_mut(); + // SAFETY: `sid` points into `buffer`, alive for the call; `text` is a + // valid out-pointer. + if unsafe { ConvertSidToStringSidW(sid, &raw mut text) } == 0 { + return Err(io::Error::last_os_error()); + } + let mut length = 0; + // SAFETY: on success `text` is a NUL-terminated wide string. + while unsafe { *text.add(length) } != 0 { + length += 1; + } + // SAFETY: `text` holds `length` initialised `u16`s, per the loop above. + let sid_text = + String::from_utf16_lossy(unsafe { std::slice::from_raw_parts(text, length) }); + // SAFETY: `text` was allocated by `ConvertSidToStringSidW` with + // `LocalAlloc` and is freed once, after its last use above. + unsafe { LocalFree(text.cast()) }; + Ok(sid_text) + } + + /// Create one instance of the pipe, restricted to the current user. + fn create_instance( + name: &[u16], + security: &UserOnlyDescriptor, + first: bool, + ) -> io::Result { + let attributes = security.attributes(); + let mut open_mode = PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED; + if first { + open_mode |= FILE_FLAG_FIRST_PIPE_INSTANCE; + } + // SAFETY: `name` is NUL-terminated; `attributes` and the descriptor it + // points to outlive the call. + let handle = unsafe { + CreateNamedPipeW( + name.as_ptr(), + open_mode, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + PIPE_UNLIMITED_INSTANCES, + PIPE_BUFFER_BYTES, + PIPE_BUFFER_BYTES, + 0, + &raw const attributes, + ) + }; + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: `handle` is a valid pipe handle returned just above. + Ok(unsafe { OwnedHandle::from_raw_handle(handle) }) + } + + pub(super) struct Listener { + name: Vec, + security: UserOnlyDescriptor, + /// The instance clients are currently being accepted on. + pipe: OwnedHandle, + event: OwnedHandle, + /// Boxed so its address is stable while a connect is pending. + overlapped: Box, + connecting: bool, + } + + impl std::fmt::Debug for Listener { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Listener") + .field("connecting", &self.connecting) + .finish_non_exhaustive() + } + } + + // SAFETY: the raw pointers inside `OVERLAPPED` are only touched through + // `&mut self`, and the kernel does not care which thread owns the handle. + unsafe impl Send for Listener {} + + impl Listener { + pub(super) fn bind(endpoint: &str) -> io::Result { + let name = wide(endpoint); + let security = UserOnlyDescriptor::new()?; + let pipe = create_instance(&name, &security, true)?; + let event = new_event()?; + let overlapped = Box::new(zeroed_overlapped(&event)); + Ok(Self { + name, + security, + pipe, + event, + overlapped, + connecting: false, + }) + } + + pub(super) fn accept_timeout(&mut self, timeout: Duration) -> io::Result> { + let connected = if self.connecting { + self.wait_for_connect(timeout)? + } else { + *self.overlapped = zeroed_overlapped(&self.event); + // SAFETY: the pipe handle is valid, and the boxed OVERLAPPED + // stays at a fixed address until the operation completes or + // is cancelled in `Drop`. + let ok = unsafe { + ConnectNamedPipe(self.pipe.as_raw_handle(), &raw mut *self.overlapped) + }; + if ok != 0 { + true + } else { + // SAFETY: reads the thread's last-error value only. + match unsafe { GetLastError() } { + // A client that connected and hung up before this call + // leaves the instance in the connected state; the first + // read on it reports end of stream. + ERROR_PIPE_CONNECTED | ERROR_NO_DATA => true, + ERROR_IO_PENDING => { + self.connecting = true; + self.wait_for_connect(timeout)? + } + _ => return Err(io::Error::last_os_error()), + } + } + }; + if !connected { + return Ok(None); + } + // Create the next instance before handing this one off, so the + // name never has a moment without a listening instance. + let next = create_instance(&self.name, &self.security, false)?; + let accepted = mem::replace(&mut self.pipe, next); + Ok(Some(Stream::new(accepted)?)) + } + + /// Wait for the pending connect. `true` once a client is connected. + fn wait_for_connect(&mut self, timeout: Duration) -> io::Result { + // SAFETY: the event handle is valid. + let wait = + unsafe { WaitForSingleObject(self.event.as_raw_handle(), wait_millis(timeout)) }; + match wait { + WAIT_TIMEOUT => Ok(false), + WAIT_OBJECT_0 => { + self.connecting = false; + let mut transferred = 0_u32; + // SAFETY: the connect on this pipe and OVERLAPPED has + // completed; the result is read without waiting. + let ok = unsafe { + GetOverlappedResult( + self.pipe.as_raw_handle(), + &raw const *self.overlapped, + &raw mut transferred, + 0, + ) + }; + if ok == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(true) + } + } + _ => Err(io::Error::last_os_error()), + } + } + } + + impl Drop for Listener { + fn drop(&mut self) { + if self.connecting { + let mut transferred = 0_u32; + // SAFETY: cancels and then waits out the one pending connect + // so the kernel is finished with the boxed OVERLAPPED before + // it is freed; the handles are still open here. + unsafe { + CancelIoEx(self.pipe.as_raw_handle(), &raw const *self.overlapped); + GetOverlappedResult( + self.pipe.as_raw_handle(), + &raw const *self.overlapped, + &raw mut transferred, + 1, + ); + } + } + } + } + + pub(super) fn connect(endpoint: &str, connect_timeout: Duration) -> io::Result { + let deadline = Instant::now() + connect_timeout; + loop { + let opened = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(FILE_FLAG_OVERLAPPED) + // The server may identify us but never impersonate us. + .security_qos_flags(SECURITY_IDENTIFICATION) + .open(endpoint); + match opened { + Ok(file) => return Stream::new(OwnedHandle::from(file)), + Err(error) + if error + .raw_os_error() + .and_then(|code| u32::try_from(code).ok()) + == Some(ERROR_PIPE_BUSY) => + { + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "the daemon control pipe stayed busy", + )); + } + std::thread::sleep(BUSY_RETRY); + } + Err(error) + if error + .raw_os_error() + .and_then(|code| u32::try_from(code).ok()) + == Some(ERROR_FILE_NOT_FOUND) => + { + return Err(io::Error::new(io::ErrorKind::NotFound, error)); + } + Err(error) => return Err(error), + } + } + } + + #[derive(Debug)] + pub(super) struct Stream { + pipe: OwnedHandle, + event: OwnedHandle, + timeout: Duration, + } + + impl Stream { + fn new(pipe: OwnedHandle) -> io::Result { + Ok(Self { + pipe, + event: new_event()?, + timeout: super::CONNECTION_IO_TIMEOUT, + }) + } + + pub(super) fn set_io_timeout(&mut self, timeout: Duration) -> io::Result<()> { + if timeout.is_zero() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a zero I/O timeout is not allowed", + )); + } + self.timeout = timeout; + Ok(()) + } + + /// Wait for the overlapped operation started by `ReadFile`/`WriteFile` + /// and return the byte count; cancel it on timeout. + fn complete(&mut self, started: i32, overlapped: &mut OVERLAPPED) -> io::Result { + if started == 0 { + // SAFETY: reads the thread's last-error value only. + let code = unsafe { GetLastError() }; + if code != ERROR_IO_PENDING { + return Err(io::Error::from_raw_os_error( + i32::try_from(code).unwrap_or(i32::MAX), + )); + } + } + // SAFETY: the event handle is valid. + let wait = unsafe { + WaitForSingleObject(self.event.as_raw_handle(), wait_millis(self.timeout)) + }; + let mut transferred = 0_u32; + if wait != WAIT_OBJECT_0 { + // Captured before the cancel call can overwrite it. + let wait_error = (wait != WAIT_TIMEOUT).then(io::Error::last_os_error); + // SAFETY: cancels the one operation on this handle that uses + // `overlapped`, then waits for the kernel to let go of it, so + // neither the OVERLAPPED nor the caller's buffer is freed + // while the I/O is still in flight. + let finished = unsafe { + CancelIoEx(self.pipe.as_raw_handle(), &raw const *overlapped); + GetOverlappedResult( + self.pipe.as_raw_handle(), + &raw const *overlapped, + &raw mut transferred, + 1, + ) + }; + if finished != 0 { + // The operation completed before the cancel took effect. + return Ok(usize::try_from(transferred).expect("u32 fits usize")); + } + return Err(wait_error.unwrap_or_else(|| { + io::Error::new(io::ErrorKind::TimedOut, "the control connection timed out") + })); + } + // SAFETY: the operation has completed; read its result without + // waiting. + let ok = unsafe { + GetOverlappedResult( + self.pipe.as_raw_handle(), + &raw const *overlapped, + &raw mut transferred, + 0, + ) + }; + if ok == 0 { + return Err(io::Error::last_os_error()); + } + Ok(usize::try_from(transferred).expect("u32 fits usize")) + } + + /// A closed peer reads as end of stream, like a socket. + fn is_closed_peer(error: &io::Error) -> bool { + error + .raw_os_error() + .and_then(|code| u32::try_from(code).ok()) + .is_some_and(|code| { + matches!( + code, + ERROR_BROKEN_PIPE | ERROR_NO_DATA | ERROR_PIPE_NOT_CONNECTED + ) + }) + } + + pub(super) fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + let length = u32::try_from(buf.len()).unwrap_or(u32::MAX); + let mut overlapped = zeroed_overlapped(&self.event); + // SAFETY: `buf` and `overlapped` live until `complete` has seen + // the operation finish or cancelled it. + let started = unsafe { + ReadFile( + self.pipe.as_raw_handle(), + buf.as_mut_ptr(), + length, + ptr::null_mut(), + &raw mut overlapped, + ) + }; + match self.complete(started, &mut overlapped) { + Err(error) if Self::is_closed_peer(&error) => Ok(0), + other => other, + } + } + + pub(super) fn write(&mut self, buf: &[u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + let length = u32::try_from(buf.len()).unwrap_or(u32::MAX); + let mut overlapped = zeroed_overlapped(&self.event); + // SAFETY: as in `read`. + let started = unsafe { + WriteFile( + self.pipe.as_raw_handle(), + buf.as_ptr(), + length, + ptr::null_mut(), + &raw mut overlapped, + ) + }; + match self.complete(started, &mut overlapped) { + Err(error) if Self::is_closed_peer(&error) => { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + } + other => other, + } + } + + #[expect( + clippy::unused_self, + clippy::unnecessary_wraps, + reason = "writes are complete when `write` returns; shared signature with Unix" + )] + pub(super) fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + + /// Close once the peer has read everything written. + /// + /// `FlushFileBuffers` waits for the peer to drain the pipe and cannot + /// be bounded, so it runs on a short-lived thread: a peer that never + /// reads pins that one idle thread, not the connection handler. + pub(super) fn finish(self) -> io::Result<()> { + let Self { pipe, event, .. } = self; + drop(event); + std::thread::Builder::new() + .name("hyperdb-control-flush".to_owned()) + .spawn(move || { + // SAFETY: the pipe handle is valid and owned by this + // thread until it is dropped below. + unsafe { FlushFileBuffers(pipe.as_raw_handle()) }; + drop(pipe); + }) + .map(drop) + } + } +} + +#[cfg(test)] +mod tests { + use std::io::{BufRead, BufReader, Read, Write}; + use std::time::{Duration, Instant}; + + use super::{ControlListener, ControlStream, HealthEndpoint, connect}; + + fn endpoint_in(dir: &tempfile::TempDir) -> HealthEndpoint { + HealthEndpoint::for_new_daemon(dir.path()).expect("endpoint") + } + + fn accept_one(listener: &mut ControlListener) -> ControlStream { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if let Some(stream) = listener + .accept_timeout(Duration::from_millis(100)) + .expect("accept") + { + return stream; + } + } + panic!("no client arrived"); + } + + #[test] + fn a_line_round_trips() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).expect("bind"); + let server = std::thread::spawn(move || { + let stream = accept_one(&mut listener); + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).expect("read"); + assert_eq!(line, "PING\n"); + let stream = reader.get_mut(); + stream.write_all(b"PONG\n").expect("write"); + let stream = reader.into_inner(); + stream.finish().expect("finish"); + listener + }); + let mut client = connect(&endpoint, Duration::from_secs(5)).expect("connect"); + client.write_all(b"PING\n").expect("send"); + let mut reply = String::new(); + BufReader::new(client).read_line(&mut reply).expect("reply"); + assert_eq!(reply, "PONG\n"); + drop(server.join().expect("server thread")); + } + + #[test] + fn accept_times_out_without_a_client() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut listener = ControlListener::bind(&endpoint_in(&dir)).expect("bind"); + let started = Instant::now(); + assert!( + listener + .accept_timeout(Duration::from_millis(100)) + .expect("accept") + .is_none() + ); + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn wake_unblocks_nothing_and_does_not_panic() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).expect("bind"); + ControlListener::wake(&endpoint); + // The wake connection is accepted like any other. + let _hung_up = accept_one(&mut listener); + } + + #[test] + fn a_second_bind_on_a_live_endpoint_fails() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let mut first = ControlListener::bind(&endpoint).expect("bind"); + assert!( + ControlListener::bind(&endpoint).is_err(), + "a live endpoint must not be taken over" + ); + // The first listener still serves. + let client = connect(&endpoint, Duration::from_secs(5)).expect("connect to the survivor"); + let _server_side = accept_one(&mut first); + drop(client); + } + + #[test] + fn connecting_to_nothing_fails() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + assert!(connect(&endpoint, Duration::from_millis(200)).is_err()); + } + + #[test] + fn a_silent_peer_hits_the_read_timeout() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).expect("bind"); + let _silent_client = connect(&endpoint, Duration::from_secs(5)).expect("connect"); + let mut server_side = accept_one(&mut listener); + server_side + .set_io_timeout(Duration::from_millis(150)) + .expect("timeout"); + let started = Instant::now(); + let error = server_side + .read(&mut [0_u8; 8]) + .expect_err("a silent peer must time out"); + assert!( + matches!( + error.kind(), + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock + ), + "unexpected error kind: {error:?}" + ); + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_hung_up_peer_reads_as_end_of_stream() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).expect("bind"); + let client = connect(&endpoint, Duration::from_secs(5)).expect("connect"); + let mut server_side = accept_one(&mut listener); + drop(client); + assert_eq!(server_side.read(&mut [0_u8; 8]).expect("read"), 0); + } + + #[test] + fn record_parsing_round_trips_and_rejects_foreign_values() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + assert_eq!( + HealthEndpoint::from_record(endpoint.as_str(), dir.path()), + Some(endpoint.clone()) + ); + assert_eq!(endpoint.to_string(), endpoint.as_str()); + assert!(HealthEndpoint::from_record("", dir.path()).is_none()); + assert!(HealthEndpoint::from_record("127.0.0.1:7485", dir.path()).is_none()); + assert!(HealthEndpoint::from_record("a\0b", dir.path()).is_none()); + #[cfg(unix)] + assert!( + HealthEndpoint::from_record("/var/run/docker.sock", dir.path()).is_none(), + "a record naming another program's socket must be refused" + ); + } + + #[cfg(unix)] + mod unix { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::net::UnixListener; + + use super::{ControlListener, endpoint_in}; + use crate::daemon::control::HealthEndpoint; + + #[test] + fn the_socket_is_owner_only() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let _listener = ControlListener::bind(&endpoint).expect("bind"); + let mode = std::fs::metadata(endpoint.as_str()) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600); + } + + #[test] + fn a_stale_socket_file_is_reclaimed() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + // A std listener leaves its socket file behind when dropped, + // which is exactly what a killed daemon does. + drop(UnixListener::bind(endpoint.as_str()).expect("make stale socket")); + assert!(std::path::Path::new(endpoint.as_str()).exists()); + ControlListener::bind(&endpoint).expect("a stale socket is reclaimed"); + } + + #[test] + fn a_regular_file_at_the_path_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + std::fs::write(endpoint.as_str(), b"precious").expect("write"); + let error = ControlListener::bind(&endpoint).expect_err("must refuse"); + assert!(error.to_string().contains("not a socket"), "{error}"); + assert_eq!(std::fs::read(endpoint.as_str()).expect("read"), b"precious"); + } + + #[test] + fn the_socket_file_goes_away_with_the_listener() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + drop(ControlListener::bind(&endpoint).expect("bind")); + assert!(!std::path::Path::new(endpoint.as_str()).exists()); + } + + #[test] + fn dropping_an_old_listener_keeps_a_successors_socket() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let old = ControlListener::bind(&endpoint).expect("bind old"); + // The old daemon's file is gone and a successor reclaimed the path. + std::fs::remove_file(endpoint.as_str()).expect("unlink"); + let _successor = ControlListener::bind(&endpoint).expect("bind successor"); + drop(old); + assert!( + std::path::Path::new(endpoint.as_str()).exists(), + "the successor's live socket must survive the old listener's drop" + ); + } + + #[test] + fn an_over_long_path_names_the_state_dir_variable() { + let dir = tempfile::tempdir().expect("tempdir"); + let long = dir.path().join("x".repeat(120)); + let error = HealthEndpoint::for_new_daemon(&long).expect_err("too long"); + assert!(error.to_string().contains("HYPERDB_STATE_DIR"), "{error}"); + } + } +} diff --git a/hyperdb-mcp/src/daemon/lock.rs b/hyperdb-mcp/src/daemon/lock.rs new file mode 100644 index 00000000..9937df45 --- /dev/null +++ b/hyperdb-mcp/src/daemon/lock.rs @@ -0,0 +1,184 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! The single-instance lock of the daemon. +//! +//! A daemon holds an exclusive lock on `/daemon.lock` for its whole +//! life, until after `hyperd` has exited. Holding it is what "a daemon is +//! alive" means: unlike the discovery record it cannot go stale, because the +//! operating system releases it when the holding process exits, however it +//! exits. A client that finds the lock held and no usable record knows a +//! daemon is starting or stopping, and waits instead of spawning a second one. +//! +//! **The lock file is never unlinked.** A process holding the lock on a +//! deleted file would no longer exclude a newcomer, which creates a fresh file +//! and locks that one. Releasing the lock only closes the handle. +//! +//! The lock conflicts within one process too, because each acquire opens the +//! file separately; the in-process test daemons rely on that. +//! +//! This module depends only on `std`, `libc` and `windows-sys`, so it can be +//! compiled for Windows on its own. + +use std::fs::File; +use std::io; +use std::path::Path; + +/// File name of the lock inside the state directory. +pub const LOCK_FILE_NAME: &str = "daemon.lock"; + +/// An exclusive hold on the daemon lock. Released when dropped. +#[derive(Debug)] +pub struct DaemonLock { + _file: File, +} + +impl DaemonLock { + /// Try to take the daemon lock in `state_dir` without waiting. + /// + /// Returns `Ok(None)` when another holder (in this process or another) + /// has it. The state directory must already exist. + /// + /// # Errors + /// + /// Returns the I/O error if the lock file cannot be opened or locked for + /// any reason other than being held, such as `state_dir` missing, or the + /// lock file being a symlink on Unix. + pub fn try_acquire(state_dir: &Path) -> io::Result> { + platform::try_lock(&state_dir.join(LOCK_FILE_NAME)) + .map(|held| held.map(|file| Self { _file: file })) + } +} + +#[cfg(unix)] +mod platform { + use std::fs::{File, OpenOptions}; + use std::io; + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + use std::path::Path; + + pub(super) fn try_lock(path: &Path) -> io::Result> { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?; + // SAFETY: `file` owns a valid open descriptor for the whole call. + // `flock` reads no memory. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + return Ok(Some(file)); + } + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::WouldBlock { + Ok(None) + } else { + Err(error) + } + } +} + +#[cfg(windows)] +mod platform { + use std::fs::{File, OpenOptions}; + use std::io; + use std::os::windows::fs::OpenOptionsExt; + use std::path::Path; + + /// `ERROR_SHARING_VIOLATION`: another handle has the file open. + const ERROR_SHARING_VIOLATION: i32 = 32; + + /// Opening with no sharing is the lock: a second open fails while the + /// first handle lives. + pub(super) fn try_lock(path: &Path) -> io::Result> { + match OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .share_mode(0) + .open(path) + { + Ok(file) => Ok(Some(file)), + Err(error) if error.raw_os_error() == Some(ERROR_SHARING_VIOLATION) => Ok(None), + Err(error) => Err(error), + } + } +} + +#[cfg(test)] +mod tests { + use super::{DaemonLock, LOCK_FILE_NAME}; + + #[test] + fn second_acquire_in_the_same_process_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let first = DaemonLock::try_acquire(dir.path()) + .expect("acquire") + .expect("free lock"); + assert!( + DaemonLock::try_acquire(dir.path()) + .expect("second acquire") + .is_none(), + "a held lock must refuse a second holder" + ); + drop(first); + } + + #[test] + fn released_on_drop_and_the_file_survives() { + let dir = tempfile::tempdir().expect("tempdir"); + let lock = DaemonLock::try_acquire(dir.path()) + .expect("acquire") + .expect("free lock"); + drop(lock); + assert!( + dir.path().join(LOCK_FILE_NAME).exists(), + "the lock file is never unlinked" + ); + assert!( + DaemonLock::try_acquire(dir.path()) + .expect("reacquire") + .is_some(), + "a released lock can be taken again" + ); + } + + #[test] + fn missing_state_dir_is_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let missing = dir.path().join("missing"); + assert!(DaemonLock::try_acquire(&missing).is_err()); + } + + #[cfg(unix)] + #[test] + fn a_symlinked_lock_file_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join("elsewhere"); + std::fs::write(&target, b"").expect("write target"); + std::os::unix::fs::symlink(&target, dir.path().join(LOCK_FILE_NAME)).expect("symlink"); + assert!( + DaemonLock::try_acquire(dir.path()).is_err(), + "O_NOFOLLOW must refuse a symlink planted at the lock path" + ); + } + + #[cfg(unix)] + #[test] + fn the_lock_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + let _lock = DaemonLock::try_acquire(dir.path()) + .expect("acquire") + .expect("free lock"); + let mode = std::fs::metadata(dir.path().join(LOCK_FILE_NAME)) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600); + } +} diff --git a/hyperdb-mcp/src/daemon/mod.rs b/hyperdb-mcp/src/daemon/mod.rs index 187e65c3..07c64887 100644 --- a/hyperdb-mcp/src/daemon/mod.rs +++ b/hyperdb-mcp/src/daemon/mod.rs @@ -3,8 +3,10 @@ //! Single-instance daemon for sharing a `hyperd` process across MCP clients. +pub mod control; pub mod discovery; pub mod health; +pub mod lock; pub mod run; pub mod spawn; pub mod state_perms; From 1047b7d731accbaadfa9c0995217209be7049fb4 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 15:32:12 -0700 Subject: [PATCH 34/61] feat(mcp)!: daemon health channel is a per-user Unix socket or named pipe Replace the unauthenticated loopback TCP health port with the per-user control endpoint, add a single-instance daemon lock, and require the state directory to be owned by the user and not group/other writable. Remove --port, HYPERDB_DAEMON_PORT, the port scan and the doctor live-from-scan path. Old clients against a new daemon fall back to local mode. --- .claude/workflows/plan-to-release.js | 2 +- hyperdb-mcp/CHANGELOG.md | 24 + hyperdb-mcp/DEVELOPMENT.md | 58 +- hyperdb-mcp/README.md | 32 +- hyperdb-mcp/src/daemon/control.rs | 5 + hyperdb-mcp/src/daemon/discovery.rs | 871 ++++------ hyperdb-mcp/src/daemon/health.rs | 830 ++++----- hyperdb-mcp/src/daemon/mod.rs | 17 +- hyperdb-mcp/src/daemon/run.rs | 163 +- hyperdb-mcp/src/daemon/spawn.rs | 289 +++- hyperdb-mcp/src/daemon/state_perms.rs | 125 ++ hyperdb-mcp/src/diagnostics.rs | 1383 +++++++-------- hyperdb-mcp/src/engine.rs | 67 +- hyperdb-mcp/src/main.rs | 145 +- hyperdb-mcp/src/server.rs | 153 +- hyperdb-mcp/tests/daemon_tests.rs | 1727 ++++++++++--------- hyperdb-mcp/tests/doctor_tests.rs | 307 ++-- hyperdb-mcp/tests/engine_tests.rs | 6 +- hyperdb-mcp/tests/health_idle_cost_tests.rs | 24 +- hyperdb-mcp/tests/readme_tests.rs | 25 +- hyperdb-mcp/tests/recovery_tests.rs | 91 +- 21 files changed, 3216 insertions(+), 3128 deletions(-) diff --git a/.claude/workflows/plan-to-release.js b/.claude/workflows/plan-to-release.js index 445a5f03..36a4faff 100644 --- a/.claude/workflows/plan-to-release.js +++ b/.claude/workflows/plan-to-release.js @@ -545,7 +545,7 @@ CRITICAL — verify against the REAL published artifact, NOT this session's conn \`npx -y ${npmPackage}@${expected} --ephemeral-only --no-daemon\` These two flags avoid two real collisions with the session's live MCP, observed on the v0.7.0 run: - \`--ephemeral-only\`: the shared persistent \`workspace.hyper\` is held open by the session's server; opening it from a second process throws SQLSTATE 55006 ("database file is locked by another process"). The KV smoke checks only need the ephemeral DB, so skip persistent entirely. - - \`--no-daemon\`: a newer client performs a daemon version-takeover on the shared port 7485 (a shipped feature) — killing the session's daemon. \`--no-daemon\` spawns a private hyperd and leaves the session's daemon alone. + - \`--no-daemon\`: a newer client performs a daemon version-takeover on the shared per-user health socket (a shipped feature) — killing the session's daemon. \`--no-daemon\` spawns a private hyperd and leaves the session's daemon alone. Do the MCP JSON-RPC handshake (initialize -> notifications/initialized -> tools/call). GATE FIRST: call \`status\` and assert its reported version starts with "${expected}". If it does not, STOP — you are not testing the new artifact; report version_match=false. Only if the version matches, run these checks and record each as a test: - kv_set returns a \`created\` field (true on first write, false on overwrite of the same key) and \`value_bytes\`. - kv_set with overwrite:false on an existing key returns \`{stored:false, existed:true}\` / does not clobber. diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 353ffd74..b4ba0d5e 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -25,6 +25,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/). existing destination is refused with `PERMISSION_DENIED` unless the call passes `overwrite: true`. `export` used to replace an existing file or directory silently; `chart` defaulted `overwrite` to true. +- **BREAKING: the daemon health channel is a per-user Unix domain socket + (`/daemon.sock`, mode `0600`) or Windows named pipe, not a + loopback TCP port.** `daemon.json` replaces the `health_port` field with + `health_endpoint` (a string), `daemon status` prints `Health endpoint:`, the + engine status key `daemon_health_port` becomes `daemon_health_endpoint`, and + `doctor` reports `health_endpoint` and a new `lock` state in place of the + health port. A single-instance lock (`daemon.lock`) replaces the port bind as + the "is a daemon running" decision. An older client talking to a daemon of + this release cannot find it (the record no longer has `health_port`) and falls + back to local mode. +- **The state directory must now be owned by the current user and not writable + by group or others.** A daemon refuses to start in one, and a client treats it + as untrusted and uses local mode rather than connecting to an endpoint it + names. + +### Removed + +- **`hyperdb-mcp daemon --port`, `daemon stop --port`, `daemon status --port` + and the `HYPERDB_DAEMON_PORT` environment variable.** There is no port to + choose. Use `HYPERDB_STATE_DIR` to run an isolated daemon. A + `HYPERDB_DAEMON_PORT` left in the environment is now ignored. +- **The daemon port scan** (the 16 ports above 7485) and doctor's + scan-discovered daemon state. Discovery reads `daemon.json` from the state + directory only. ### Fixed diff --git a/hyperdb-mcp/DEVELOPMENT.md b/hyperdb-mcp/DEVELOPMENT.md index 6fe598b8..732a8826 100644 --- a/hyperdb-mcp/DEVELOPMENT.md +++ b/hyperdb-mcp/DEVELOPMENT.md @@ -237,35 +237,43 @@ existing daemon via `~/.hyperdb/daemon.json` (overridable via `HYPERDB_STATE_DIR` — see `daemon::state_perms`, which restricts that directory and the files in it to the owning user, and which needs the override to name a path that can carry those permissions: inside `%USERPROFILE%` on Windows, on a -mode-supporting filesystem on Unix), else scans the port range for a running -daemon, else -auto-spawns one on the first free port as a detached background process. The -Engine then connects via TCP (`Connection::connect(endpoint, …)`) without -owning any `HyperProcess`, and records the daemon's `health_port` so the -server's debounced `HEARTBEAT` targets the actual discovered port rather than -re-resolving. - -Falls back to local mode (per-session `hyperd` via `HyperProcess::new`) when the daemon can't be reached (including `AllOccupied` — the whole scan range is held by foreign processes), or always when `--no-daemon` is passed. - -**Port resolution + identity.** `resolve_port_scan()` returns a -`PortScan { base, span }`: when `HYPERDB_DAEMON_PORT` is set it pins that exact -port (`span = 1`); otherwise it scans `span = DAEMON_PORT_SCAN_SPAN` (16) ports -up from `DEFAULT_DAEMON_BASE_PORT` (7485 — deliberately *not* 7484, which is -hyperd's conventional gRPC port). `probe_port` classifies each port as -`OurDaemon` / `Camped` / `Refused`: liveness is no longer a bare TCP connect -but an identity handshake — `health::ping_identified` sends `PING` and requires -the reply's first two tokens to be exactly `PONG` and `hyperdb-mcp` (the third -token is the daemon version). A foreign process that merely accepts TCP is -`Camped` and skipped; only a `Refused` (connection-refused) port is treated as -free to spawn on. `discover()` applies the same identity check before trusting -`daemon.json`, so a stale or foreign-owned file is detected and removed. +mode-supporting filesystem on Unix), else auto-spawns one as a detached +background process. The Engine then connects via TCP +(`Connection::connect(endpoint, …)`) without owning any `HyperProcess`, and +records the daemon's `health_endpoint` so the server's debounced `HEARTBEAT` +targets the discovered endpoint rather than re-resolving. + +Falls back to local mode (per-session `hyperd` via `HyperProcess::new`) when the daemon can't be reached (including a state directory that is not owned by the user or is group/other-writable), or always when `--no-daemon` is passed. + +**Health channel + identity.** The health channel is not a TCP port. It is +`/daemon.sock` (Unix, mode `0600`) or a per-user named pipe +(Windows), implemented in `daemon::control` (`HealthEndpoint`, +`ControlListener`, `ControlStream`, `connect`). `daemon.json` carries it as +`health_endpoint`; a client accepts the recorded value only if +`HealthEndpoint::from_record` finds it equals the endpoint its own state +directory implies, so a tampered record can never redirect it. Liveness is an +identity handshake: `health::ping_identified` sends `PING` and requires the +reply's first two tokens to be exactly `PONG` and `hyperdb-mcp` (the third is +the daemon version). `discover()` is a pure read (record + identified `PING`); +stale-record removal lives in `ensure_daemon`, under the lock. + +**Single instance.** `daemon::lock::DaemonLock` is an `flock` (Windows: a locked +file region) on `/daemon.lock`, never unlinked. `run_daemon` creates +and checks the state directory (`state_perms::verify_state_dir_trusted`: owned +by the effective user, no group/other write), takes the lock (retrying up to +`STARTUP_WAIT`; a daemon that answers an identified `PING` continuously for +`ALREADY_RUNNING_DEBOUNCE` is reported as already running), binds the endpoint, +starts `hyperd`, then writes the record. On exit it removes the record, joins +the listener, drops `hyperd`, then releases the lock. `ensure_daemon` therefore +treats a held lock with no usable record as "a daemon is starting" and waits; +it spawns only when the lock is free, after removing any stale record. **Version takeover.** When discovery finds a running daemon, `maybe_take_over` compares the client's `version::MCP_VERSION` against the daemon's reported version via the pure `client_should_take_over` helper (`semver`). If the client is *strictly newer* it sends `STOP` (which drops the daemon's `HyperProcess`, -stopping `hyperd`), waits for the health port to stop answering the identity -ping, then respawns a fresh daemon on the same port. Equal/older/unparseable +stopping `hyperd`), waits for the health endpoint to stop answering the identity +ping, then respawns a fresh daemon on the same endpoint. Equal/older/unparseable versions reuse the daemon — never a downgrade-kill. This makes upgrades take effect immediately instead of waiting for the old daemon to disappear. @@ -281,7 +289,7 @@ and only matter when the timeout is enabled. The hyperd restart-limit shutdown ### hyperd liveness monitoring and restart -A second monitor task in `run.rs::hyperd_monitor` polls the owned `HyperProcess` every 5 seconds via `HyperProcess::has_exited` (a `Child::try_wait()` call — zero SQL, correct on Unix and Windows, reaps zombies as a side effect). When the process is detected dead — or when a client sends `REPORT_HYPERD_ERROR` to the health port via `daemon::health::report_hyperd_error_to_daemon` — the monitor enters `try_restart_hyperd`: +A second monitor task in `run.rs::hyperd_monitor` polls the owned `HyperProcess` every 5 seconds via `HyperProcess::has_exited` (a `Child::try_wait()` call — zero SQL, correct on Unix and Windows, reaps zombies as a side effect). When the process is detected dead — or when a client sends `REPORT_HYPERD_ERROR` to the health endpoint via `daemon::health::report_hyperd_error_to_daemon` — the monitor enters `try_restart_hyperd`: 1. Prune the rolling restart-history vector to entries within `RESTART_WINDOW` (60s); reject if `RESTART_LIMIT` (3) attempts have already happened. 2. Drop the old `HyperProcess` (its `Drop` impl waits up to 5s for graceful shutdown — near-instant for an already-exited process). diff --git a/hyperdb-mcp/README.md b/hyperdb-mcp/README.md index c06c3847..f7e2cdb0 100644 --- a/hyperdb-mcp/README.md +++ b/hyperdb-mcp/README.md @@ -290,7 +290,7 @@ hyperdb-mcp daemon stop # Gracefully shut down the daemon hyperdb-mcp daemon # Run as a daemon explicitly (rarely needed) ``` -`status` and `stop` locate the running daemon automatically (reading `daemon.json`, then scanning the port range), so they work even if the daemon scanned onto a non-default port. Pass `--port ` to target a specific port explicitly. +`status` and `stop` locate the running daemon automatically by reading `daemon.json` from the state directory and checking that the daemon answers on the health channel. There is no port to pass. State files live at `~/.hyperdb/` by default (override with `HYPERDB_STATE_DIR`). They record the `hyperd` endpoint, so the daemon restricts them to your own @@ -317,15 +317,19 @@ details are bounded, optional launcher-reported provenance. A live daemon is attributed only after a fresh `STATUS` response is verified. Reports contain local paths; review them before sharing. -**Port discovery.** MCP auto-spawn discovers a live daemon first, then scans -upward from **7485** across 16 candidates before starting one at the selected -exact port. Setting `HYPERDB_DAEMON_PORT` pins auto-spawn to one candidate. -In contrast, a manually launched foreground `hyperdb-mcp daemon` never scans: -`--port ` binds that exact port, while an omitted `--port` binds the -configured/base port exactly. The health port doubles as a single-instance -lock and identity check: clients send `PING` and require a +**Health channel.** The daemon's health channel is a per-user Unix domain +socket (`daemon.sock`, mode `0600`) in the state directory, or a named pipe on +Windows, not a TCP port. Nothing listens on the network for it, and another +account cannot reach it. A single-instance lock (`daemon.lock`, held for the +daemon's lifetime and released by the OS if it dies) decides whether a daemon is +running: a client that finds the lock held waits for the record rather than +starting a second daemon. Clients send `PING` and require a `PONG hyperdb-mcp ` reply before trusting a daemon, so an unrelated -process is not mistaken for HyperDB. +process is not mistaken for HyperDB. The state directory must be owned by you +and not writable by group or others; otherwise the client uses local mode (a +private `hyperd`) instead of trusting it. To run a second, isolated daemon (for +example in tests), point `HYPERDB_STATE_DIR` at a different directory; keep the +path short, because Unix socket paths are limited to about 100 bytes. **Staying resident.** By default the daemon never idle-shuts-down — keeping `hyperd` warm means the next tool call connects immediately instead of triggering a "restarting, please retry" round-trip. To opt into auto-shutdown (e.g. on CI), pass `--idle-timeout ` or set `HYPERDB_DAEMON_IDLE_TIMEOUT`. @@ -1061,11 +1065,10 @@ Deprecated: stderr warning, and will be removed in a future release. Daemon subcommand: - hyperdb-mcp daemon Start foreground on the configured/base port exactly + hyperdb-mcp daemon Start in the foreground; the health channel is a + per-user socket or named pipe in the state directory hyperdb-mcp daemon stop Gracefully stop the running daemon hyperdb-mcp daemon status Show running daemon info - hyperdb-mcp daemon --port Bind this exact health/lock port; foreground - startup never performs the auto-spawn scan. hyperdb-mcp daemon --idle-timeout Opt into idle shutdown after SECS idle. When omitted, the daemon stays resident. @@ -1073,11 +1076,10 @@ Environment: HYPERD_PATH Hyperd executable or containing directory; when absent or non-UTF-8, walk upward for .hyperd/current/hyperd (no PATH lookup) HYPERDB_PERSISTENT_DB Override the default persistent-db path - HYPERDB_STATE_DIR Override daemon state directory (default ~/.hyperdb/); keep it + HYPERDB_STATE_DIR Override daemon state directory (default ~/.hyperdb/); one + daemon runs per state directory; keep it under your user profile on Windows and on a filesystem with Unix modes on Unix, or it cannot be restricted to your account - HYPERDB_DAEMON_PORT Pin auto-spawn discovery to one health/lock candidate; - foreground startup binds this configured/base port exactly HYPERDB_DAEMON_IDLE_TIMEOUT Opt into idle shutdown (seconds); default: stay resident ``` diff --git a/hyperdb-mcp/src/daemon/control.rs b/hyperdb-mcp/src/daemon/control.rs index f42e13dd..18034b43 100644 --- a/hyperdb-mcp/src/daemon/control.rs +++ b/hyperdb-mcp/src/daemon/control.rs @@ -457,6 +457,11 @@ mod platform { Ok(format!("{PIPE_PREFIX}{}-{random:016x}", std::process::id())) } + /// Accepts any pipe name with this daemon family's prefix and character + /// set; `_state_dir` is deliberately not consulted. Unlike the Unix socket + /// path, a pipe name does not encode the state directory, so the check + /// cannot pin the exact endpoint. Protection on Windows is the pipe's DACL + /// (owner only), not this check. pub(super) fn is_valid_endpoint(record: &str, _state_dir: &Path) -> bool { record.strip_prefix(PIPE_PREFIX).is_some_and(|rest| { !rest.is_empty() diff --git a/hyperdb-mcp/src/daemon/discovery.rs b/hyperdb-mcp/src/daemon/discovery.rs index 1eea80c3..05f145c1 100644 --- a/hyperdb-mcp/src/daemon/discovery.rs +++ b/hyperdb-mcp/src/daemon/discovery.rs @@ -3,9 +3,12 @@ //! Discovery file management for the single-instance daemon. //! -//! The daemon writes a JSON file to `~/.hyperdb/daemon.json` containing its -//! PID and the `hyperd` endpoint. Clients read this file to locate the running -//! daemon, validating liveness via a TCP health check before trusting it. +//! The daemon writes a JSON file to `/daemon.json` (by default +//! `~/.hyperdb/daemon.json`) containing its PID, the `hyperd` endpoint and its +//! health endpoint. Clients read this file to locate the running daemon, +//! validating liveness via an identified `PING` on the health endpoint before +//! trusting it. The file is a locator, not a lock: whether a daemon is running +//! is decided by [`super::lock::DaemonLock`]. use std::io::{self, Read as _}; use std::path::{Path, PathBuf}; @@ -13,7 +16,7 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; -use super::{DAEMON_PORT_SCAN_SPAN, DEFAULT_DAEMON_BASE_PORT}; +use super::control::HealthEndpoint; const MAX_DISCOVERY_FILE_BYTES: usize = 64 * 1024; @@ -24,8 +27,11 @@ pub struct DaemonInfo { pub pid: u32, /// The `hyperd` libpq endpoint clients should connect to (e.g. `127.0.0.1:54321`). pub hyperd_endpoint: String, - /// The TCP port the daemon's health listener is bound to. - pub health_port: u16, + /// Where the daemon's health listener is bound: a Unix socket path, or a + /// named-pipe name on Windows. Clients accept it only if it is the + /// endpoint their own state directory implies (see + /// [`HealthEndpoint::from_record`]). + pub health_endpoint: String, /// ISO-8601 timestamp when the daemon started. pub started_at: String, /// Version of the daemon binary. @@ -318,9 +324,28 @@ fn write_discovery_record(record: &(impl Serialize + ?Sized)) -> io::Result<()> } /// Read the discovery file and validate that the daemon is still alive. -/// Returns `None` if no daemon is running (file missing, stale, or unreachable). +/// Returns `None` if no daemon is running (file missing, state directory not +/// trusted, health endpoint not the one this state directory implies, or the +/// daemon does not answer an identified `PING`). +/// +/// Side-effect free: a stale record is left for `remove_stale_record`, which +/// runs under the daemon lock where it cannot race a starting daemon. pub fn discover() -> Option { - let discovery_path = discovery_file_path().ok()?; + discover_in(&state_dir().ok()?) +} + +/// [`discover`] for an explicit state directory. +pub fn discover_in(dir: &Path) -> Option { + // A state directory another user can write to could hold a record that + // points this client at an endpoint of someone else's choosing. + match super::state_perms::verify_state_dir_trusted(dir) { + Ok(()) => {} + Err(error) => { + tracing::debug!(kind = ?error.kind(), "daemon state directory is not usable for discovery"); + return None; + } + } + let discovery_path = dir.join("daemon.json"); // Preserve the historical client-discovery contract: normal discovery // follows symlinks and accepts any valid record size. Doctor uses the // separate bounded, no-follow raw reader above because it must never @@ -329,16 +354,12 @@ pub fn discover() -> Option { // The `RawDiscoveryRead` classification below chooses a debug-log message // and short-circuits a missing/unreadable file. It must never gate whether // a *live* daemon is discovered: an unrecognized, reshaped, or absent - // `identity` block is forward-compatible noise to this fast path — see - // docs/superpowers/specs/2026-08-13-hyperdb-mcp-agent-ux-design.md - // ("Unknown fields remain forward compatible"). So when the strict - // `DaemonRecord` parse fails, we don't give up: we re-parse the bytes the - // classification already carries as a tolerant `DaemonInfo`, which ignores - // unknown fields and never fails on a nested object (like `identity`) that - // this fast path doesn't need. (The doctor's raw reader keeps the strict - // `DaemonRecord` contract because it deliberately wants to know about a - // malformed `identity` block.) - let (info, from_fallback) = match read_discovery_file_legacy(&discovery_path) { + // `identity` block is forward-compatible noise to this fast path (see + // docs/superpowers/specs/2026-08-13-hyperdb-mcp-agent-ux-design.md, + // "Unknown fields remain forward compatible"). So when the strict + // `DaemonRecord` parse fails, re-parse the bytes the classification already + // carries as a tolerant `DaemonInfo`, which ignores unknown fields. + let info = match read_discovery_file_legacy(&discovery_path) { RawDiscoveryRead::Missing { path } => { tracing::debug!(encoding = ?path.encoding, "daemon discovery file is missing"); return None; @@ -349,48 +370,47 @@ pub fn discover() -> Option { } // `read_discovery_file_legacy` applies no size cap, so it does not // produce `Oversized` today. Folding it in with `Malformed` keeps that - // from mattering: were this reader ever given a cap, a large-but-valid - // record would take the tolerant fallback rather than silently - // becoming an undiscoverable daemon — the exact defect this fallback - // exists to prevent. + // from mattering. RawDiscoveryRead::Oversized { path, contents } | RawDiscoveryRead::Malformed { path, contents } => { let Ok(info) = serde_json::from_slice::(contents.as_slice()) else { tracing::debug!(encoding = ?path.encoding, "daemon discovery file is malformed"); return None; }; - tracing::debug!(encoding = ?path.encoding, "daemon discovery file failed the strict parse; accepted as legacy DaemonInfo"); - (info, true) + tracing::debug!(encoding = ?path.encoding, "daemon discovery file failed the strict parse; accepted as DaemonInfo"); + info } RawDiscoveryRead::Parsed { path, record } => { tracing::debug!(encoding = ?path.encoding, "daemon discovery file parsed (enriched)"); - (record.info().clone(), false) + record.info().clone() } }; - // Validate liveness by connecting to the health port - if is_daemon_alive(info.health_port) { - return Some(info); + let endpoint = HealthEndpoint::from_record(&info.health_endpoint, dir)?; + if is_daemon_alive(&endpoint) { + Some(info) + } else { + None } +} - if from_fallback { - // Never stale-clean a record we could not strictly parse. This branch - // exists precisely because a *newer* daemon may write a record this - // client cannot fully understand, and a single failed 300 ms PING is - // not grounds for destroying it: the daemon rewrites `daemon.json` - // only on `hyperd` restart, so deleting here would hide a live newer - // daemon from every client, not just this one. Leaving it also keeps - // the promise doctor already makes to operators about a record it - // could not parse ("it was left unchanged"). - tracing::debug!( - "leaving the leniently parsed discovery record in place after a failed health check" - ); - return None; +/// Delete a daemon record that a dead daemon left behind. +/// +/// The caller must hold the [`DaemonLock`](super::lock::DaemonLock) for +/// `dir`: a held lock proves no daemon of this shape is running, and holding +/// it keeps one from starting while the record is judged. Only a record of the +/// current shape (it carries `health_endpoint`) is removed. A record in +/// another shape may belong to a daemon from a release that does not take the +/// lock, and is left alone. +pub(super) fn remove_stale_record(dir: &Path) { + let path = dir.join("daemon.json"); + // `Parsed` requires `health_endpoint`, which no earlier shape carries. + if matches!( + read_discovery_file_raw(&path), + RawDiscoveryRead::Parsed { .. } + ) { + let _ = std::fs::remove_file(&path); } - - // Stale file — daemon crashed. Clean up. - let _ = std::fs::remove_file(&discovery_path); - None } /// Remove the discovery file (called during graceful shutdown). @@ -400,57 +420,15 @@ pub fn remove_discovery_file() { } } -/// Check if the daemon is alive by sending PING and verifying the identifying token. -/// No longer accepts a bare TCP connect (prevents collisions with foreign services). -fn is_daemon_alive(port: u16) -> bool { - super::health::ping_identified(port, Duration::from_millis(300), Duration::from_millis(300)) - .is_some() -} - -/// Port scan configuration: a base port and the number of ports to scan. -/// When `span == 1`, the port is pinned (no scan). Used by the later -/// port-scanning stage to discover or spawn a daemon across a range. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PortScan { - pub base: u16, - pub span: u16, -} - -/// Resolve the daemon health port scan configuration from environment or default. -/// If `HYPERDB_DAEMON_PORT` is set and valid, returns a pinned scan (span=1) at -/// that exact port. Otherwise, returns the default base port with the full scan span. -/// -/// `0` is not a valid value here even though `"0".parse::()` succeeds, and -/// is rejected into the same default fallback as unparseable input. A pinned -/// `PortScan { base: 0, span: 1 }` is unsatisfiable by construction: `bind` -/// would take an OS-assigned *ephemeral* port rather than port 0, while every -/// client's scan would keep probing port 0, get a connection error, and read -/// `ProbeResult::Refused` as "free" — so each client that missed the discovery -/// fast path would spawn another daemon-and-`hyperd` pair on another ephemeral -/// port that no scan can find, accumulating them silently. -pub fn resolve_port_scan() -> PortScan { - if let Some(port) = std::env::var(super::ENV_DAEMON_PORT) - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|port| *port != 0) - { - PortScan { - base: port, - span: 1, - } - } else { - PortScan { - base: DEFAULT_DAEMON_BASE_PORT, - span: DAEMON_PORT_SCAN_SPAN, - } - } -} - -/// Resolve the daemon health port from environment or default. Back-compat -/// wrapper for single-port callers; returns the base port from [`resolve_port_scan`]. -/// New code that needs scan-aware logic should call [`resolve_port_scan`] directly. -pub fn resolve_port() -> u16 { - resolve_port_scan().base +/// Check if the daemon is alive by sending PING and verifying the identifying +/// token. Connecting alone is not enough: any program can hold a socket. +fn is_daemon_alive(endpoint: &HealthEndpoint) -> bool { + super::health::ping_identified( + endpoint, + Duration::from_millis(300), + Duration::from_millis(300), + ) + .is_some() } /// Cross-platform home directory resolution. @@ -475,108 +453,6 @@ fn home_dir() -> Option { .map(PathBuf::from) } -/// Result of probing a single port: either our daemon, something else, or refused. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ProbeResult { - /// A hyperdb-mcp daemon answered with valid STATUS. - OurDaemon(Box), - /// The port accepted TCP but isn't our daemon (foreign service or broken STATUS). - Camped, - /// Connection refused (port is free). - Refused, -} - -/// Probe a single port to determine if it's occupied by our daemon, a foreign service, or free. -fn probe_port(port: u16) -> ProbeResult { - let ping_timeout = Duration::from_millis(300); - - if let Some(_version) = super::health::ping_identified(port, ping_timeout, ping_timeout) { - // PING succeeded — something is answering with our token. Now send STATUS - // to retrieve the full daemon info. If STATUS fails we can't trust this - // process (might be a test stub or a broken daemon), so treat it as Camped. - match super::health::send_command_with_timeout(port, "STATUS", ping_timeout, ping_timeout) { - Ok(response) => { - if let Ok(info) = serde_json::from_str::(response.trim()) { - ProbeResult::OurDaemon(Box::new(info)) - } else { - // Parsed PING but STATUS is malformed — treat as Camped. - ProbeResult::Camped - } - } - Err(_) => ProbeResult::Camped, - } - } else { - // PING failed or returned no identifying token. Distinguish "refused" - // from "camped non-daemon" via a raw TCP connect attempt. - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); - match std::net::TcpStream::connect_timeout(&addr, ping_timeout) { - Ok(_) => ProbeResult::Camped, // TCP accepted but PING failed → foreign - Err(_) => ProbeResult::Refused, // Connection refused → port is free - } - } -} - -/// The outcome of scanning a port range for a running daemon or a free port to spawn on. -#[derive(Debug)] -pub enum ScanOutcome { - /// Found a running hyperdb-mcp daemon. - Found(Box), - /// No daemon found, but this port is free (can spawn here). - FreePort(u16), - /// All ports in the range are occupied (either by our daemon, foreign services, or both). - AllOccupied, -} - -/// Scan the configured port range to find a running daemon or identify a free port. -/// If any port in the range answers identified-PING and returns valid STATUS, we return -/// `Found` immediately (first wins). Otherwise, we return `FreePort` with the first -/// refused port encountered, or `AllOccupied` if everything is in use. -/// -/// Product decision: prefer finding an existing daemon anywhere in range over -/// spawning a new one. Only spawn if no daemon exists. -pub fn scan_for_daemon(scan: PortScan) -> ScanOutcome { - let mut first_free: Option = None; - - for offset in 0..scan.span { - let Some(port) = scan.base.checked_add(offset) else { - break; // Overflow guard: stop at u16::MAX - }; - - match probe_port(port) { - ProbeResult::OurDaemon(info) => { - // Found a running daemon — return immediately. - return ScanOutcome::Found(info); - } - ProbeResult::Refused => { - // Port is free. Remember the first one we see. - if first_free.is_none() { - first_free = Some(port); - } - } - ProbeResult::Camped => { - // Port is occupied by something else. Keep scanning. - } - } - } - - // No daemon found. Return the first free port, or AllOccupied if none. - match first_free { - Some(port) => ScanOutcome::FreePort(port), - None => ScanOutcome::AllOccupied, - } -} - -/// Discover a running daemon via the discovery file, or by scanning the configured -/// port range. Returns `None` if no daemon is found in either place. -/// -/// Used by CLI commands (status/stop) that want to find a daemon but not spawn one. -pub fn find_running_daemon() -> Option { - discover().or_else(|| match scan_for_daemon(resolve_port_scan()) { - ScanOutcome::Found(info) => Some(*info), - _ => None, - }) -} - #[cfg(test)] mod tests { use std::ffi::{OsStr, OsString}; @@ -595,7 +471,7 @@ mod tests { DaemonInfo { pid: 4242, hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: 7485, + health_endpoint: "/run/hyperdb/daemon.sock".to_string(), started_at: "2026-08-13T12:34:56Z".to_string(), version: "0.7.0".to_string(), } @@ -691,7 +567,7 @@ mod tests { let old_wire = json!({ "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7485, + "health_endpoint": "/run/hyperdb/daemon.sock", "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0" }); @@ -702,7 +578,7 @@ mod tests { let expected_new_wire = json!({ "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7485, + "health_endpoint": "/run/hyperdb/daemon.sock", "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0", "identity": { @@ -788,22 +664,79 @@ mod tests { ); } - /// Exercises the actual scan, health check, and cleanup logic across a - /// process boundary so `discover()`'s `HYPERDB_STATE_DIR` env override - /// can't race other tests in this binary. - fn run_identity_forward_compat_scenario() { - assert!( - std::env::var_os("HYPERDB_STATE_DIR").is_some(), - "child scenario requires an isolated state directory" - ); + /// A live daemon: a real `HealthListener` on a state directory's endpoint. + struct LiveDaemon { + state: Arc, + thread: Option>, + } + + impl LiveDaemon { + fn start(dir: &Path, info: &DaemonInfo) -> Self { + // Serve exactly the endpoint the record names: deriving a second + // endpoint here would mint a different random pipe name on Windows. + let endpoint = HealthEndpoint::from_record(&info.health_endpoint, dir) + .expect("record names the endpoint of its own state directory"); + let listener = HealthListener::bind(&endpoint).unwrap(); + let state = Arc::new(DaemonState::new(endpoint)); + let info = Arc::new(Mutex::new(info.clone())); + let run_state = Arc::clone(&state); + let thread = std::thread::spawn(move || listener.run(run_state, info)); + Self { + state, + thread: Some(thread), + } + } + fn stop(&mut self) { + self.state.request_shutdown(); + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } + } + + impl Drop for LiveDaemon { + fn drop(&mut self) { + self.stop(); + } + } + + /// A per-test state directory short enough for a Unix socket path. + fn short_state_dir() -> TempDir { + TempDir::new().unwrap() + } + + /// Info whose `health_endpoint` is the endpoint `dir` implies. + fn info_for(dir: &Path, pid: u32) -> DaemonInfo { + DaemonInfo { + pid, + health_endpoint: HealthEndpoint::for_new_daemon(dir) + .unwrap() + .as_str() + .to_string(), + ..legacy_info() + } + } + + fn wire_of(info: &DaemonInfo, identity: &Value) -> Value { + json!({ + "pid": info.pid, + "hyperd_endpoint": info.hyperd_endpoint, + "health_endpoint": info.health_endpoint, + "started_at": info.started_at, + "version": info.version, + "identity": identity, + }) + } + + #[test] + fn discover_tolerates_forward_incompatible_identity_shapes() { // Three real-world shapes a client fast path must still discover a // live daemon through, per the forward-compatibility contract in // docs/superpowers/specs/2026-08-13-hyperdb-mcp-agent-ux-design.md // ("Unknown fields remain forward compatible"). None of these is // valid input for `DaemonBuildIdentity`, so a strict `DaemonRecord` - // parse of the whole file must fail for every one of them — that's - // asserted below as the fixture sanity check. + // parse must fail for every one of them (fixture sanity check). let shapes: [(&str, Value); 3] = [ ("identity retyped as a string", json!("not-an-object")), ( @@ -821,24 +754,10 @@ mod tests { let mut failures = Vec::new(); for (index, (label, identity_json)) in shapes.into_iter().enumerate() { + let tmp = short_state_dir(); let pid = 9_000 + u32::try_from(index).expect("fixture index fits in u32"); - let health_listener = HealthListener::bind(0).unwrap(); - let info = DaemonInfo { - pid, - hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: health_listener.port, - started_at: "2026-08-13T12:34:56Z".to_string(), - version: "0.7.0".to_string(), - }; - - let wire = json!({ - "pid": info.pid, - "hyperd_endpoint": info.hyperd_endpoint, - "health_port": info.health_port, - "started_at": info.started_at, - "version": info.version, - "identity": identity_json, - }); + let info = info_for(tmp.path(), pid); + let wire = wire_of(&info, &identity_json); if serde_json::from_value::(wire.clone()).is_ok() { failures.push(format!( @@ -846,32 +765,24 @@ mod tests { it no longer exercises the forward-compatibility regression" )); } + std::fs::write( + tmp.path().join("daemon.json"), + serde_json::to_vec(&wire).unwrap(), + ) + .unwrap(); - std::fs::create_dir_all(state_dir().unwrap()).unwrap(); - let path = discovery_file_path().unwrap(); - std::fs::write(&path, serde_json::to_vec(&wire).unwrap()).unwrap(); - - let health_state = Arc::new(DaemonState::new()); - let health_info = Arc::new(Mutex::new(info.clone())); - let run_state = Arc::clone(&health_state); - let run_info = Arc::clone(&health_info); - let health_server = - std::thread::spawn(move || health_listener.run(run_state, run_info)); - - match discover() { + let mut daemon = LiveDaemon::start(tmp.path(), &info); + match discover_in(tmp.path()) { Some(discovered) if discovered == info => {} Some(other) => failures.push(format!( - "{label}: discover() returned different facts than the live daemon: {other:?}" + "{label}: discover_in returned different facts than the live daemon: {other:?}" )), None => failures.push(format!( - "{label}: discover() returned None for a live, healthy daemon whose only \ + "{label}: discover_in returned None for a live, healthy daemon whose only \ defect is an unrecognized `identity` block" )), } - - health_state.request_shutdown(); - health_server.join().unwrap(); - let _ = std::fs::remove_file(&path); + daemon.stop(); } assert!( @@ -881,133 +792,112 @@ mod tests { ); } + /// `discover_in` never mutates: a stale record (nothing listening) is left + /// in place for `remove_stale_record`, which runs under the daemon lock. #[test] - fn discover_tolerates_forward_incompatible_identity_shapes() { - const CHILD_SENTINEL_ENV: &str = "HYPERDB_MCP_IDENTITY_FORWARD_COMPAT_CHILD"; - const TEST_NAME: &str = - "daemon::discovery::tests::discover_tolerates_forward_incompatible_identity_shapes"; - - let _process_guard = crate::diagnostics::real_network_test_guard(); - if let Some(marker) = std::env::var_os(CHILD_SENTINEL_ENV) { - std::fs::write(std::path::PathBuf::from(marker), b"started").unwrap(); - run_identity_forward_compat_scenario(); - return; - } - run_discovery_compatibility_child(TEST_NAME, CHILD_SENTINEL_ENV); + fn discover_in_leaves_a_stale_record_alone() { + let tmp = short_state_dir(); + let info = info_for(tmp.path(), 9_101); + write_record_for_test(tmp.path(), &info); + let path = tmp.path().join("daemon.json"); + let before = std::fs::read(&path).unwrap(); + + assert!(discover_in(tmp.path()).is_none()); + assert_eq!(std::fs::read(&path).unwrap(), before); } - /// Returns a port that nothing is listening on, by binding one and - /// releasing it immediately. Anything that later camps on it still fails - /// `ping_identified` (no PONG token), so the port reads as dead either way. - fn released_port() -> u16 { - let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); - listener.local_addr().unwrap().port() + fn write_record_for_test(dir: &Path, info: &DaemonInfo) { + std::fs::write( + dir.join("daemon.json"), + serde_json::to_vec_pretty(info).unwrap(), + ) + .unwrap(); } - fn run_lenient_stale_cleanup_scenario() { - assert!( - std::env::var_os("HYPERDB_STATE_DIR").is_some(), - "child scenario requires an isolated state directory" + /// A record only the lenient `DaemonInfo` fallback can read must survive + /// stale cleanup (it may belong to a newer daemon), while a current-shape + /// record is removed: the guard is narrow, not a blanket disable. + #[test] + fn remove_stale_record_preserves_a_leniently_parsed_record() { + let tmp = short_state_dir(); + let path = tmp.path().join("daemon.json"); + let info = info_for(tmp.path(), 9_100); + let lenient_wire = wire_of(&info, &json!("not-an-object")); + assert!(serde_json::from_value::(lenient_wire.clone()).is_err()); + assert!(serde_json::from_value::(lenient_wire.clone()).is_ok()); + let lenient_bytes = serde_json::to_vec(&lenient_wire).unwrap(); + std::fs::write(&path, &lenient_bytes).unwrap(); + + assert!(discover_in(tmp.path()).is_none()); + remove_stale_record(tmp.path()); + assert_eq!( + std::fs::read(&path).unwrap(), + lenient_bytes, + "a record this client could only read leniently must not be deleted" ); - let path = discovery_file_path().unwrap(); - let mut failures = Vec::new(); + write_record_for_test(tmp.path(), &info); + remove_stale_record(tmp.path()); + assert!(!path.exists(), "a strict current-shape record is stale"); + } - // A record whose `identity` block this client cannot understand, but - // whose legacy fields are perfectly readable — exactly what a *newer* - // daemon looks like to an older client, which is the only reason the - // lenient fallback exists. - let lenient_wire = json!({ - "pid": 9_100, + /// A record in a shape without `health_endpoint` (an earlier release) is + /// not this build's to delete. + #[test] + fn remove_stale_record_leaves_a_legacy_shaped_record() { + let tmp = short_state_dir(); + let path = tmp.path().join("daemon.json"); + let bytes = serde_json::to_vec(&json!({ + "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": released_port(), + "health_port": 7485, "started_at": "2026-08-13T12:34:56Z", - "version": "0.7.0", - "identity": "not-an-object", - }); - if serde_json::from_value::(lenient_wire.clone()).is_ok() { - failures.push( - "fixture satisfied the strict DaemonRecord parse; it no longer reaches the \ - lenient fallback this test is about" - .to_string(), - ); - } - if serde_json::from_value::(lenient_wire.clone()).is_err() { - failures.push( - "fixture is not readable as a legacy DaemonInfo either; it would be rejected \ - before the branch this test pins" - .to_string(), - ); - } - - std::fs::create_dir_all(state_dir().unwrap()).unwrap(); - let lenient_bytes = serde_json::to_vec(&lenient_wire).unwrap(); - std::fs::write(&path, &lenient_bytes).unwrap(); - - if let Some(info) = discover() { - failures.push(format!( - "discover() reported a live daemon on a released port: {info:?}" - )); - } - // The pin: a failed health check must not destroy a record this - // client could only read leniently. The daemon rewrites `daemon.json` - // only on `hyperd` restart, so deleting it here would hide a live, - // newer daemon from every client on the machine. - match std::fs::read(&path) { - Ok(after) if after == lenient_bytes => {} - Ok(_) => { - failures.push("discover() changed the leniently parsed record's bytes".to_string()); - } - Err(error) => failures.push(format!( - "discover() removed the leniently parsed discovery record it could not \ - strictly parse: {error}" - )), - } + "version": "0.7.0" + })) + .unwrap(); + std::fs::write(&path, &bytes).unwrap(); - // Contrast: a strictly parseable record on a dead port is still - // stale-cleaned, so the guard above is narrow rather than a blanket - // disabling of cleanup. - let mut strict_info = legacy_info(); - strict_info.pid = 9_101; - strict_info.health_port = released_port(); - write_discovery_file(&strict_info).unwrap(); - if let Some(info) = discover() { - failures.push(format!( - "discover() reported a live daemon for the strict record on a released port: {info:?}" - )); - } - if path.exists() { - failures.push( - "discover() did not stale-clean a strictly parsed record on a dead port" - .to_string(), - ); - } + assert!(discover_in(tmp.path()).is_none()); + remove_stale_record(tmp.path()); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + } - assert!( - failures.is_empty(), - "lenient stale-cleanup failures:\n{}", - failures.join("\n") - ); + /// A record pointing at an endpoint this state directory does not imply is + /// never connected to, even if something answers there. + #[test] + fn discover_in_rejects_a_foreign_endpoint() { + let live_dir = short_state_dir(); + let other_dir = short_state_dir(); + let live_info = info_for(live_dir.path(), 9_200); + let _daemon = LiveDaemon::start(live_dir.path(), &live_info); + + // `other_dir`'s record names `live_dir`'s live socket. + write_record_for_test(other_dir.path(), &live_info); + assert!(discover_in(other_dir.path()).is_none()); + // And the live daemon is found through its own directory. + write_record_for_test(live_dir.path(), &live_info); + assert_eq!(discover_in(live_dir.path()), Some(live_info)); } - /// A record that only the lenient `DaemonInfo` fallback could read must - /// survive a failed health check. Before the fallback existed, a strict - /// parse failure returned from `discover()` before ever reaching the - /// stale-cleanup `remove_file`, so tolerating the record must not also - /// start deleting it. + /// A state directory another account can write to is not trusted. + #[cfg(unix)] #[test] - fn discover_preserves_a_leniently_parsed_stale_record() { - const CHILD_SENTINEL_ENV: &str = "HYPERDB_MCP_LENIENT_STALE_CLEANUP_CHILD"; - const TEST_NAME: &str = - "daemon::discovery::tests::discover_preserves_a_leniently_parsed_stale_record"; + fn discover_in_ignores_an_untrusted_state_directory() { + use std::os::unix::fs::PermissionsExt as _; - let _process_guard = crate::diagnostics::real_network_test_guard(); - if let Some(marker) = std::env::var_os(CHILD_SENTINEL_ENV) { - std::fs::write(std::path::PathBuf::from(marker), b"started").unwrap(); - run_lenient_stale_cleanup_scenario(); - return; - } - run_discovery_compatibility_child(TEST_NAME, CHILD_SENTINEL_ENV); + let tmp = short_state_dir(); + let info = info_for(tmp.path(), 9_300); + let mut daemon = LiveDaemon::start(tmp.path(), &info); + write_record_for_test(tmp.path(), &info); + assert_eq!(discover_in(tmp.path()), Some(info)); + + std::fs::set_permissions(tmp.path(), std::fs::Permissions::from_mode(0o770)).unwrap(); + assert!( + discover_in(tmp.path()).is_none(), + "a group-writable state directory must not be trusted" + ); + std::fs::set_permissions(tmp.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + daemon.stop(); } #[test] @@ -1027,7 +917,7 @@ mod tests { let parsed_bytes = serde_json::to_vec(&json!({ "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7485, + "health_endpoint": "/run/hyperdb/daemon.sock", "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0" })) @@ -1162,7 +1052,7 @@ mod tests { let base_bytes = serde_json::to_vec(&json!({ "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7485, + "health_endpoint": "/run/hyperdb/daemon.sock", "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0", "ignored_padding": "" @@ -1229,146 +1119,48 @@ mod tests { ); } - fn run_oversized_discovery_compatibility_scenario() { - const RAW_DOCTOR_LIMIT_BYTES: usize = 64 * 1024; - - assert!( - std::env::var_os("HYPERDB_STATE_DIR").is_some(), - "child scenario requires an isolated state directory" - ); - let health_listener = HealthListener::bind(0).unwrap(); - let health_port = health_listener.port; - let oversized_info = DaemonInfo { - pid: 5_252, - hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port, - started_at: "2026-08-13T12:34:56Z".to_string(), - version: "v".repeat(RAW_DOCTOR_LIMIT_BYTES + 1), - }; - write_discovery_file(&oversized_info).unwrap(); - let path = discovery_file_path().unwrap(); - let original_bytes = std::fs::read(&path).unwrap(); - let mut failures = Vec::new(); - - if original_bytes.len() <= RAW_DOCTOR_LIMIT_BYTES { - failures.push(format!( - "public writer produced only {} bytes, expected more than {RAW_DOCTOR_LIMIT_BYTES}", - original_bytes.len() - )); - } - match serde_json::from_slice::(&original_bytes) { - Ok(parsed) if parsed == oversized_info => {} - Ok(_) => { - failures.push("oversized public DaemonInfo did not round-trip exactly".to_string()); - } - Err(error) => failures.push(format!( - "public writer did not produce valid oversized DaemonInfo JSON: {error}" - )), - } - match read_discovery_file_raw(&path) { - RawDiscoveryRead::Oversized { path: reported, .. } - if reported == ReportedPath::from_os_str(path.as_os_str()) => {} - RawDiscoveryRead::Missing { .. } => { - failures - .push("doctor raw reader reported the oversized record missing".to_string()); - } - RawDiscoveryRead::Unreadable { kind, .. } => failures.push(format!( - "doctor raw reader reported the oversized record unreadable: {kind:?}" - )), - RawDiscoveryRead::Malformed { .. } => { - failures.push( - "doctor raw reader misclassified the well-formed oversized record as malformed" - .to_string(), - ); - } - RawDiscoveryRead::Oversized { .. } => { - failures.push("doctor raw reader reported the wrong oversized path".to_string()); - } - RawDiscoveryRead::Parsed { .. } => { - failures.push("doctor raw reader accepted the oversized record".to_string()); - } - } - match std::fs::read(&path) { - Ok(after) if after == original_bytes => {} - Ok(_) => failures.push("doctor raw reader changed oversized bytes".to_string()), - Err(error) => failures.push(format!( - "doctor raw reader removed the oversized record: {error}" - )), - } - - let health_state = Arc::new(DaemonState::new()); - let health_info = Arc::new(Mutex::new(oversized_info.clone())); - let run_state = Arc::clone(&health_state); - let run_info = Arc::clone(&health_info); - let health_server = std::thread::spawn(move || health_listener.run(run_state, run_info)); - - match discover() { - Some(info) if info == oversized_info => {} - Some(_) => { - failures - .push("normal discover returned different live oversized facts".to_string()); - } - None => failures - .push("normal discover did not accept the live oversized record".to_string()), - } - match std::fs::read(&path) { - Ok(after) if after == original_bytes => {} - Ok(_) => failures.push("live discover changed oversized bytes".to_string()), - Err(error) => failures.push(format!( - "live discover removed the oversized record: {error}" - )), - } - - health_state.request_shutdown(); - health_server.join().unwrap(); - if discover().is_some() { - failures.push("stopped oversized record was incorrectly retained as live".to_string()); - } - if path.exists() { - failures - .push("normal discover did not stale-clean the valid oversized record".to_string()); - } - - assert!( - failures.is_empty(), - "oversized legacy discover failures:\n{}", - failures.join("\n") - ); - } - + /// A valid record larger than the doctor's raw-read cap is still a live + /// daemon for the client fast path, and is never rewritten by it. #[test] - fn discover_preserves_legacy_oversized_stale_cleanup() { - const CHILD_SENTINEL_ENV: &str = "HYPERDB_MCP_OVERSIZED_DISCOVERY_COMPATIBILITY_CHILD"; - const TEST_NAME: &str = - "daemon::discovery::tests::discover_preserves_legacy_oversized_stale_cleanup"; + fn discover_in_accepts_a_live_oversized_record() { + const RAW_DOCTOR_LIMIT_BYTES: usize = 64 * 1024; - let _process_guard = crate::diagnostics::real_network_test_guard(); - if let Some(marker) = std::env::var_os(CHILD_SENTINEL_ENV) { - std::fs::write(std::path::PathBuf::from(marker), b"started").unwrap(); - run_oversized_discovery_compatibility_scenario(); - return; - } - run_discovery_compatibility_child(TEST_NAME, CHILD_SENTINEL_ENV); + let tmp = short_state_dir(); + let mut oversized = info_for(tmp.path(), 5_252); + oversized.version = "v".repeat(RAW_DOCTOR_LIMIT_BYTES + 1); + let path = tmp.path().join("daemon.json"); + write_record_for_test(tmp.path(), &oversized); + let original = std::fs::read(&path).unwrap(); + assert!(original.len() > RAW_DOCTOR_LIMIT_BYTES); + + assert!(matches!( + read_discovery_file_raw(&path), + RawDiscoveryRead::Oversized { .. } + )); + let mut daemon = LiveDaemon::start(tmp.path(), &oversized); + assert_eq!(discover_in(tmp.path()), Some(oversized)); + assert_eq!(std::fs::read(&path).unwrap(), original); + + daemon.stop(); + assert!(discover_in(tmp.path()).is_none()); + // Oversized records are not the current-shape fast path: left alone. + remove_stale_record(tmp.path()); + assert_eq!(std::fs::read(&path).unwrap(), original); } + /// The client fast path follows a discovery symlink; the doctor reader + /// rejects it without touching either the link or its target. #[cfg(unix)] - fn run_symlink_discovery_compatibility_scenario() { - assert!( - std::env::var_os("HYPERDB_STATE_DIR").is_some(), - "child scenario requires an isolated state directory" - ); - let health_listener = HealthListener::bind(0).unwrap(); - let health_port = health_listener.port; - let mut linked_info = legacy_info(); - linked_info.pid = 6_363; - linked_info.health_port = health_port; - write_discovery_file(&linked_info).unwrap(); - let link_path = discovery_file_path().unwrap(); - let target_path = link_path.with_file_name("legacy-daemon-target.json"); + #[test] + fn discover_in_follows_a_live_discovery_symlink() { + let tmp = short_state_dir(); + let linked = info_for(tmp.path(), 6_363); + write_record_for_test(tmp.path(), &linked); + let link_path = tmp.path().join("daemon.json"); + let target_path = tmp.path().join("legacy-daemon-target.json"); std::fs::rename(&link_path, &target_path).unwrap(); std::os::unix::fs::symlink(&target_path, &link_path).unwrap(); let target_bytes = std::fs::read(&target_path).unwrap(); - let mut failures = Vec::new(); match read_discovery_file_raw(&link_path) { RawDiscoveryRead::Unreadable { @@ -1376,90 +1168,29 @@ mod tests { kind, } if reported == ReportedPath::from_os_str(link_path.as_os_str()) && kind != io::ErrorKind::NotFound => {} - other => failures.push(format!( - "doctor raw reader did not reject the symlink without following it: {other:?}" - )), - } - match std::fs::symlink_metadata(&link_path) { - Ok(metadata) if metadata.file_type().is_symlink() => {} - Ok(metadata) => failures.push(format!( - "doctor raw reader replaced the link with {:?}", - metadata.file_type() - )), - Err(error) => failures.push(format!( - "doctor raw reader removed the discovery symlink: {error}" - )), - } - match std::fs::read(&target_path) { - Ok(after) if after == target_bytes => {} - Ok(_) => failures.push("doctor raw reader changed symlink target bytes".to_string()), - Err(error) => failures.push(format!( - "doctor raw reader removed the symlink target: {error}" - )), - } - - let health_state = Arc::new(DaemonState::new()); - let health_info = Arc::new(Mutex::new(linked_info.clone())); - let run_state = Arc::clone(&health_state); - let run_info = Arc::clone(&health_info); - let health_server = std::thread::spawn(move || health_listener.run(run_state, run_info)); - - match discover() { - Some(info) if info == linked_info => {} - other => failures.push(format!( - "normal discover did not follow the live valid symlink: {other:?}" - )), - } - match std::fs::symlink_metadata(&link_path) { - Ok(metadata) if metadata.file_type().is_symlink() => {} - Ok(_) => failures.push("live discover replaced the discovery symlink".to_string()), - Err(error) => failures.push(format!( - "live discover removed the discovery symlink: {error}" - )), - } - - health_state.request_shutdown(); - health_server.join().unwrap(); - if discover().is_some() { - failures.push("stopped symlinked record was incorrectly retained as live".to_string()); - } - match std::fs::symlink_metadata(&link_path) { - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Ok(_) => failures - .push("normal discover did not remove the stale discovery symlink".to_string()), - Err(error) => failures.push(format!( - "stale discovery symlink cleanup failed unexpectedly: {error}" - )), - } - match std::fs::read(&target_path) { - Ok(after) if after == target_bytes => {} - Ok(_) => failures.push("normal discover changed symlink target bytes".to_string()), - Err(error) => failures.push(format!( - "normal discover removed the symlink target instead of the link: {error}" - )), + other => panic!("doctor reader did not reject the symlink: {other:?}"), } + let mut daemon = LiveDaemon::start(tmp.path(), &linked); + assert_eq!(discover_in(tmp.path()), Some(linked)); assert!( - failures.is_empty(), - "symlink legacy discover failures:\n{}", - failures.join("\n") + std::fs::symlink_metadata(&link_path) + .unwrap() + .file_type() + .is_symlink() ); - } - - #[cfg(unix)] - #[test] - fn discover_preserves_legacy_symlink_stale_cleanup() { - const CHILD_SENTINEL_ENV: &str = "HYPERDB_MCP_SYMLINK_DISCOVERY_COMPATIBILITY_CHILD"; - const TEST_NAME: &str = - "daemon::discovery::tests::discover_preserves_legacy_symlink_stale_cleanup"; - let _process_guard = crate::diagnostics::real_network_test_guard(); - if let Some(marker) = std::env::var_os(CHILD_SENTINEL_ENV) { - std::fs::write(std::path::PathBuf::from(marker), b"started").unwrap(); - run_symlink_discovery_compatibility_scenario(); - return; - } - run_discovery_compatibility_child(TEST_NAME, CHILD_SENTINEL_ENV); + daemon.stop(); + assert!(discover_in(tmp.path()).is_none()); + // Cleanup refuses to follow or remove a symlinked record. + remove_stale_record(tmp.path()); + assert!( + std::fs::symlink_metadata(&link_path) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(std::fs::read(&target_path).unwrap(), target_bytes); } #[cfg(unix)] diff --git a/hyperdb-mcp/src/daemon/health.rs b/hyperdb-mcp/src/daemon/health.rs index 3fb092a9..ff15fa0e 100644 --- a/hyperdb-mcp/src/daemon/health.rs +++ b/hyperdb-mcp/src/daemon/health.rs @@ -1,74 +1,64 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! TCP health listener for the daemon. +//! Health listener for the daemon. //! -//! The health listener serves two purposes: -//! 1. **Single-instance lock** — binding the port guarantees at most one daemon per user. -//! 2. **Liveness probe + heartbeat** — clients connect and send simple text commands. +//! The listener serves a liveness probe and a small command channel on the +//! daemon's [control endpoint](super::control): a Unix domain socket in the +//! state directory (mode `0600`, inside a directory that is owned by the +//! current user and closed to group and others), or a named pipe whose DACL +//! admits only the current user on Windows. Other local users cannot open +//! either, so `STOP`, `STATUS` and `REPORT_HYPERD_ERROR` are the owning user's +//! business only. There is no TCP port. +//! +//! The listener is not the single-instance guard; that is the +//! [`DaemonLock`](super::lock::DaemonLock), which the daemon holds for its +//! whole life. The listener only answers the people who already found the +//! endpoint through `daemon.json`. //! //! Protocol (line-based, newline-terminated): -//! - `PING\n` → `PONG hyperdb-mcp \n` (liveness check; the identifying -//! token proves it's a hyperdb-mcp daemon, not a foreign process on the same port) -//! - `HEARTBEAT\n` → `OK\n` (resets idle timer) -//! - `STOP\n` → `STOPPING\n` (triggers graceful shutdown) -//! - `STATUS\n` → JSON line with daemon info (reports the *current* hyperd +//! - `PING\n` -> `PONG hyperdb-mcp \n` (liveness check; the identifying +//! token proves it's a hyperdb-mcp daemon, not some other program that +//! happens to answer on the endpoint) +//! - `HEARTBEAT\n` -> `OK\n` (resets idle timer) +//! - `STOP\n` -> `STOPPING\n` (triggers graceful shutdown) +//! - `STATUS\n` -> JSON line with daemon info (reports the *current* hyperd //! endpoint, which can change after a restart). -//! - `REPORT_HYPERD_ERROR\n` → `OK\n` (sets the restart-requested flag — +//! - `REPORT_HYPERD_ERROR\n` -> `OK\n` (sets the restart-requested flag -- //! the monitor task picks it up on its next tick). +//! +//! Known limits, accepted because only the owning user can reach the channel: +//! the server does not bound the length of a request line or the number of +//! connection threads. use std::io::{BufRead, BufReader, Read, Write}; -use std::net::{TcpListener, TcpStream}; -use std::sync::atomic::{AtomicBool, AtomicU16, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tracing::{debug, warn}; +use super::control::{self, CONNECTION_IO_TIMEOUT, ControlListener, ControlStream, HealthEndpoint}; use super::discovery::{DaemonInfo, DaemonRecord}; -/// Identifying token included in PONG responses. Used to verify that a bound -/// port is owned by a hyperdb-mcp daemon (not a foreign service). +/// Identifying token included in PONG responses. Used to verify that an +/// endpoint is served by a hyperdb-mcp daemon (not some other program). pub const PONG_TOKEN: &str = "hyperdb-mcp"; -/// Bound on the throwaway loopback connect that wakes an accept loop waiting -/// for readiness. -/// -/// The common cases are fast: the kernel completes a loopback connection into -/// the listen backlog without waiting for `accept()` (measured at ~140 µs), and -/// a listener that has already gone away refuses the connection in ~8 ms. The -/// worst case is *not* fast, though — with a full backlog the connect blocks -/// for this entire duration rather than failing, measured at 251 ms on macOS -/// against a `listen(1)` socket with nothing accepting. That is why -/// [`DaemonState::wake_accept_loop`] runs the connect on a detached thread: a -/// 250 ms stall must not land on the tokio worker that calls -/// `request_shutdown`, nor delay the `STOPPING` reply that `handle_client` -/// writes right after it. -const WAKE_CONNECT_TIMEOUT: Duration = Duration::from_millis(250); - -/// Upper bound on how long [`HealthListener::run`] waits for accept readiness -/// before re-checking `should_shutdown` under its own power. +/// Upper bound on how long [`HealthListener::run`] waits for a client before +/// re-checking `should_shutdown` under its own power. /// /// This is a deliberate correctness floor, not a leftover of the old 5 ms poll. /// [`DaemonState::request_shutdown`]'s wake connection is what makes shutdown -/// *prompt* (sub-millisecond), but every wake can be lost — an unregistered -/// `wake_port`, a listener sharing a `DaemonState` with another one, a connect -/// that never lands. Since every `request_shutdown` caller in the tree follows -/// it with a `join()`, a lost wake against a purely wake-driven loop is a -/// permanent hang rather than a slow shutdown. One second bounds that -/// unconditionally while still costing only one wakeup per second, versus the -/// ~174/s the 5 ms poll cost (see issue #274 for the measurements). -#[cfg(unix)] +/// *prompt* (sub-millisecond), but every wake can be lost -- no listener +/// registered yet, a listener sharing a `DaemonState` with another one, a +/// connect that never lands. Since every `request_shutdown` caller in the tree +/// follows it with a `join()`, a lost wake against a purely wake-driven loop is +/// a permanent hang rather than a slow shutdown, and the daemon lock is held +/// until that join returns. One second bounds that unconditionally while still +/// costing only one wakeup per second (see issue #274 for the measurements). const ACCEPT_POLL_INTERVAL: Duration = Duration::from_secs(1); -/// Windows keeps the historical 5 ms cadence: [`wait_for_accept_readiness`] -/// there is a plain sleep rather than a readiness wait, so this interval also -/// bounds how long a real client waits to be accepted. Stretching it to a -/// second would trade a shutdown bound for a second of accept latency, so -/// Windows stays exactly where it shipped and only Unix takes the win. -#[cfg(not(unix))] -const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(5); - /// Construct the PONG response with the identifying token and version. fn pong_response() -> String { format!("PONG {PONG_TOKEN} {}\n", crate::version::MCP_VERSION) @@ -77,8 +67,8 @@ fn pong_response() -> String { /// Handle to the health listener, used to check binding success and manage lifecycle. #[derive(Debug)] pub struct HealthListener { - listener: TcpListener, - pub port: u16, + listener: ControlListener, + endpoint: HealthEndpoint, } /// Shared state between the health listener and the daemon main loop. @@ -87,8 +77,8 @@ pub struct HealthListener { /// invariant. `shutdown` in particular *must not* be settable directly: a bare /// `store(true)` looks identical to `request_shutdown()` through /// `should_shutdown()`, but skips the wake, so the listener stays in its -/// readiness wait for a further `ACCEPT_POLL_INTERVAL` instead of returning -/// at once. The other two are private for the same reason — one way in, one +/// accept wait for a further `ACCEPT_POLL_INTERVAL` instead of returning +/// at once. The other two are private for the same reason -- one way in, one /// way out, so the pairing cannot be bypassed by a caller who only sees the /// field. #[derive(Debug)] @@ -104,27 +94,26 @@ pub struct DaemonState { /// [`Self::request_restart`], read-and-cleared only by /// [`Self::consume_restart_request`]. restart_requested: AtomicBool, - /// Loopback port to connect to in order to wake a [`HealthListener::run`] - /// loop waiting for accept readiness. `0` is an unambiguous - /// "no listener registered" sentinel: [`HealthListener::bind`] resolves an - /// ephemeral `bind(0)` through `local_addr()`, so a bound listener's port - /// is never `0`. - wake_port: AtomicU16, -} - -impl Default for DaemonState { - fn default() -> Self { - Self::new() - } + /// Where the listener this state serves is bound, fixed at construction so + /// [`Self::request_shutdown`] can connect to it to wake the accept loop. + /// (A path cannot live in an atomic, so what changes at run time is + /// [`Self::listener_registered`], not this.) + endpoint: HealthEndpoint, + /// Whether a [`HealthListener::run`] loop is currently serving + /// [`Self::endpoint`] and so can be woken by connecting to it. + listener_registered: AtomicBool, } impl DaemonState { - pub fn new() -> Self { + /// A fresh state for a daemon whose health listener is (or will be) bound + /// at `endpoint`. + pub fn new(endpoint: HealthEndpoint) -> Self { Self { last_activity: Mutex::new(Instant::now()), shutdown: AtomicBool::new(false), restart_requested: AtomicBool::new(false), - wake_port: AtomicU16::new(0), + endpoint, + listener_registered: AtomicBool::new(false), } } @@ -144,8 +133,7 @@ impl DaemonState { self.last_activity.lock().expect("mutex poisoned").elapsed() } - /// Request shutdown and wake a health listener waiting for accept - /// readiness. + /// Request shutdown and wake a health listener waiting in its accept. /// /// Every caller in the tree follows this with a `join()` on the listener /// thread (`run_daemon`'s step 7, and the test harnesses), so the wake is @@ -156,21 +144,22 @@ impl DaemonState { /// /// The `SeqCst` here is load-bearing and must not be relaxed to /// `Release`/`Acquire`. This half stores `shutdown` then loads - /// `wake_port`; [`HealthListener::run`] stores `wake_port` then loads - /// `shutdown`. That is Dekker's algorithm: each thread writes its own flag - /// and reads the other's. Release/Acquire orders store→store and - /// load→load, but says nothing about a store followed by a load of a - /// *different* location, so both loads may return stale values in the same - /// interleaving — `run` sees no shutdown and waits, while this sees no - /// wake port and connects to nobody. Only `SeqCst` gives the single total - /// order that rules the interleaving out. + /// `listener_registered`; [`HealthListener::run`] stores + /// `listener_registered` then loads `shutdown`. That is Dekker's + /// algorithm: each thread writes its own flag and reads the other's. + /// Release/Acquire orders store->store and load->load, but says nothing + /// about a store followed by a load of a *different* location, so both + /// loads may return stale values in the same interleaving -- `run` sees no + /// shutdown and waits, while this sees no registered listener and connects + /// to nobody. Only `SeqCst` gives the single total order that rules the + /// interleaving out. /// /// Both mainstream targets really do permit the reordering, which is worth /// recording because the arm64 half is easy to get wrong. On x86-64 a /// `Release` store is a plain `mov` with nothing between it and the /// following load, so the store buffer may sink it past that load; a /// `SeqCst` store is an `xchg`, which is a full barrier. On arm64 the - /// store is `stlr` either way — but LLVM lowers an `Acquire` load to + /// store is `stlr` either way -- but LLVM lowers an `Acquire` load to /// `ldapr` (RCpc) wherever FEAT_LRCPC is available, Apple Silicon /// included, and `ldapr` is specifically *not* ordered against a preceding /// `stlr`. Only the `ldar` (RCsc) that `SeqCst` emits carries that @@ -182,62 +171,58 @@ impl DaemonState { self.wake_accept_loop(); } - /// Nudge the listening socket so a waiting `accept()` becomes ready and - /// the loop re-reads `should_shutdown`. + /// Nudge the listening endpoint so a waiting accept returns and the loop + /// re-reads `should_shutdown`. /// - /// Runs on a detached thread. The connect is usually immediate, but its - /// worst case is a full [`WAKE_CONNECT_TIMEOUT`] (see that constant), and - /// this is called both from `run_daemon`'s async shutdown path — where it - /// would stall a tokio worker — and from the `STOP` handler, ahead of the - /// `STOPPING` reply. Neither can afford to wait on it, and nothing needs - /// the result: shutdown is bounded by the loop's own readiness timeout - /// whether or not the wake ever lands. + /// Runs on a detached thread. A Unix socket connect is immediate, but a + /// Windows pipe connect can wait for a busy instance, and this is called + /// both from `run_daemon`'s async shutdown path -- where it would stall a + /// tokio worker -- and from the `STOP` handler, ahead of the `STOPPING` + /// reply. Neither can afford to wait on it, and nothing needs the result: + /// shutdown is bounded by the loop's own accept timeout whether or not the + /// wake ever lands. /// /// Every failure mode is benign and ignored: no listener registered yet - /// (`wake_port == 0`, and the loop's pre-`accept` `should_shutdown` check - /// catches that), the listener already gone (connection refused), or a full - /// backlog — in which case the loop still becomes ready through one of the - /// queued connections. The wake makes shutdown prompt; correctness rests on - /// the flag check the loop performs on every pass. + /// (the loop's pre-accept `should_shutdown` check catches that), or the + /// listener already gone (connection refused). The wake makes shutdown + /// prompt; correctness rests on the flag check the loop performs on every + /// pass. fn wake_accept_loop(&self) { // Read on the caller's thread, not the spawned one: this load is the // second half of the `SeqCst` pairing documented on // `request_shutdown`, and moving it off-thread would break it. - let port = self.wake_port.load(Ordering::SeqCst); - if port == 0 { + if !self.listener_registered.load(Ordering::SeqCst) { return; } - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); + let endpoint = self.endpoint.clone(); // `Builder::spawn` rather than `thread::spawn`: a best-effort wake must // not panic its caller if the process is out of threads. Losing the // wake costs at most one `ACCEPT_POLL_INTERVAL` of shutdown latency. let spawned = std::thread::Builder::new() .name("health-wake".to_string()) - .spawn(move || { - let _ = TcpStream::connect_timeout(&addr, WAKE_CONNECT_TIMEOUT); - }); + .spawn(move || ControlListener::wake(&endpoint)); if let Err(error) = spawned { debug!(%error, "could not spawn the health listener wake connection"); } } - /// Point [`Self::wake_accept_loop`] at a live listening port. Called by + /// Mark the listener as serving [`Self::endpoint`]. Called by /// [`HealthListener::run`] before its first `should_shutdown` check. /// /// `SeqCst` for the reason spelled out on [`Self::request_shutdown`]: this /// store is the first half of the Dekker pairing whose second half is the /// `shutdown` load in [`HealthListener::run`]. - fn register_wake_port(&self, port: u16) { - self.wake_port.store(port, Ordering::SeqCst); + fn register_listener(&self) { + self.listener_registered.store(true, Ordering::SeqCst); } - /// Forget the wake port once the listener has stopped, so a later - /// `request_shutdown` does not connect to whatever has since taken it. - fn clear_wake_port(&self) { - self.wake_port.store(0, Ordering::SeqCst); + /// Forget the listener once it has stopped, so a later `request_shutdown` + /// does not connect to whatever has since taken the endpoint. + fn clear_listener(&self) { + self.listener_registered.store(false, Ordering::SeqCst); } - /// `SeqCst` to match [`Self::request_shutdown`]'s store — see the memory + /// `SeqCst` to match [`Self::request_shutdown`]'s store -- see the memory /// ordering note there. pub fn should_shutdown(&self) -> bool { self.shutdown.load(Ordering::SeqCst) @@ -256,70 +241,70 @@ impl DaemonState { } impl HealthListener { - /// Try to bind the health port. + /// Bind the control endpoint. /// - /// The listening socket is non-blocking. [`Self::run`] waits for accept - /// readiness with a timeout rather than parking in `accept()` itself, so - /// that a lost shutdown wake costs one `ACCEPT_POLL_INTERVAL` instead of - /// blocking the loop in the kernel forever; - /// `bind_leaves_the_listening_socket_nonblocking` pins that precondition. - /// Passing `0` binds an ephemeral port, which `local_addr()` below - /// resolves into [`Self::port`], so a bound listener's port is never `0`. + /// The caller must hold the [`DaemonLock`](super::lock::DaemonLock): on + /// Unix a leftover socket file from a dead daemon is removed first, which + /// is only safe while no other daemon can be starting. /// /// # Errors - /// Returns `Err` if the port is already in use (another daemon is running) - /// or the bind fails for another reason. - pub fn bind(port: u16) -> std::io::Result { - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); - let listener = TcpListener::bind(addr)?; - listener.set_nonblocking(true)?; - let port = listener.local_addr()?.port(); - Ok(Self { listener, port }) + /// Returns `Err` if something live already serves the endpoint, the path is + /// too long for a Unix socket, or the bind fails for another reason. + pub fn bind(endpoint: &HealthEndpoint) -> std::io::Result { + Ok(Self { + listener: ControlListener::bind(endpoint)?, + endpoint: endpoint.clone(), + }) + } + + /// The endpoint this listener serves. + pub fn endpoint(&self) -> &HealthEndpoint { + &self.endpoint } /// Run the health listener loop. Spawns per-connection threads until shutdown. /// Consumes `self` because this is intended to be called from a dedicated thread. /// /// `info` is shared (`Arc>`) so the listener reports the - /// *current* hyperd endpoint after a restart — the monitor task updates the + /// *current* hyperd endpoint after a restart -- the monitor task updates the /// same Arc once a new hyperd is running. /// - /// The loop does not poll on a timer. It waits for the listening socket to - /// become readable — for up to `ACCEPT_POLL_INTERVAL` at a time — so an - /// arriving client is accepted with no added latency and an idle daemon - /// costs one wakeup per second instead of the ~174/s the old 5 ms sleep - /// cost (see issue #274 for the measurements). + /// The loop does not poll on a timer. It waits for a client for up to + /// `ACCEPT_POLL_INTERVAL` at a time, so an arriving client is accepted with + /// no added latency and an idle daemon costs one wakeup per second. + /// + /// [`DaemonState::request_shutdown`] makes a throwaway connection to this + /// listener, which returns the wait immediately; the loop re-checks + /// `should_shutdown` on every pass and drops the wake connection unread. + /// The accept timeout is the floor underneath that: it bounds shutdown at + /// one interval even when the wake never arrives, which matters because + /// every `request_shutdown` caller joins this thread. /// - /// [`DaemonState::request_shutdown`] makes a throwaway loopback connection - /// to this listener, which returns the wait immediately; the loop - /// re-checks `should_shutdown` on every pass and drops the wake connection - /// unread. The readiness timeout is the floor underneath that: it bounds - /// shutdown at one interval even when the wake never arrives, which - /// matters because every `request_shutdown` caller joins this thread. + /// Dropping the listener when this returns removes the socket file (Unix). #[expect( clippy::needless_pass_by_value, reason = "Arcs are cloned into per-connection threads" )] - pub fn run(self, state: Arc, info: Arc>) { + pub fn run(mut self, state: Arc, info: Arc>) { // Register before the first `should_shutdown` check, so a shutdown - // racing this setup either finds the port (and wakes us) or is seen by - // the check below before we ever wait. The guard clears it again on - // every exit path, including an unwind out of the loop — dropping - // `self.listener` frees the port, and a `wake_port` still naming it - // would send a later `request_shutdown` to whatever took it next. - let _wake_port = WakePortRegistration::register(&state, self.port); + // racing this setup either finds the listener (and wakes it) or is seen + // by the check below before we ever wait. The guard clears it again on + // every exit path, including an unwind out of the loop -- dropping the + // listener frees the endpoint, and a registration still naming it would + // send a later `request_shutdown` to whatever took it next. + let _registration = ListenerRegistration::register(&state); loop { if state.should_shutdown() { break; } - match accept_and_force_blocking(&self.listener) { - Ok(AcceptedConnection::Ready(stream)) => { + match self.listener.accept_timeout(ACCEPT_POLL_INTERVAL) { + Ok(Some(stream)) => { // The shutdown wake arrives as an ordinary connection. // Re-check before spending a thread on it; a genuine // client arriving in the same instant is dropped, exactly - // as the pre-`accept` check has always dropped it. + // as the pre-accept check has always dropped it. if state.should_shutdown() { break; } @@ -329,20 +314,9 @@ impl HealthListener { handle_client(stream, &state, &info); }); } - Ok(AcceptedConnection::ForceBlockingFailed(error)) => { - warn!( - error = %error, - "could not make accepted health connection blocking" - ); - } - Err(ref error) if error.kind() == std::io::ErrorKind::Interrupted => { - // Not an error: loop round, re-check `should_shutdown`, - // wait again. - } - Err(ref error) if error.kind() == std::io::ErrorKind::WouldBlock => { - // Nothing queued. Sleep in the kernel until the socket is - // readable or the interval expires, then re-check the flag. - wait_for_accept_readiness(&self.listener, ACCEPT_POLL_INTERVAL); + Ok(None) => { + // Timed out (or a client that gave up): loop round and + // re-check `should_shutdown`. } Err(error) => { warn!(error = %error, "health listener accept error"); @@ -354,127 +328,27 @@ impl HealthListener { } } -/// Keeps [`DaemonState::wake_port`] pointing at a live listener for exactly as -/// long as that listener is running. +/// Keeps [`DaemonState::listener_registered`] true for exactly as long as the +/// listener loop is running. /// /// A `Drop` impl rather than a call at the end of [`HealthListener::run`]: /// the loop can unwind (`std::thread::spawn` panics when the process is out of -/// threads), and an unwind that skipped the clear would leave `wake_port` -/// naming a port the dropped listener had just released. -struct WakePortRegistration<'a> { - state: &'a DaemonState, +/// threads), and an unwind that skipped the clear would leave the flag set for +/// an endpoint the dropped listener had just released. +struct ListenerRegistration<'state> { + state: &'state DaemonState, } -impl<'a> WakePortRegistration<'a> { - fn register(state: &'a DaemonState, port: u16) -> Self { - state.register_wake_port(port); +impl<'state> ListenerRegistration<'state> { + fn register(state: &'state DaemonState) -> Self { + state.register_listener(); Self { state } } } -impl Drop for WakePortRegistration<'_> { +impl Drop for ListenerRegistration<'_> { fn drop(&mut self) { - self.state.clear_wake_port(); - } -} - -/// Wait until `listener` has a connection queued, or `timeout` elapses. -/// -/// Best-effort: the caller re-checks `should_shutdown` and retries `accept()` -/// regardless of the outcome, so a spurious early return costs one extra -/// `accept()` syscall and an error costs nothing. Returning early is always -/// safe; returning *late* is what must not happen, because the shutdown bound -/// documented on [`ACCEPT_POLL_INTERVAL`] rests on this call being finite. -#[cfg(unix)] -fn wait_for_accept_readiness(listener: &TcpListener, timeout: Duration) { - use std::os::fd::AsRawFd; - - let deadline = Instant::now() + timeout; - loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return; - } - - let mut poll_fd = libc::pollfd { - fd: listener.as_raw_fd(), - events: libc::POLLIN, - revents: 0, - }; - // Saturating rather than wrapping: a timeout too large for a `c_int` - // becomes the longest wait `poll` can express, never a negative value, - // which `poll` reads as "block forever". Floored at 1 ms so a - // sub-millisecond remainder cannot round to a zero-timeout spin. - let timeout_ms = i32::try_from(remaining.as_millis()) - .unwrap_or(i32::MAX) - .max(1); - - // SAFETY: `poll_fd` is a single live, correctly initialized `pollfd`, - // and the `1` matches it; `listener` keeps the fd open for the whole - // call. `poll` only reads `fd`/`events` and writes `revents`. - if unsafe { libc::poll(&raw mut poll_fd, 1, timeout_ms) } >= 0 { - // Readable, or the interval expired. Either way the caller's next - // `accept()` decides what happened. - return; - } - - let error = std::io::Error::last_os_error(); - if error.kind() != std::io::ErrorKind::Interrupted { - // Nothing a retry fixes, and returning immediately would spin: the - // caller's `accept()` would come straight back with `WouldBlock` - // and call this again. Sleep out the rest of the interval so a - // persistently broken `poll` degrades to the old timer, keeping - // the shutdown bound while costing no more than the 5 ms poll did. - warn!(%error, "health listener readiness wait failed; falling back to a timer"); - std::thread::sleep(remaining); - return; - } - } -} - -/// Windows has no `poll` in this crate's dependency set, so the wait degrades -/// to a plain sleep and cannot be cut short by an arriving connection. That is -/// why [`ACCEPT_POLL_INTERVAL`] is 5 ms there: the interval doubles as the -/// accept latency, exactly as it did before the listener stopped polling. -#[cfg(not(unix))] -fn wait_for_accept_readiness(_listener: &TcpListener, timeout: Duration) { - std::thread::sleep(timeout); -} - -/// Outcome of [`accept_and_force_blocking`] once a connection has actually -/// been accepted (as opposed to `accept()` itself erroring, which is -/// propagated through the outer `io::Result`). -enum AcceptedConnection { - /// Accepted and confirmed blocking; ready to hand to [`handle_client`]. - Ready(TcpStream), - /// Accepted, but the attempt to clear the non-blocking flag failed. The - /// connection is dropped; the caller logs and moves on to the next - /// `accept()`. - ForceBlockingFailed(std::io::Error), -} - -/// Accept one connection and force it into blocking mode. -/// -/// On BSD-derived kernels — macOS and other BSDs, but **not** Linux, which -/// keeps a newly accepted socket's blocking mode independent of the -/// listener's — `accept()` propagates the listening socket's `O_NONBLOCK` -/// flag to the accepted socket. Left non-blocking, the accepted stream would -/// return `WouldBlock` from `read_line` in [`handle_client`] in microseconds — -/// typically before the client has even written its first byte — tearing the -/// connection down before it ever received a command. -/// -/// [`HealthListener::bind`] puts the listening socket into non-blocking mode -/// so [`HealthListener::run`] can bound its wait for readiness, which is -/// exactly the configuration that triggers the propagation. -/// `set_nonblocking(false)` below undoes it unconditionally, which is a no-op -/// (not a bug) on platforms that never had the problem, and states the -/// invariant [`handle_client`] actually depends on rather than leaving it to -/// follow from how the listener happens to be configured. -fn accept_and_force_blocking(listener: &TcpListener) -> std::io::Result { - let (stream, _addr) = listener.accept()?; - match stream.set_nonblocking(false) { - Ok(()) => Ok(AcceptedConnection::Ready(stream)), - Err(error) => Ok(AcceptedConnection::ForceBlockingFailed(error)), + self.state.clear_listener(); } } @@ -489,20 +363,17 @@ fn status_json(info: &Mutex) -> String { } } -#[expect( - clippy::needless_pass_by_value, - reason = "TcpStream must be owned for BufReader" -)] -fn handle_client(stream: TcpStream, state: &DaemonState, info: &Mutex) { - let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); - let mut reader = BufReader::new(&stream); - let mut writer = &stream; +fn handle_client(mut stream: ControlStream, state: &DaemonState, info: &Mutex) { + if stream.set_io_timeout(CONNECTION_IO_TIMEOUT).is_err() { + return; + } + let mut reader = BufReader::new(stream); let mut line = String::new(); loop { line.clear(); match reader.read_line(&mut line) { - Ok(0) => break, + Ok(0) | Err(_) => break, Ok(_) => { let cmd = line.trim(); let response = match cmd { @@ -522,16 +393,18 @@ fn handle_client(stream: TcpStream, state: &DaemonState, info: &Mutex "ERR unknown command\n".to_string(), }; - if writer.write_all(response.as_bytes()).is_err() { + if reader.get_mut().write_all(response.as_bytes()).is_err() { break; } } - Err(_) => break, } } + // Deliver what was written before closing (a pipe instance needs the + // flush); a peer that already hung up is not an error. + let _ = reader.into_inner().finish(); } -/// Send a command to the daemon's health port and return the response. +/// Send a command to the daemon's health endpoint and return the response. /// /// Uses generous timeouts (2s connect, 5s read) suitable for `STOP`/`STATUS` /// where the caller is willing to wait. Use [`send_command_with_timeout`] for @@ -539,9 +412,9 @@ fn handle_client(stream: TcpStream, state: &DaemonState, info: &Mutex std::io::Result { +pub fn send_command(endpoint: &HealthEndpoint, command: &str) -> std::io::Result { send_command_with_timeout( - port, + endpoint, command, Duration::from_secs(2), Duration::from_secs(5), @@ -552,9 +425,9 @@ pub fn send_command(port: u16, command: &str) -> std::io::Result { /// be dead from this client's perspective. Uses short timeouts (200ms each) so /// the calling tool handler isn't stalled if the daemon itself is slow. /// Errors are logged at debug level and otherwise ignored. -pub fn report_hyperd_error_to_daemon(health_port: u16) { +pub fn report_hyperd_error_to_daemon(endpoint: &HealthEndpoint) { let timeout = Duration::from_millis(200); - match send_command_with_timeout(health_port, "REPORT_HYPERD_ERROR", timeout, timeout) { + match send_command_with_timeout(endpoint, "REPORT_HYPERD_ERROR", timeout, timeout) { Ok(response) => { debug!(response = %response.trim(), "reported hyperd error to daemon"); } @@ -573,13 +446,12 @@ pub fn report_hyperd_error_to_daemon(health_port: u16) { /// Returns an error if the connection fails or the response cannot be read /// within the supplied timeouts. pub fn send_command_with_timeout( - port: u16, + endpoint: &HealthEndpoint, command: &str, connect_timeout: Duration, read_timeout: Duration, ) -> std::io::Result { - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); - let mut stream = TcpStream::connect_timeout(&addr, connect_timeout)?; + let mut stream = control::connect(endpoint, connect_timeout)?; let io_deadline = Instant::now().checked_add(read_timeout).ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::InvalidInput, @@ -590,7 +462,7 @@ pub fn send_command_with_timeout( let msg = format!("{command}\n"); let mut written = 0; while written < msg.len() { - stream.set_write_timeout(Some(remaining_io_time(io_deadline)?))?; + stream.set_io_timeout(remaining_io_time(io_deadline)?)?; match stream.write(&msg.as_bytes()[written..]) { Ok(0) => { return Err(std::io::Error::new( @@ -609,7 +481,7 @@ pub fn send_command_with_timeout( const MAX_HEALTH_RESPONSE_BYTES: usize = 64 * 1024; let mut response = Vec::new(); loop { - stream.set_read_timeout(Some(remaining_io_time(io_deadline)?))?; + rearm_io_timeout(&mut stream, remaining_io_time(io_deadline)?)?; let mut byte = [0]; match stream.read(&mut byte) { Ok(0) if response.is_empty() => { @@ -644,6 +516,22 @@ pub fn send_command_with_timeout( }) } +/// Re-arm the per-operation timeout before a read. +/// +/// macOS rejects `SO_RCVTIMEO` with `EINVAL` (`InvalidInput`) on a Unix socket +/// whose peer has already replied and closed. The reply is still readable and +/// the previously armed timeout still bounds the read, so only that case is +/// ignored; every other error is returned. +pub(crate) fn rearm_io_timeout( + stream: &mut ControlStream, + timeout: Duration, +) -> std::io::Result<()> { + match stream.set_io_timeout(timeout) { + Err(error) if error.kind() != std::io::ErrorKind::InvalidInput => Err(error), + _ => Ok(()), + } +} + fn remaining_io_time(io_deadline: Instant) -> std::io::Result { let remaining = io_deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { @@ -678,14 +566,15 @@ fn normalize_expired_io_error(error: std::io::Error, io_deadline: Instant) -> st /// (`Some(String::new())`) is returned if the PONG prefix matches but no version /// token is present (graceful degradation for forward/backward compat). /// -/// This is the primitive for liveness checks now that bare TCP connect is -/// insufficient (a foreign service on the same port would cause collisions). +/// This is the primitive for liveness checks: merely connecting proves nothing +/// about who answered, the identifying token does. pub fn ping_identified( - port: u16, + endpoint: &HealthEndpoint, connect_timeout: Duration, read_timeout: Duration, ) -> Option { - let response = send_command_with_timeout(port, "PING", connect_timeout, read_timeout).ok()?; + let response = + send_command_with_timeout(endpoint, "PING", connect_timeout, read_timeout).ok()?; // Validate by exact tokens, not a string prefix: a prefix check on // "PONG hyperdb-mcp" would also match a foreign reply like // "PONG hyperdb-mcpEVIL 1.0.0". Require the first two whitespace-separated @@ -711,8 +600,17 @@ mod tests { use super::*; + /// A per-test state directory and the endpoint inside it. Kept short so + /// the socket path stays under the platform's `sun_path` limit. + fn test_endpoint() -> (tempfile::TempDir, HealthEndpoint) { + let dir = tempfile::tempdir().expect("create test state dir"); + let endpoint = HealthEndpoint::for_new_daemon(dir.path()).expect("test endpoint"); + (dir, endpoint) + } + struct TestPeer { - port: u16, + _dir: tempfile::TempDir, + endpoint: HealthEndpoint, stop: Option>, handle: Option>>, } @@ -744,32 +642,23 @@ mod tests { fn spawn_test_peer(script: F) -> TestPeer where T: Send + 'static, - F: FnOnce(TcpStream, Receiver<()>) -> Result + Send + 'static, + F: FnOnce(ControlStream, Receiver<()>) -> Result + Send + 'static, { - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind test health peer"); - listener - .set_nonblocking(true) - .expect("make test health peer nonblocking"); - let port = listener.local_addr().expect("test peer address").port(); + let (dir, endpoint) = test_endpoint(); + let mut listener = ControlListener::bind(&endpoint).expect("bind test health peer"); let (stop_tx, stop_rx) = mpsc::channel(); let handle = std::thread::spawn(move || { - let accept_deadline = Instant::now() + Duration::from_secs(2); + let accept_deadline = Instant::now() + Duration::from_secs(5); loop { if stop_rx.try_recv().is_ok() { return Err("test peer stopped before accepting a connection".to_string()); } - match listener.accept() { - Ok((stream, _)) => { - stream.set_nonblocking(false).map_err(|error| { - format!("make accepted test health peer blocking: {error}") - })?; - return script(stream, stop_rx); - } - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + match listener.accept_timeout(Duration::from_millis(20)) { + Ok(Some(stream)) => return script(stream, stop_rx), + Ok(None) => { if Instant::now() >= accept_deadline { return Err("test peer timed out waiting for a connection".to_string()); } - std::thread::sleep(Duration::from_millis(2)); } Err(error) => return Err(format!("test peer accept failed: {error}")), } @@ -777,31 +666,41 @@ mod tests { }); TestPeer { - port, + _dir: dir, + endpoint, stop: Some(stop_tx), handle: Some(handle), } } - fn read_test_command(stream: &TcpStream) -> Result { - let reader_stream = stream - .try_clone() - .map_err(|error| format!("clone test peer stream: {error}"))?; - reader_stream - .set_read_timeout(Some(Duration::from_secs(1))) + /// Read one request line a byte at a time, so no buffered reader swallows + /// bytes the script may want afterwards. + fn read_test_command(stream: &mut ControlStream) -> Result { + stream + .set_io_timeout(Duration::from_secs(1)) .map_err(|error| format!("set test peer read timeout: {error}"))?; - let mut request = String::new(); - BufReader::new(reader_stream) - .read_line(&mut request) - .map_err(|error| format!("read test health command: {error}"))?; - Ok(request) + let mut request = Vec::new(); + let mut byte = [0]; + loop { + match stream.read(&mut byte) { + Ok(0) => break, + Ok(_) => { + request.push(byte[0]); + if byte[0] == b'\n' { + break; + } + } + Err(error) => return Err(format!("read test health command: {error}")), + } + } + String::from_utf8(request).map_err(|error| format!("test command not UTF-8: {error}")) } - fn daemon_info(health_port: u16) -> DaemonInfo { + fn daemon_info(endpoint: &HealthEndpoint) -> DaemonInfo { DaemonInfo { pid: 4242, hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port, + health_endpoint: endpoint.as_str().to_string(), started_at: "2026-08-13T12:34:56Z".to_string(), version: "0.7.0".to_string(), } @@ -814,7 +713,7 @@ mod tests { json!({ "pid": info.pid, "hyperd_endpoint": info.hyperd_endpoint, - "health_port": info.health_port, + "health_endpoint": info.health_endpoint, "started_at": info.started_at, "version": info.version, "identity": { @@ -847,15 +746,14 @@ mod tests { #[test] fn health_status_returns_flat_enriched_record() { - let _network_guard = crate::diagnostics::real_network_test_guard(); let public_run_signature: fn(HealthListener, Arc, Arc>) = HealthListener::run; std::hint::black_box(public_run_signature); - let listener = HealthListener::bind(0).unwrap(); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(daemon_info(port))); + let (_dir, endpoint) = test_endpoint(); + let listener = HealthListener::bind(&endpoint).unwrap(); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(daemon_info(&endpoint))); let initial_expected = expected_status(&info.lock().unwrap()); let mut failures = Vec::new(); @@ -873,7 +771,7 @@ mod tests { let run_info = Arc::clone(&info); let handle = std::thread::spawn(move || listener.run(run_state, run_info)); - let initial_response = send_command(port, "STATUS").unwrap(); + let initial_response = send_command(&endpoint, "STATUS").unwrap(); check_status_json( "initial STATUS response", &initial_response, @@ -886,7 +784,7 @@ mod tests { current.hyperd_endpoint = "127.0.0.1:60000".to_string(); } let updated_expected = expected_status(&info.lock().unwrap()); - let updated_response = send_command(port, "STATUS").unwrap(); + let updated_response = send_command(&endpoint, "STATUS").unwrap(); check_status_json( "STATUS response after shared DaemonInfo update", &updated_response, @@ -894,7 +792,7 @@ mod tests { &mut failures, ); - let _ = send_command(port, "STOP"); + let _ = send_command(&endpoint, "STOP"); handle.join().unwrap(); assert!( @@ -912,7 +810,7 @@ mod tests { const GENEROUS_COMPLETION_BOUND: Duration = Duration::from_millis(500); let peer = spawn_test_peer(|mut stream, stop| { - let request = read_test_command(&stream)?; + let request = read_test_command(&mut stream)?; if request != "PING\n" { return Err(format!("unexpected health command: {request:?}")); } @@ -931,8 +829,12 @@ mod tests { let call = catch_unwind(AssertUnwindSafe(|| { let started = Instant::now(); - let result = - send_command_with_timeout(peer.port, "PING", Duration::from_secs(1), IO_TIMEOUT); + let result = send_command_with_timeout( + &peer.endpoint, + "PING", + Duration::from_secs(1), + IO_TIMEOUT, + ); (result, started.elapsed()) })); let completed_full_drip = peer @@ -964,12 +866,12 @@ mod tests { const OVERSIZED_RESPONSE_BYTES: usize = MAX_HEALTH_RESPONSE_BYTES + 1; let peer = spawn_test_peer(|mut stream, _stop| { - let request = read_test_command(&stream)?; + let request = read_test_command(&mut stream)?; if request != "PING\n" { return Err(format!("unexpected health command: {request:?}")); } stream - .set_write_timeout(Some(Duration::from_secs(1))) + .set_io_timeout(Duration::from_secs(1)) .map_err(|error| format!("set test peer write timeout: {error}"))?; let response = vec![b'x'; OVERSIZED_RESPONSE_BYTES]; let mut emitted = 0; @@ -981,6 +883,9 @@ mod tests { )); } Ok(written) => emitted += written, + // The client stops reading at the cap and hangs up, so a + // broken pipe after the cap is the expected end. + Err(_) if emitted > MAX_HEALTH_RESPONSE_BYTES => break, Err(error) => { return Err(format!( "oversized health peer stopped after {emitted} bytes: {error}" @@ -993,7 +898,7 @@ mod tests { let call = catch_unwind(AssertUnwindSafe(|| { send_command_with_timeout( - peer.port, + &peer.endpoint, "PING", Duration::from_secs(1), Duration::from_secs(1), @@ -1011,7 +916,7 @@ mod tests { Err(error) => format!("returned {:?}: {error}", error.kind()), }; - assert_eq!(response_bytes, OVERSIZED_RESPONSE_BYTES); + assert!(response_bytes > MAX_HEALTH_RESPONSE_BYTES); assert_eq!( result.as_ref().err().map(std::io::Error::kind), Some(std::io::ErrorKind::InvalidData), @@ -1019,71 +924,60 @@ mod tests { ); } - /// Asserts the accepted socket's actual blocking state via `fcntl`, - /// rather than inferring it from read-timing behavior. On Linux an - /// accepted socket is blocking regardless of the listener's mode, so a - /// behavioral test (send late, expect it to still be read) passes - /// trivially there even with `accept_and_force_blocking`'s - /// `set_nonblocking(false)` reverted — it would only catch a regression - /// on the BSD-derived kernels (macOS and other BSDs) that actually - /// propagate `O_NONBLOCK` to accepted sockets. Reading the `O_NONBLOCK` - /// flag directly makes the test verify the real contract everywhere, - /// rather than a platform-dependent behavioral proxy for it. - /// - /// The listener is put into non-blocking mode explicitly. `bind` leaves it - /// blocking now that [`HealthListener::run`] parks in `accept()`, so - /// without this the accepted socket would be blocking for free and the - /// assertion would hold no matter what `accept_and_force_blocking` did — - /// re-opening the verification gap #273 closed. - #[cfg(unix)] + /// Deterministic form of the test below: re-arming the timeout on a + /// stream whose peer already replied and closed must succeed, and the + /// reply must still be readable. #[test] - fn accept_and_force_blocking_clears_nonblocking_flag() { - use std::os::unix::io::AsRawFd; - - let listener = HealthListener::bind(0).expect("bind test health listener"); - let port = listener.port; - - let client = std::thread::spawn(move || { - // Held for the duration of the accept below; dropped (and thus - // closed) only once this thread returns. - std::net::TcpStream::connect(("127.0.0.1", port)).expect("connect test client") + fn rearming_the_timeout_after_the_peer_closed_keeps_the_reply_readable() { + let peer = spawn_test_peer(|mut stream, _stop| { + read_test_command(&mut stream)?; + stream + .write_all(b"OK\n") + .map_err(|error| format!("write reply: {error}"))?; + drop(stream); + Ok(()) }); + let mut client = control::connect(&peer.endpoint, Duration::from_secs(1)).unwrap(); + client.write_all(b"X\n").unwrap(); + peer.finish().expect("peer must finish"); + std::thread::sleep(Duration::from_millis(100)); - let accept_deadline = Instant::now() + Duration::from_secs(2); - let accepted = loop { - match accept_and_force_blocking(&listener.listener) { - Ok(AcceptedConnection::Ready(stream)) => break stream, - Ok(AcceptedConnection::ForceBlockingFailed(error)) => { - panic!("force-blocking the accepted test connection failed: {error}") - } - Err(ref error) if error.kind() == std::io::ErrorKind::WouldBlock => { - assert!( - Instant::now() < accept_deadline, - "timed out waiting to accept the test client connection" - ); - std::thread::sleep(Duration::from_millis(2)); - } - Err(error) => panic!("accept failed: {error}"), + rearm_io_timeout(&mut client, Duration::from_secs(1)) + .expect("a closed peer must not fail the re-arm"); + let mut reply = [0_u8; 3]; + client.read_exact(&mut reply).unwrap(); + assert_eq!(&reply, b"OK\n"); + } + + /// A peer that replies and closes at once must not turn into an error when + /// the client re-arms its read timeout (macOS rejects that with `EINVAL`). + #[test] + fn a_reply_followed_by_an_immediate_close_is_still_returned_promptly() { + let peer = spawn_test_peer(|mut stream, _stop| { + let request = read_test_command(&mut stream)?; + if request != "PING\n" { + return Err(format!("unexpected health command: {request:?}")); } - }; - client.join().expect("test client thread must not panic"); + stream + .write_all(b"PONG hyperdb-mcp 9.9.9\n") + .map_err(|error| format!("write reply: {error}"))?; + drop(stream); + Ok(()) + }); - // SAFETY: `accepted` is a live, owned, valid socket for the duration - // of this call; `F_GETFL` only reads flags and mutates nothing. - let flags = unsafe { libc::fcntl(accepted.as_raw_fd(), libc::F_GETFL) }; - assert!( - flags >= 0, - "fcntl(F_GETFL) on the accepted test connection failed: {}", - std::io::Error::last_os_error() + let started = Instant::now(); + let result = send_command_with_timeout( + &peer.endpoint, + "PING", + Duration::from_secs(1), + Duration::from_secs(5), ); + peer.finish().expect("reply-and-close peer must finish"); assert_eq!( - flags & libc::O_NONBLOCK, - 0, - "accepted health connection must be blocking (O_NONBLOCK must be clear); \ - reverting accept_and_force_blocking's set_nonblocking(false) call would \ - leave O_NONBLOCK set on BSD-derived kernels that propagate it from the \ - listening socket" + result.expect("reply must survive the close"), + "PONG hyperdb-mcp 9.9.9\n" ); + assert!(started.elapsed() < Duration::from_secs(2)); } /// A peer that reads the request and then closes without writing anything @@ -1093,8 +987,8 @@ mod tests { /// answered at all. #[test] fn peer_close_without_any_response_is_an_error_not_empty_success() { - let peer = spawn_test_peer(|stream, _stop| { - let request = read_test_command(&stream)?; + let peer = spawn_test_peer(|mut stream, _stop| { + let request = read_test_command(&mut stream)?; if request != "PING\n" { return Err(format!("unexpected health command: {request:?}")); } @@ -1105,7 +999,7 @@ mod tests { let call = catch_unwind(AssertUnwindSafe(|| { send_command_with_timeout( - peer.port, + &peer.endpoint, "PING", Duration::from_secs(1), Duration::from_secs(1), @@ -1125,48 +1019,14 @@ mod tests { ); } - /// The precondition the bounded-shutdown floor rests on. `run` never calls - /// `accept()` on a socket that can park: it waits for readiness with an - /// [`ACCEPT_POLL_INTERVAL`] timeout and then accepts what is already - /// queued. Make the listening socket blocking and that inverts — `accept()` - /// itself parks in the kernel with no timeout of any kind, and shutdown - /// stops being bounded by anything except the wake connection landing. - /// Asserting the flag rather than timing the loop keeps this deterministic; - /// a CPU or wakeup-rate threshold would be a flake on a loaded CI runner. - #[cfg(unix)] - #[test] - fn bind_leaves_the_listening_socket_nonblocking() { - use std::os::unix::io::AsRawFd; - - let listener = HealthListener::bind(0).expect("bind test health listener"); - - // SAFETY: `listener.listener` is a live, owned, valid socket for the - // duration of this call; `F_GETFL` only reads flags and mutates nothing. - let flags = unsafe { libc::fcntl(listener.listener.as_raw_fd(), libc::F_GETFL) }; - assert!( - flags >= 0, - "fcntl(F_GETFL) on the health listening socket failed: {}", - std::io::Error::last_os_error() - ); - assert_eq!( - flags & libc::O_NONBLOCK, - libc::O_NONBLOCK, - "the health listening socket must stay non-blocking so run()'s accept() can \ - never park; a blocking listener makes shutdown depend entirely on the wake \ - connection landing, and a lost wake becomes a permanent hang in the join()" - ); - } - /// Deadline for the two tests that assert the *wake* is doing the work. /// - /// Deliberately far below [`ACCEPT_POLL_INTERVAL`], because the readiness - /// floor would otherwise make them pass with the wake deleted: the loop - /// would simply notice the flag when its interval expired. Measured + /// Deliberately far below [`ACCEPT_POLL_INTERVAL`], because the accept + /// timeout floor would otherwise make them pass with the wake deleted: the + /// loop would simply notice the flag when its interval expired. Measured /// shutdown latency through the wake is well under a millisecond, so half - /// a second is loose enough for a loaded CI runner and still an order of - /// magnitude clear of the one-second floor it has to distinguish itself - /// from. On Windows [`ACCEPT_POLL_INTERVAL`] is 5 ms and the two paths are - /// indistinguishable; the assertion holds there but proves nothing. + /// a second is loose enough for a loaded CI runner and still clear of the + /// one-second floor it has to distinguish itself from. const PROMPT_SHUTDOWN_BOUND: Duration = Duration::from_millis(500); /// The promptness half of the design. `run_daemon` (and every test @@ -1178,10 +1038,10 @@ mod tests { /// regression fails with a message rather than stalling the suite. #[test] fn request_shutdown_wakes_a_waiting_accept_loop() { - let listener = HealthListener::bind(0).expect("bind test health listener"); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(daemon_info(port))); + let (_dir, endpoint) = test_endpoint(); + let listener = HealthListener::bind(&endpoint).expect("bind test health listener"); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(daemon_info(&endpoint))); let run_state = Arc::clone(&state); let run_info = Arc::clone(&info); @@ -1192,12 +1052,16 @@ mod tests { }); // Prove the loop serves a real client with no added latency, then idle - // long enough that it is certainly back inside its readiness wait when - // shutdown is requested — so the elapsed time below measures the wake + // long enough that it is certainly back inside its accept wait when + // shutdown is requested, so the elapsed time below measures the wake // and not a coincidental interval boundary. - let pong = - send_command_with_timeout(port, "PING", Duration::from_secs(2), Duration::from_secs(2)) - .expect("the accept loop must serve a real client while waiting for readiness"); + let pong = send_command_with_timeout( + &endpoint, + "PING", + Duration::from_secs(2), + Duration::from_secs(2), + ) + .expect("the accept loop must serve a real client while waiting"); assert!(pong.starts_with("PONG"), "unexpected PING reply: {pong:?}"); std::thread::sleep(Duration::from_millis(100)); @@ -1212,36 +1076,33 @@ mod tests { assert!( elapsed < PROMPT_SHUTDOWN_BOUND, "request_shutdown took {elapsed:?} to stop the listener, past the \ - {PROMPT_SHUTDOWN_BOUND:?} bound: the self-connect wake did not land and the \ - loop fell through to its {ACCEPT_POLL_INTERVAL:?} readiness floor instead" + {PROMPT_SHUTDOWN_BOUND:?} bound: the wake connection did not land and the \ + loop fell through to its {ACCEPT_POLL_INTERVAL:?} accept floor instead" ); server .join() .expect("health listener thread must not panic"); } - /// The floor underneath the wake, and the reason `run` may never call a - /// blocking `accept()`. + /// The floor underneath the wake, and the reason `run` may never wait + /// without a timeout. /// - /// Clearing `wake_port` makes `request_shutdown` return without connecting - /// to anything — the same observable outcome as a shutdown that races - /// registration, a second listener having overwritten the port, or a - /// connect that never lands. Every `request_shutdown` caller in the tree - /// joins the listener thread, so with nothing but the wake to rely on this - /// is not a slow shutdown but a permanent one. - /// - /// Red against a blocking listening socket: the loop sits in `accept()` - /// with no timeout of any kind and the channel below never fires. + /// Clearing the registration makes `request_shutdown` return without + /// connecting to anything, the same observable outcome as a shutdown that + /// races registration or a connect that never lands. Every + /// `request_shutdown` caller in the tree joins the listener thread, so with + /// nothing but the wake to rely on this is not a slow shutdown but a + /// permanent one. #[test] fn shutdown_is_bounded_when_the_wake_cannot_be_delivered() { // Comfortably past `ACCEPT_POLL_INTERVAL`, so the test measures // "bounded at all" rather than racing the floor it is verifying. const LOST_WAKE_SHUTDOWN_BOUND: Duration = Duration::from_secs(5); - let listener = HealthListener::bind(0).expect("bind test health listener"); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(daemon_info(port))); + let (_dir, endpoint) = test_endpoint(); + let listener = HealthListener::bind(&endpoint).expect("bind test health listener"); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(daemon_info(&endpoint))); let run_state = Arc::clone(&state); let (finished_tx, finished_rx) = mpsc::channel(); @@ -1253,12 +1114,16 @@ mod tests { // Round-trip a real command first: that proves `run` has reached its // loop and registered, so the clear below genuinely un-registers a // live listener rather than winning a race against startup. - let pong = - send_command_with_timeout(port, "PING", Duration::from_secs(2), Duration::from_secs(2)) - .expect("the accept loop must serve a real client before the wake is removed"); + let pong = send_command_with_timeout( + &endpoint, + "PING", + Duration::from_secs(2), + Duration::from_secs(2), + ) + .expect("the accept loop must serve a real client before the wake is removed"); assert!(pong.starts_with("PONG"), "unexpected PING reply: {pong:?}"); - state.clear_wake_port(); + state.clear_listener(); let requested_at = Instant::now(); state.request_shutdown(); @@ -1268,7 +1133,7 @@ mod tests { finished.is_ok(), "the listener did not stop within {LOST_WAKE_SHUTDOWN_BOUND:?} ({elapsed:?}) once \ the wake could not be delivered; shutdown must stay bounded by the loop's own \ - {ACCEPT_POLL_INTERVAL:?} readiness timeout, because run_daemon and every test \ + {ACCEPT_POLL_INTERVAL:?} accept timeout, because run_daemon and every test \ harness join this thread and an unbounded wait there is a permanent hang" ); server @@ -1276,16 +1141,15 @@ mod tests { .expect("health listener thread must not panic"); } - /// The ordering hazard the pre-`accept` flag check covers: a shutdown - /// requested before `run` has registered a wake port sends no wake at all - /// (`wake_port` is still the `0` sentinel), so the loop must notice the - /// flag on its own rather than waiting out an interval first. + /// The ordering hazard the pre-accept flag check covers: a shutdown + /// requested before `run` has registered sends no wake at all, so the loop + /// must notice the flag on its own rather than waiting out an interval. #[test] fn run_returns_promptly_when_shutdown_precedes_it() { - let listener = HealthListener::bind(0).expect("bind test health listener"); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(daemon_info(port))); + let (_dir, endpoint) = test_endpoint(); + let listener = HealthListener::bind(&endpoint).expect("bind test health listener"); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(daemon_info(&endpoint))); state.request_shutdown(); @@ -1302,11 +1166,21 @@ mod tests { assert!( finished.is_ok() && elapsed < PROMPT_SHUTDOWN_BOUND, "run() must observe a pre-existing shutdown flag before waiting on the socket; \ - it returned after {elapsed:?}, which means it waited out a readiness interval \ + it returned after {elapsed:?}, which means it waited out an accept interval \ ({ACCEPT_POLL_INTERVAL:?}) first" ); server .join() .expect("health listener thread must not panic"); } + + /// Dropping the listener releases the endpoint, so a second listener can + /// bind it. (Single-instance is the lock's job, not the socket's.) + #[test] + fn dropping_the_listener_frees_the_endpoint() { + let (_dir, endpoint) = test_endpoint(); + let first = HealthListener::bind(&endpoint).expect("first bind"); + drop(first); + HealthListener::bind(&endpoint).expect("rebind after drop"); + } } diff --git a/hyperdb-mcp/src/daemon/mod.rs b/hyperdb-mcp/src/daemon/mod.rs index 07c64887..a0ac1340 100644 --- a/hyperdb-mcp/src/daemon/mod.rs +++ b/hyperdb-mcp/src/daemon/mod.rs @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT //! Single-instance daemon for sharing a `hyperd` process across MCP clients. +//! +//! A daemon is identified by its state directory (`~/.hyperdb`, or +//! `HYPERDB_STATE_DIR`): the single-instance lock is `daemon.lock` in it, the +//! health channel is a per-user Unix socket or named pipe (see [`control`]), and +//! `daemon.json` tells clients where to find both. There is no TCP port. pub mod control; pub mod discovery; @@ -11,22 +16,10 @@ pub mod run; pub mod spawn; pub mod state_perms; -/// Default base TCP port for the daemon health listener. When no env var is set, -/// the daemon scans `[base, base + DAEMON_PORT_SCAN_SPAN)` to find a free port. -/// Previously 7484; changed to 7485 to avoid collision with hyperd's default gRPC port. -pub const DEFAULT_DAEMON_BASE_PORT: u16 = 7485; - -/// Number of ports to scan starting from the base port when discovering or spawning -/// a daemon. Used by the later port-scanning stage (not yet implemented). -pub const DAEMON_PORT_SCAN_SPAN: u16 = 16; - /// Suggested idle timeout value (30 minutes) for use with the `--idle-timeout` flag /// or `HYPERDB_DAEMON_IDLE_TIMEOUT` env var. By default (when neither is set), the /// daemon never auto-shuts down due to inactivity. pub const DEFAULT_IDLE_TIMEOUT_SECS: u64 = 30 * 60; -/// Environment variable to override the daemon port. -pub const ENV_DAEMON_PORT: &str = "HYPERDB_DAEMON_PORT"; - /// Environment variable to override the idle timeout (seconds). pub const ENV_IDLE_TIMEOUT: &str = "HYPERDB_DAEMON_IDLE_TIMEOUT"; diff --git a/hyperdb-mcp/src/daemon/run.rs b/hyperdb-mcp/src/daemon/run.rs index 6bd98ec2..29893675 100644 --- a/hyperdb-mcp/src/daemon/run.rs +++ b/hyperdb-mcp/src/daemon/run.rs @@ -1,8 +1,16 @@ // Copyright (c) 2026, Salesforce, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Daemon main loop: spawns `hyperd`, runs health listener, monitors hyperd liveness -//! and optional idle timeout, restarts hyperd if it dies. +//! Daemon main loop: takes the single-instance lock, spawns `hyperd`, runs the +//! health listener, monitors hyperd liveness and optional idle timeout, +//! restarts hyperd if it dies. +//! +//! Startup order: create and check the state directory, take the +//! [`DaemonLock`], bind the control endpoint, start `hyperd`, write +//! `daemon.json`. Shutdown runs it backwards: remove the record, stop the +//! listener (which removes the socket), drop `hyperd`, and only then release +//! the lock. The lock is what a client waits on, so it must outlive everything +//! the next daemon could collide with. //! //! By default, the daemon never auto-shuts down due to inactivity (opt-in via //! `--idle-timeout` flag or `HYPERDB_DAEMON_IDLE_TIMEOUT` env var). When enabled, @@ -18,13 +26,26 @@ use tracing::{error, info, warn}; use hyperdb_api::{HyperProcess, Parameters, TransportMode}; use super::ENV_IDLE_TIMEOUT; +use super::control::HealthEndpoint; use super::discovery::{self, DaemonInfo}; use super::health::{DaemonState, HealthListener}; +use super::lock::DaemonLock; + +/// How long startup keeps retrying for the daemon lock and for `hyperd`'s +/// socket to come free. A predecessor that was just told to stop needs a moment +/// to let go of both, and a fresh daemon should wait it out rather than fail. +pub const STARTUP_WAIT: Duration = Duration::from_secs(10); + +/// Pause between startup retries. +const STARTUP_RETRY_INTERVAL: Duration = Duration::from_millis(100); + +/// A live daemon must answer for this long before startup believes it is +/// really running (and not merely finishing a shutdown) and gives up. +const ALREADY_RUNNING_DEBOUNCE: Duration = Duration::from_secs(1); /// Configuration for the daemon process. #[derive(Debug)] pub struct DaemonConfig { - pub port: u16, /// Idle timeout duration. When `None`, the daemon never auto-shuts down due to /// inactivity (default behavior). When `Some`, the daemon shuts down after the /// specified duration without client activity. @@ -38,7 +59,7 @@ impl DaemonConfig { /// 1. If `idle_timeout_secs` is `Some`, use that value. /// 2. Otherwise, if `HYPERDB_DAEMON_IDLE_TIMEOUT` env var is set and parseable, use it. /// 3. Otherwise, `None` (never auto-shutdown). - pub fn from_args(port: u16, idle_timeout_secs: Option) -> Self { + pub fn from_args(idle_timeout_secs: Option) -> Self { let idle_timeout = idle_timeout_secs .or_else(|| { std::env::var(ENV_IDLE_TIMEOUT) @@ -47,7 +68,7 @@ impl DaemonConfig { }) .map(Duration::from_secs); - Self { port, idle_timeout } + Self { idle_timeout } } } @@ -74,64 +95,127 @@ enum RestartError { SpawnFailed(String), } +/// Take the daemon lock, retrying for up to `deadline`. +/// +/// A held lock means another daemon is running, starting, or stopping. Only a +/// daemon that keeps answering an identified `PING` for +/// [`ALREADY_RUNNING_DEBOUNCE`] counts as running: one that is mid-shutdown +/// stops answering within that time and the lock frees up behind it. +async fn acquire_lock(state_dir: &Path, deadline: Instant) -> Result { + let mut seen_alive_since: Option = None; + loop { + match DaemonLock::try_acquire(state_dir) { + Ok(Some(lock)) => return Ok(lock), + Ok(None) => {} + Err(error) => { + return Err(format!( + "Failed to take the daemon lock in {}: {error}", + state_dir.display() + )); + } + } + + if discovery::discover_in(state_dir).is_some() { + let since = *seen_alive_since.get_or_insert_with(Instant::now); + if since.elapsed() >= ALREADY_RUNNING_DEBOUNCE { + return Err("Another hyperdb daemon is already running. \ + Use `hyperdb-mcp daemon status` to check or `hyperdb-mcp daemon stop` to stop it." + .to_string()); + } + } else { + seen_alive_since = None; + } + + if Instant::now() >= deadline { + return Err(format!( + "Timed out after {}s waiting for another hyperdb daemon in {} to release \ + the daemon lock", + STARTUP_WAIT.as_secs(), + state_dir.display() + )); + } + tokio::time::sleep(STARTUP_RETRY_INTERVAL).await; + } +} + +/// [`build_params`], retrying while another `hyperd` still holds the socket. +/// The previous daemon's `hyperd` can outlive its lock by a moment on some +/// platforms; the pre-flight failing with `AddrInUse` is the signal to wait. +async fn build_params_waiting(state_dir: &Path, deadline: Instant) -> std::io::Result { + loop { + match build_params(state_dir) { + Err(error) + if error.kind() == std::io::ErrorKind::AddrInUse && Instant::now() < deadline => + { + tokio::time::sleep(STARTUP_RETRY_INTERVAL).await; + } + other => return other, + } + } +} + /// Run the daemon. This function blocks until shutdown is triggered. /// /// # Errors -/// Returns an error if the health port cannot be bound, `hyperd` fails to start, -/// or the discovery file cannot be written. +/// Returns an error if the state directory is not private to the current user, +/// another daemon holds the lock, the control endpoint cannot be bound, +/// `hyperd` fails to start, or the discovery file cannot be written. pub async fn run_daemon(config: DaemonConfig) -> Result<(), Box> { - // Step 1: Bind health port (single-instance lock) - let listener = HealthListener::bind(config.port).map_err(|e| { - if e.kind() == std::io::ErrorKind::AddrInUse { - format!( - "Another hyperdb daemon is already running on port {}. \ - Use `hyperdb-mcp daemon status` to check or `hyperdb-mcp daemon stop` to stop it.", - config.port - ) - } else { - format!("Failed to bind health port {}: {e}", config.port) - } - })?; - let bound_port = listener.port; - info!(port = bound_port, "daemon health listener bound"); + let deadline = Instant::now() + STARTUP_WAIT; - // Step 2: Spawn hyperd over a local IPC channel (Unix domain socket on - // Unix/macOS, named pipe on Windows). + // Step 1: Create the state directory, refuse one that others can write to, + // and take the single-instance lock. let state_dir = discovery::state_dir()?; - let hyper = HyperProcess::new(None, Some(&build_params(&state_dir)?))?; + super::state_perms::ensure_owner_only_dir(&state_dir)?; + super::state_perms::verify_state_dir_trusted(&state_dir)?; + let lock = acquire_lock(&state_dir, deadline).await?; + + // Step 2: Bind the control endpoint. We hold the lock, so anything still + // sitting at the socket path is a dead daemon's leftover. + let endpoint = HealthEndpoint::for_new_daemon(&state_dir)?; + let listener = HealthListener::bind(&endpoint) + .map_err(|e| format!("Failed to bind the health endpoint {endpoint}: {e}"))?; + info!(endpoint = %endpoint, "daemon health listener bound"); + + // Step 3: Spawn hyperd over a local IPC channel (Unix domain socket on + // Unix/macOS, named pipe on Windows). + let hyper = HyperProcess::new( + None, + Some(&build_params_waiting(&state_dir, deadline).await?), + )?; // Publish the *connection* endpoint, not the raw callback descriptor. For a // Unix domain socket the raw `endpoint()` string reconstructs the path as // `/domain/hyper` (a `tab.domain://` scheme artifact), whereas hyperd // actually binds `/hyper`; `connection_endpoint_string()` carries the path a // client can connect to. For TCP and Windows named pipes the two agree. - let endpoint = hyper + let hyperd_endpoint = hyper .connection_endpoint_string() .ok_or("hyperd did not report a connection endpoint")?; - info!(endpoint = %endpoint, "hyperd started"); + info!(endpoint = %hyperd_endpoint, "hyperd started"); - // Step 3: Build DaemonInfo and write discovery file + // Step 4: Build DaemonInfo and write discovery file let info = DaemonInfo { pid: std::process::id(), - hyperd_endpoint: endpoint.clone(), - health_port: bound_port, + hyperd_endpoint, + health_endpoint: endpoint.as_str().to_string(), started_at: chrono::Utc::now().to_rfc3339(), version: env!("CARGO_PKG_VERSION").to_string(), }; discovery::write_enriched_discovery_file(&info)?; info!(path = %discovery::discovery_file_path()?.display(), "discovery file written"); - // Step 4: Build the shared state. + // Step 5: Build the shared state. // - `info_arc` is shared with the health listener so STATUS reports the // current endpoint even after a restart. // - `hyper_state` is owned by the monitor task; the listener never touches it. - let state = Arc::new(DaemonState::new()); + let state = Arc::new(DaemonState::new(endpoint)); let info_arc = Arc::new(Mutex::new(info)); let hyper_state = Arc::new(Mutex::new(HyperState { hyper: Some(hyper), restart_history: Vec::new(), })); - // Step 5: Start health listener in a background thread + // Step 6: Start health listener in a background thread let health_state = Arc::clone(&state); let health_info = Arc::clone(&info_arc); let health_handle = std::thread::spawn(move || { @@ -145,7 +229,7 @@ pub async fn run_daemon(config: DaemonConfig) -> Result<(), Box Result<(), Box Result<(), Box std::io::Result { // this guard hyperd would die with "unable to listen on domain socket: // domain socket is in use" while `HyperProcess::new` masked it as a // 60-second "Timeout waiting for Hyper to connect to callback listener". - // Reachable today only when two daemons share one state dir (different - // ports). A refused/missing socket (stale file, dead owner) is fine — - // proceed and let hyperd's pid-liveness staleness check reclaim it. This + // Reachable only when a previous daemon's hyperd has outlived it + // (`build_params_waiting` retries on this). A refused/missing socket + // (stale file, dead owner) is fine: proceed and let hyperd's pid-liveness staleness check reclaim it. This // keeps the crash-restart path safe: `try_restart_hyperd` reaps the // SIGKILLed child (`guard.hyper = None`) before calling `build_params`, // so here the connect is refused and we proceed to rebind. @@ -399,7 +486,7 @@ fn try_restart_hyperd( // Two independent channels advertise the endpoint: the `daemon.json` // discovery file that `discovery::discover()` reads — the path every // client's `Engine::new` takes — and the in-memory `DaemonInfo` that the - // health port's STATUS command serves. Updating STATUS first and the file + // health channel's STATUS command serves. Updating STATUS first and the file // afterwards left a window in which STATUS announced the freshly spawned // `hyperd` while the file still named the one we just dropped, so a client // discovering during that window connected to a dead port. Persisting diff --git a/hyperdb-mcp/src/daemon/spawn.rs b/hyperdb-mcp/src/daemon/spawn.rs index ef3e3e24..1c71202f 100644 --- a/hyperdb-mcp/src/daemon/spawn.rs +++ b/hyperdb-mcp/src/daemon/spawn.rs @@ -13,7 +13,11 @@ use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; -use super::discovery::{self, DaemonInfo, PortScan, ScanOutcome}; +use std::path::Path; + +use super::control::HealthEndpoint; +use super::discovery::{self, DaemonInfo}; +use super::lock::DaemonLock; /// Maximum time to wait for the daemon to write its discovery file after spawning. const SPAWN_TIMEOUT: Duration = Duration::from_secs(10); @@ -21,68 +25,87 @@ const SPAWN_TIMEOUT: Duration = Duration::from_secs(10); /// Polling interval while waiting for the discovery file. const POLL_INTERVAL: Duration = Duration::from_millis(100); -/// Ensure a daemon is running and return its info. If a daemon is found (via -/// discovery file or port scan), we MAY take it over if the client is newer. -/// Otherwise we spawn a fresh daemon on the first free port in the scan range. +/// Ensure a daemon is running and return its info. If a daemon is found, we MAY +/// take it over if the client is newer. Otherwise we spawn a fresh one. +/// +/// Whether a daemon exists is decided by the daemon lock, not by the record or +/// by probing: a held lock means a daemon is running, starting or stopping, so +/// the client waits for it; a free lock means none is, so a leftover record is +/// stale, is removed, and a daemon is spawned. /// /// # Errors -/// Returns an error if no free port is available, the daemon cannot be spawned, -/// or it does not become ready within the timeout period. -pub fn ensure_daemon(scan: PortScan) -> io::Result { - // Check discovery file first (fast path). - if let Some(info) = discovery::discover() { - return maybe_take_over(info, scan); +/// Returns an error if the state directory is not private to the current user, +/// the daemon cannot be spawned, or it does not become ready within the timeout +/// period. +pub fn ensure_daemon() -> io::Result { + let dir = discovery::state_dir()?; + ensure_daemon_in(&dir, spawn_detached, SPAWN_TIMEOUT) +} + +/// [`ensure_daemon`] for an explicit state directory, spawner and wait budget. +fn ensure_daemon_in( + dir: &Path, + spawn: impl FnOnce() -> io::Result<()>, + timeout: Duration, +) -> io::Result { + match super::state_perms::verify_state_dir_trusted(dir) { + Ok(()) => {} + // No directory yet: nobody has ever run a daemon here. + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), } - // Scan the port range for a running daemon or a free port. - match discovery::scan_for_daemon(scan) { - ScanOutcome::Found(info) => maybe_take_over(*info, scan), - ScanOutcome::FreePort(port) => { - info!(port, "no running daemon detected, spawning on free port"); - spawn_detached(port)?; - let info = wait_for_daemon()?; - // If the daemon we just spawned bound a port above the scan base (because - // concurrent clients raced and one of them grabbed the base port first), - // prefer the lower-port daemon so we don't accumulate redundant - // daemon+hyperd pairs on adjacent ports. The lower-port daemon wins - // because it bound first and is the canonical single instance. - if info.health_port > scan.base { - let lower_scan = PortScan { - base: scan.base, - span: info.health_port.saturating_sub(scan.base), - }; - if let ScanOutcome::Found(lower_info) = discovery::scan_for_daemon(lower_scan) { - debug!( - prefer_port = lower_info.health_port, - stop_port = info.health_port, - "found lower-port daemon from concurrent spawn; stopping off-base daemon" - ); - // Best-effort STOP — if it fails the off-base daemon idles - // harmlessly (it has no clients and will only cost background CPU). - let _ = super::health::send_command(info.health_port, "STOP"); - return Ok(*lower_info); - } - } - Ok(info) + let start = Instant::now(); + loop { + if let Some(info) = discovery::discover_in(dir) { + return maybe_take_over(info, dir); + } + + // The state directory may not exist yet; the daemon creates it. + let lock = match std::fs::metadata(dir) { + Ok(_) => DaemonLock::try_acquire(dir)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => None::, + Err(error) => return Err(error), + }; + if let Some(lock) = lock { + // Nothing holds the lock, so no daemon is running and any record + // left behind is stale. Clean it up while we hold the lock so a + // daemon cannot start between the check and the removal. + discovery::remove_stale_record(dir); + drop(lock); + info!("no running daemon detected, spawning one"); + spawn()?; + return wait_for_daemon(dir, timeout); + } + if !dir.exists() { + info!("no state directory yet, spawning a daemon"); + spawn()?; + return wait_for_daemon(dir, timeout); + } + + // The lock is held but no live record answers: a daemon is starting + // or stopping. Wait for it; spawning a second one would only lose the + // race for the lock. + if start.elapsed() >= timeout { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "a daemon holds the lock in {} but did not become ready within {} seconds", + dir.display(), + timeout.as_secs() + ), + )); } - ScanOutcome::AllOccupied => Err(io::Error::new( - io::ErrorKind::AddrInUse, - format!( - "no free hyperdb daemon port in {}..{}", - scan.base, - scan.base.saturating_add(scan.span) - ), - )), + std::thread::sleep(POLL_INTERVAL); } } /// Spawn `hyperdb-mcp daemon` as a fully detached background process. -fn spawn_detached(port: u16) -> io::Result<()> { +fn spawn_detached() -> io::Result<()> { let exe = std::env::current_exe()?; - let port_str = port.to_string(); let mut cmd = Command::new(&exe); - cmd.arg("daemon").arg("--port").arg(&port_str); + cmd.arg("daemon"); // Detach from parent: redirect stdio to null cmd.stdin(std::process::Stdio::null()); @@ -118,20 +141,20 @@ fn spawn_detached(port: u16) -> io::Result<()> { } /// Poll for the discovery file to appear (daemon is ready). -fn wait_for_daemon() -> io::Result { +fn wait_for_daemon(dir: &Path, timeout: Duration) -> io::Result { let start = Instant::now(); loop { - if let Some(info) = discovery::discover() { + if let Some(info) = discovery::discover_in(dir) { info!(endpoint = %info.hyperd_endpoint, "daemon is ready"); return Ok(info); } - if start.elapsed() >= SPAWN_TIMEOUT { + if start.elapsed() >= timeout { return Err(io::Error::new( io::ErrorKind::TimedOut, format!( "daemon did not become ready within {} seconds", - SPAWN_TIMEOUT.as_secs() + timeout.as_secs() ), )); } @@ -154,19 +177,12 @@ pub fn client_should_take_over(client_ver: &str, daemon_ver: &str) -> bool { } /// Decide whether to reuse the running daemon or take it over with a newer version. -/// If the client is newer, we send STOP to the old daemon, wait for it to release -/// the port, then spawn a fresh daemon on the same port. Otherwise we reuse the -/// existing daemon. +/// If the client is newer, we send STOP to the old daemon, wait for it to stop +/// answering, then spawn a fresh daemon. Otherwise we reuse the existing daemon. /// -/// The `scan` argument is intentionally unused for *where* to respawn: a takeover -/// always reuses the port the discovered daemon already holds (`info.health_port`), -/// because that is the port guaranteed to free up when the old daemon stops. A -/// mid-session change to `HYPERDB_DAEMON_PORT` (so the pinned `scan.base` differs -/// from `info.health_port`) is not honored here — spawning on a *different* port -/// would leave the old daemon alive and create two daemons rather than replace one. -/// That edge case is pathological (operators don't repin a live daemon) and the -/// daemon found via discovery is authoritative for its own port. -fn maybe_take_over(info: DaemonInfo, _scan: PortScan) -> io::Result { +/// The health endpoint is the same for the old and the new daemon (it is derived +/// from the state directory), so a fresh daemon is told apart by its pid. +fn maybe_take_over(info: DaemonInfo, dir: &Path) -> io::Result { let client_ver = crate::version::MCP_VERSION; if !client_should_take_over(client_ver, &info.version) { @@ -178,7 +194,7 @@ fn maybe_take_over(info: DaemonInfo, _scan: PortScan) -> io::Result debug!( daemon_version = %info.version, client_version = %client_ver, - port = info.health_port, + endpoint = %info.health_endpoint, reason = if parse_failed { "version unparseable" } else { "client not newer" }, "reusing existing daemon" ); @@ -189,47 +205,45 @@ fn maybe_take_over(info: DaemonInfo, _scan: PortScan) -> io::Result info!( daemon_version = %info.version, client_version = %client_ver, - port = info.health_port, + endpoint = %info.health_endpoint, "newer MCP client taking over older daemon" ); + // `discover_in` only returns a record whose endpoint it validated against + // this state directory, so this parse succeeds for any `info` it produced. + let Some(endpoint) = HealthEndpoint::from_record(&info.health_endpoint, dir) else { + return Ok(info); + }; + // Send STOP (best-effort; ignore error if daemon is already dying). - let _ = super::health::send_command(info.health_port, "STOP"); + let _ = super::health::send_command(&endpoint, "STOP"); - // Wait for the old daemon to release the port (confirmed by ping_identified returning None). + // Wait for the old daemon to stop answering. let deadline = Instant::now() + SPAWN_TIMEOUT; while Instant::now() < deadline { if super::health::ping_identified( - info.health_port, + &endpoint, Duration::from_millis(200), Duration::from_millis(200), ) .is_none() { - // Port is free — spawn the new daemon on the same port. + // The old daemon is gone, so spawn the new one. // - // There is a benign TOCTOU window here: another client could also - // observe the freed port and spawn concurrently. That is safe by the - // same argument as the FreePort path — `spawn_detached` is - // fire-and-forget (it does not itself bind the port; the spawned - // daemon's `HealthListener::bind` is the real single-instance lock). - // The OS grants the bind to exactly one daemon; the loser exits at - // step 1 of `run_daemon` (before spawning hyperd or writing the - // discovery file), and `wait_for_daemon` (which polls `discover()`) - // converges on whichever daemon won. No duplicate daemon survives - // and no AddrInUse surfaces to the client. + // Another client could observe the same and spawn concurrently. + // That is safe: the daemon lock lets exactly one daemon through, the + // loser exits (or waits for the winner), and `wait_for_daemon` + // converges on whichever daemon won. // - // Defensive narrowing of that window: if a concurrent takeover has - // already published a fresh, identity-verified daemon on this port, - // adopt it instead of spawning — this avoids returning the stale - // `info` (old endpoint) we were carrying and skips a redundant spawn. - if let Some(fresh) = discovery::discover() - && fresh.health_port == info.health_port + // Narrowing that window: if a concurrent takeover has already + // published a fresh daemon, adopt it instead of spawning. + if let Some(fresh) = discovery::discover_in(dir) + && fresh.pid != info.pid { return Ok(fresh); } - spawn_detached(info.health_port)?; - return wait_for_daemon(); + spawn_detached()?; + return wait_for_daemon(dir, SPAWN_TIMEOUT); } std::thread::sleep(POLL_INTERVAL); } @@ -237,9 +251,100 @@ fn maybe_take_over(info: DaemonInfo, _scan: PortScan) -> io::Result // Old daemon didn't die within the deadline — log a warning and reuse it // rather than fail the client. warn!( - port = info.health_port, + endpoint = %info.health_endpoint, timeout_secs = SPAWN_TIMEOUT.as_secs(), "old daemon did not stop within timeout, reusing it" ); Ok(info) } + +#[cfg(test)] +mod tests { + use std::cell::Cell; + + use super::*; + + /// A spawner that only counts its calls and succeeds without starting + /// anything. + fn counting_spawner(calls: &Cell) -> impl FnOnce() -> io::Result<()> + '_ { + move || { + calls.set(calls.get() + 1); + Ok(()) + } + } + + /// A current-shape record whose endpoint nobody serves. + fn write_stale_record(dir: &Path) -> std::path::PathBuf { + let endpoint = HealthEndpoint::for_new_daemon(dir).expect("derive endpoint"); + let info = DaemonInfo { + pid: 4_242, + hyperd_endpoint: "127.0.0.1:1".to_string(), + health_endpoint: endpoint.as_str().to_string(), + started_at: "2026-01-01T00:00:00Z".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + }; + let path = dir.join("daemon.json"); + std::fs::write(&path, serde_json::to_vec(&info).expect("serialize record")) + .expect("write stale record"); + path + } + + /// A held lock with no answering daemon means one is starting or + /// stopping: never spawn a second, time out instead. + #[test] + fn held_lock_without_a_record_never_spawns_and_times_out() { + let dir = tempfile::tempdir().expect("state dir"); + let _held = DaemonLock::try_acquire(dir.path()) + .expect("lock probe") + .expect("fresh dir is lockable"); + let calls = Cell::new(0); + + let started = Instant::now(); + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(300), + ) + .expect_err("a held lock with no daemon must not succeed"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 0, "the spawner must never run"); + assert!(started.elapsed() >= Duration::from_millis(300)); + } + + /// With the lock free, a leftover record is stale: it is removed under + /// the lock and exactly one daemon is spawned. + #[test] + fn free_lock_with_a_stale_record_removes_it_and_spawns_once() { + let dir = tempfile::tempdir().expect("state dir"); + let record = write_stale_record(dir.path()); + let calls = Cell::new(0); + + // The fake spawner starts nothing, so the wait afterwards times out; + // what matters is what happened before it. + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + ) + .expect_err("the fake spawner publishes no daemon"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 1, "exactly one spawn"); + assert!(!record.exists(), "the stale record must have been removed"); + } + + /// No state directory yet: nobody ever ran a daemon here, so spawn one. + #[test] + fn missing_state_dir_spawns() { + let parent = tempfile::tempdir().expect("parent dir"); + let dir = parent.path().join("never-created"); + let calls = Cell::new(0); + + let error = ensure_daemon_in(&dir, counting_spawner(&calls), Duration::from_millis(200)) + .expect_err("the fake spawner publishes no daemon"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 1, "the spawner must run once"); + } +} diff --git a/hyperdb-mcp/src/daemon/state_perms.rs b/hyperdb-mcp/src/daemon/state_perms.rs index 9089f8ce..727efd75 100644 --- a/hyperdb-mcp/src/daemon/state_perms.rs +++ b/hyperdb-mcp/src/daemon/state_perms.rs @@ -48,6 +48,71 @@ const STATE_DIR_MODE: u32 = 0o700; /// File mode for state files: readable and writable only by the owner. const STATE_FILE_MODE: u32 = 0o600; +/// Check that `dir` is a state directory only the current user can write to. +/// +/// On Unix the state directory is what keeps other users away from the control +/// socket, the lock and the discovery record, so it is checked rather than +/// assumed: it must be owned by the effective user and carry no group or other +/// write bit. [`ensure_owner_only_dir`] tightens the mode of a directory this +/// user owns, so in practice the check fails for a directory someone else owns +/// (which `chmod` cannot fix) and for a filesystem that refuses `chmod`. The +/// daemon calls this after `ensure_owner_only_dir` and refuses to start on an +/// error; a client calls it on its own, without creating or changing anything, +/// and goes straight to local mode. +/// +/// A symlink to a directory is judged by its target. On Windows the default +/// location is per-user through ACL inheritance and the pipe DACL is the +/// boundary, so this always succeeds there. +/// +/// # Errors +/// Returns `PermissionDenied` naming the path when the directory is not owned +/// by the current user or is writable by its group or by others, and the +/// underlying error when it cannot be inspected. +pub fn verify_state_dir_trusted(dir: &Path) -> io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + + let metadata = std::fs::metadata(dir)?; + if !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::NotADirectory, + format!("the state directory {} is not a directory", dir.display()), + )); + } + // SAFETY: `geteuid` takes no arguments, reads no memory and cannot fail. + let effective_uid = unsafe { libc::geteuid() }; + if metadata.uid() != effective_uid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "the state directory {} is owned by another user (uid {}); \ + set HYPERDB_STATE_DIR to a directory you own", + dir.display(), + metadata.uid() + ), + )); + } + let mode = metadata.mode() & 0o777; + if mode & 0o022 != 0 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "the state directory {} is writable by its group or by others (mode \ + {mode:04o}); remove the group and other write bits", + dir.display() + ), + )); + } + Ok(()) + } + #[cfg(not(unix))] + { + let _ = dir; + Ok(()) + } +} + /// Create `dir`, and any parents this call creates, restricted to the owning /// user. /// @@ -693,4 +758,64 @@ mod tests { "no record must be published when its mode could not be restricted" ); } + + #[test] + fn trusted_check_accepts_an_owner_only_directory() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().join("state"); + ensure_owner_only_dir(&dir).unwrap(); + verify_state_dir_trusted(&dir).expect("a 0700 directory we own is trusted"); + } + + #[test] + fn trusted_check_refuses_a_group_or_other_writable_directory() { + let tmp = TempDir::new().unwrap(); + for mode in [0o770, 0o707, 0o777, 0o720] { + let dir = tmp.path().join(format!("state-{mode:o}")); + ensure_owner_only_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(mode)).unwrap(); + let error = verify_state_dir_trusted(&dir) + .expect_err("a directory writable by group or others must be refused"); + assert_eq!( + error.kind(), + io::ErrorKind::PermissionDenied, + "mode {mode:o}" + ); + assert!( + error.to_string().contains(&dir.display().to_string()), + "the error must name the path: {error}" + ); + } + } + + #[test] + fn trusted_check_tolerates_group_and_other_read() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().join("state"); + ensure_owner_only_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + verify_state_dir_trusted(&dir).expect("read and traverse for others is not a write risk"); + } + + #[test] + fn trusted_check_refuses_a_directory_owned_by_someone_else() { + // SAFETY: `geteuid` takes no arguments and cannot fail. + if unsafe { libc::geteuid() } == 0 { + return; // root owns "/" too, so there is nothing foreign to point at + } + let error = verify_state_dir_trusted(Path::new("/")) + .expect_err("the root directory is owned by root, not by this user"); + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert!( + error.to_string().contains("owned by another user"), + "{error}" + ); + } + + #[test] + fn trusted_check_reports_a_missing_directory() { + let tmp = TempDir::new().unwrap(); + let error = verify_state_dir_trusted(&tmp.path().join("absent")).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } } diff --git a/hyperdb-mcp/src/diagnostics.rs b/hyperdb-mcp/src/diagnostics.rs index 6ad60f40..84ffa26f 100644 --- a/hyperdb-mcp/src/diagnostics.rs +++ b/hyperdb-mcp/src/diagnostics.rs @@ -12,7 +12,8 @@ use std::time::{Duration, Instant}; use semver::Version; use serde::{Deserialize, Serialize}; -use crate::daemon::discovery::{DaemonRecord, PortScan, RawDiscoveryRead}; +use crate::daemon::control::HealthEndpoint; +use crate::daemon::discovery::{DaemonRecord, RawDiscoveryRead}; const MAX_LAUNCHER_INFO_BYTES: usize = 16 * 1024; const MAX_REPORTED_STRING_BYTES: usize = 4 * 1024; @@ -272,7 +273,9 @@ struct DoctorDaemonSection { state: DoctorDaemonState, pid: Option, hyperd_endpoint: Option, - health_port: Option, + health_endpoint: Option, + /// Whether a daemon currently holds the single-instance lock. + lock: DoctorLockState, started_at: Option, version: Option, mcp_version: Option, @@ -308,24 +311,26 @@ pub struct DoctorReport { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct DoctorMoment(pub(crate) u64); -/// Finite monotonic deadline shared by doctor scans and probes. +/// Whether the daemon lock is held, observed without taking it for long or +/// creating it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DoctorLockState { + /// The lock file does not exist, so no daemon has ever run here. + Absent, + /// A daemon holds the lock (running, starting or stopping). + Held, + /// The lock file exists and nothing holds it. + Free, + /// The lock could not be inspected. + Unknown, +} + +/// Finite monotonic deadline shared by doctor probes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct DoctorDeadline(pub(crate) u64); -/// Bounded candidate-scan request. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct DoctorScanRequest { - pub(crate) ports: PortScan, - pub(crate) deadline: DoctorDeadline, -} - -/// A candidate location returned by the bounded scanner. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct DoctorScanCandidate { - pub(crate) responding_port: u16, -} - -/// Raw outcome from fetching enriched `STATUS` at one candidate port. +/// Raw outcome from fetching enriched `STATUS` at the recorded health endpoint. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum DoctorStatusProbe { Unreachable, @@ -335,7 +340,6 @@ pub(crate) enum DoctorStatusProbe { /// Finite collection policy supplied independently of process globals. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct DoctorCollectRequest { - pub(crate) ports: PortScan, pub(crate) timeout: Duration, } @@ -352,7 +356,6 @@ pub(crate) enum DoctorDaemonState { Oversized, ParsedUnreachable, LiveFromDiscovery, - LiveFromScan, } /// One recorded discovery fact that disagrees with fresh enriched `STATUS`. @@ -381,24 +384,24 @@ pub(crate) enum DoctorDaemonWarning { MalformedDiscovery, OversizedDiscovery, DiscoveryCandidateUnreachable { - responding_port: u16, + health_endpoint: String, }, StaleOrReplacedDiscovery { mismatches: Vec, }, - StatusHealthPortMismatch { - responding_port: u16, - reported_port: u16, + StatusHealthEndpointMismatch { + recorded_endpoint: String, + reported_endpoint: String, }, MalformedStatus { - responding_port: u16, + health_endpoint: String, }, } -/// Fresh daemon facts accepted only after candidate-port verification. +/// Fresh daemon facts accepted only after verification against the recorded health endpoint. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct VerifiedDoctorDaemon { - pub(crate) responding_port: u16, + pub(crate) health_endpoint: String, pub(crate) record: DaemonRecord, } @@ -416,8 +419,7 @@ pub(crate) struct DoctorDaemonReport { /// mutation, and unbounded network operations. pub(crate) struct DoctorCollectorDependencies<'a> { pub(crate) read_raw_discovery: &'a dyn Fn() -> RawDiscoveryRead, - pub(crate) probe_enriched_status: &'a dyn Fn(u16, DoctorDeadline) -> DoctorStatusProbe, - pub(crate) scan_candidates: &'a dyn Fn(DoctorScanRequest) -> Vec, + pub(crate) probe_enriched_status: &'a dyn Fn(&str, DoctorDeadline) -> DoctorStatusProbe, pub(crate) now: &'a dyn Fn() -> DoctorMoment, pub(crate) deadline_after: &'a dyn Fn(DoctorMoment, Duration) -> DoctorDeadline, } @@ -451,8 +453,8 @@ pub(crate) fn collect_doctor_daemon( }; if let Some(recorded) = discovery_record.as_ref() { - let responding_port = recorded.info().health_port; - match verify_status_candidate(dependencies, responding_port, deadline, &mut warnings) { + let health_endpoint = recorded.info().health_endpoint.clone(); + match verify_status_candidate(dependencies, &health_endpoint, deadline, &mut warnings) { Some(fresh) => { let mismatches = discovery_fact_mismatches(recorded, &fresh); if !mismatches.is_empty() { @@ -461,7 +463,7 @@ pub(crate) fn collect_doctor_daemon( return DoctorDaemonReport { state: DoctorDaemonState::LiveFromDiscovery, verified: Some(VerifiedDoctorDaemon { - responding_port, + health_endpoint, record: fresh, }), warnings, @@ -469,33 +471,12 @@ pub(crate) fn collect_doctor_daemon( } None if warnings.is_empty() => { warnings - .push(DoctorDaemonWarning::DiscoveryCandidateUnreachable { responding_port }); + .push(DoctorDaemonWarning::DiscoveryCandidateUnreachable { health_endpoint }); } None => {} } } - for candidate in (dependencies.scan_candidates)(DoctorScanRequest { - ports: request.ports, - deadline, - }) { - if let Some(record) = verify_status_candidate( - dependencies, - candidate.responding_port, - deadline, - &mut warnings, - ) { - return DoctorDaemonReport { - state: DoctorDaemonState::LiveFromScan, - verified: Some(VerifiedDoctorDaemon { - responding_port: candidate.responding_port, - record, - }), - warnings, - }; - } - } - DoctorDaemonReport { state: fallback_state, verified: None, @@ -505,27 +486,31 @@ pub(crate) fn collect_doctor_daemon( fn verify_status_candidate( dependencies: &DoctorCollectorDependencies<'_>, - responding_port: u16, + health_endpoint: &str, deadline: DoctorDeadline, warnings: &mut Vec, ) -> Option { let DoctorStatusProbe::Response(response) = - (dependencies.probe_enriched_status)(responding_port, deadline) + (dependencies.probe_enriched_status)(health_endpoint, deadline) else { return None; }; let Ok(record) = serde_json::from_str::(&response) else { - warnings.push(DoctorDaemonWarning::MalformedStatus { responding_port }); + warnings.push(DoctorDaemonWarning::MalformedStatus { + health_endpoint: health_endpoint.to_owned(), + }); return None; }; if record.identity().is_none() { - warnings.push(DoctorDaemonWarning::MalformedStatus { responding_port }); + warnings.push(DoctorDaemonWarning::MalformedStatus { + health_endpoint: health_endpoint.to_owned(), + }); return None; } - if record.info().health_port != responding_port { - warnings.push(DoctorDaemonWarning::StatusHealthPortMismatch { - responding_port, - reported_port: record.info().health_port, + if record.info().health_endpoint != health_endpoint { + warnings.push(DoctorDaemonWarning::StatusHealthEndpointMismatch { + recorded_endpoint: health_endpoint.to_owned(), + reported_endpoint: record.info().health_endpoint.clone(), }); return None; } @@ -612,9 +597,12 @@ pub fn collect_doctor_report( let daemon_report = collect_real_doctor_daemon( discovery_path.as_deref(), state_error_kind, - crate::daemon::discovery::resolve_port_scan(), + state_dir.as_deref(), ); - let daemon = doctor_daemon_section(&daemon_report); + let mut daemon = doctor_daemon_section(&daemon_report); + daemon.lock = state_dir + .as_deref() + .map_or(DoctorLockState::Unknown, probe_daemon_lock); let catalog = crate::server::HyperMcpServer::doctor_catalog_snapshot(options.read_only)?; let mut warnings = installation @@ -623,6 +611,14 @@ pub fn collect_doctor_report( .map(identity_doctor_warning) .collect::>(); warnings.extend(daemon_report.warnings.iter().map(daemon_doctor_warning)); + if let Some(error) = state_dir.as_deref().and_then(untrusted_state_dir_error) { + warnings.push(doctor_warning( + "untrusted_state_dir", + format!( + "The daemon state directory is not trusted, so doctor did not read its record, connect to its socket or inspect its lock: {error}" + ), + )); + } if let Some(kind) = state_error_kind { warnings.push(doctor_warning( "daemon_state_path_unavailable", @@ -860,9 +856,14 @@ pub fn render_doctor_human(report: &DoctorReport) -> String { if let Some(endpoint) = report.daemon.hyperd_endpoint.as_deref() { let _ = writeln!(output, " Hyperd endpoint: {}", escape_human(endpoint)); } - if let Some(port) = report.daemon.health_port { - let _ = writeln!(output, " Health port: {port}"); + if let Some(endpoint) = report.daemon.health_endpoint.as_deref() { + let _ = writeln!(output, " Health endpoint: {}", escape_human(endpoint)); } + let _ = writeln!( + output, + " Daemon lock: {}", + lock_state_label(report.daemon.lock) + ); if let Some(started_at) = report.daemon.started_at.as_deref() { let _ = writeln!(output, " Started: {}", escape_human(started_at)); } @@ -1051,13 +1052,48 @@ fn resolve_configured_hyperd( ) } +/// Look at the daemon lock without creating it. Taking it for an instant is +/// the only way to learn whether it is held; a daemon that starts in that +/// instant simply retries. +fn probe_daemon_lock(state_dir: &Path) -> DoctorLockState { + if untrusted_state_dir_error(state_dir).is_some() { + return DoctorLockState::Unknown; + } + let lock_path = state_dir.join(crate::daemon::lock::LOCK_FILE_NAME); + match std::fs::symlink_metadata(&lock_path) { + Ok(metadata) if metadata.file_type().is_file() => {} + Ok(_) => return DoctorLockState::Unknown, + Err(error) if error.kind() == io::ErrorKind::NotFound => return DoctorLockState::Absent, + Err(_) => return DoctorLockState::Unknown, + } + match crate::daemon::lock::DaemonLock::try_acquire(state_dir) { + Ok(Some(_lock)) => DoctorLockState::Free, + Ok(None) => DoctorLockState::Held, + Err(_) => DoctorLockState::Unknown, + } +} + +/// Why `state_dir` must not be trusted (owned by someone else, or writable by +/// group or others), or `None` when it is trusted or does not exist. Doctor +/// neither reads a record from, connects through, nor locks in such a +/// directory: anything in it could have been planted by another account. +fn untrusted_state_dir_error(state_dir: &Path) -> Option { + crate::daemon::state_perms::verify_state_dir_trusted(state_dir) + .err() + .filter(|error| error.kind() != io::ErrorKind::NotFound) +} + fn collect_real_doctor_daemon( discovery_path: Option<&Path>, state_error_kind: Option, - ports: PortScan, + state_dir: Option<&Path>, ) -> DoctorDaemonReport { let origin = Instant::now(); - let probing_scan = std::cell::Cell::new(false); + let untrusted = state_dir.and_then(untrusted_state_dir_error); + let (discovery_path, state_error_kind, state_dir) = match &untrusted { + Some(error) => (None, Some(error.kind()), None), + None => (discovery_path, state_error_kind, state_dir), + }; let read_raw_discovery = || match discovery_path { Some(path) => crate::daemon::discovery::read_discovery_file_raw(path), None => RawDiscoveryRead::Unreadable { @@ -1065,19 +1101,15 @@ fn collect_real_doctor_daemon( kind: state_error_kind.unwrap_or(io::ErrorKind::NotFound), }, }; - let probe_enriched_status = |port: u16, deadline: DoctorDeadline| { - // Preserve the established scan handshake, but verify each identified - // port immediately instead of gathering PONGs across the whole range. - // Discovery candidates already have a recorded identity and go - // straight to the stronger fresh STATUS verification. - if probing_scan.get() { - match send_doctor_command(port, "PING", origin, deadline) { - Ok(response) if is_identified_doctor_pong(&response) => {} - _ => return DoctorStatusProbe::Unreachable, - } - } - - match send_doctor_command(port, "STATUS", origin, deadline) { + let probe_enriched_status = |recorded: &str, deadline: DoctorDeadline| { + // The record is untrusted input: only the endpoint this state + // directory implies is ever connected to, so a tampered record cannot + // point doctor at an arbitrary socket. + let Some(endpoint) = state_dir.and_then(|dir| HealthEndpoint::from_record(recorded, dir)) + else { + return DoctorStatusProbe::Unreachable; + }; + match send_doctor_command(&endpoint, "STATUS", origin, deadline) { Ok(response) => DoctorStatusProbe::Response(response), Err(error) if error.kind() == io::ErrorKind::InvalidData => { DoctorStatusProbe::Response(String::new()) @@ -1085,19 +1117,6 @@ fn collect_real_doctor_daemon( Err(_) => DoctorStatusProbe::Unreachable, } }; - let scan_candidates = |request: DoctorScanRequest| { - probing_scan.set(true); - let mut candidates = Vec::new(); - for offset in 0..request.ports.span { - let Some(port) = request.ports.base.checked_add(offset) else { - break; - }; - candidates.push(DoctorScanCandidate { - responding_port: port, - }); - } - candidates - }; let now = || DoctorMoment(elapsed_millis(origin)); let deadline_after = |now: DoctorMoment, timeout: Duration| { DoctorDeadline(now.0.saturating_add(duration_millis(timeout))) @@ -1105,14 +1124,12 @@ fn collect_real_doctor_daemon( let dependencies = DoctorCollectorDependencies { read_raw_discovery: &read_raw_discovery, probe_enriched_status: &probe_enriched_status, - scan_candidates: &scan_candidates, now: &now, deadline_after: &deadline_after, }; collect_doctor_daemon( &dependencies, DoctorCollectRequest { - ports, timeout: DOCTOR_DAEMON_TIMEOUT, }, ) @@ -1143,21 +1160,20 @@ fn duration_millis(duration: Duration) -> u64 { } fn send_doctor_command( - port: u16, + endpoint: &HealthEndpoint, command: &str, origin: Instant, deadline: DoctorDeadline, ) -> io::Result { use std::io::{Read, Write}; - let address = std::net::SocketAddr::from(([127, 0, 0, 1], port)); let connect_timeout = doctor_network_timeout(origin, deadline)?; - let mut stream = std::net::TcpStream::connect_timeout(&address, connect_timeout)?; + let mut stream = crate::daemon::control::connect(endpoint, connect_timeout)?; let message = format!("{command}\n"); let mut written = 0; while written < message.len() { let timeout = doctor_network_timeout(origin, deadline)?; - stream.set_write_timeout(Some(timeout))?; + stream.set_io_timeout(timeout)?; match stream.write(&message.as_bytes()[written..]) { Ok(0) => { return Err(io::Error::new( @@ -1175,7 +1191,7 @@ fn send_doctor_command( let mut chunk = [0_u8; 1024]; loop { let timeout = doctor_network_timeout(origin, deadline)?; - stream.set_read_timeout(Some(timeout))?; + crate::daemon::health::rearm_io_timeout(&mut stream, timeout)?; let remaining_capacity = MAX_STATUS_RESPONSE_BYTES .saturating_add(1) .saturating_sub(response.len()); @@ -1223,18 +1239,14 @@ fn send_doctor_command( .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string())) } -fn is_identified_doctor_pong(response: &str) -> bool { - let mut tokens = response.split_whitespace(); - tokens.next() == Some("PONG") && tokens.next() == Some(crate::daemon::health::PONG_TOKEN) -} - fn doctor_daemon_section(report: &DoctorDaemonReport) -> DoctorDaemonSection { let Some(verified) = report.verified.as_ref() else { return DoctorDaemonSection { state: report.state, pid: None, hyperd_endpoint: None, - health_port: None, + health_endpoint: None, + lock: DoctorLockState::Unknown, started_at: None, version: None, mcp_version: None, @@ -1246,7 +1258,8 @@ fn doctor_daemon_section(report: &DoctorDaemonReport) -> DoctorDaemonSection { state: report.state, pid: Some(verified.record.info().pid), hyperd_endpoint: Some(bounded_string(&verified.record.info().hyperd_endpoint)), - health_port: Some(verified.record.info().health_port), + health_endpoint: Some(bounded_string(&verified.record.info().health_endpoint)), + lock: DoctorLockState::Unknown, started_at: Some(bounded_string(&verified.record.info().started_at)), version: Some(bounded_string(&verified.record.info().version)), mcp_version: identity.map(|identity| bounded_string(identity.mcp_version())), @@ -1328,10 +1341,11 @@ fn daemon_doctor_warning(warning: &DoctorDaemonWarning) -> DoctorWarning { "daemon_discovery_oversized", "The daemon discovery file is valid but larger than any legitimate record should be; it was left unchanged.", ), - DoctorDaemonWarning::DiscoveryCandidateUnreachable { responding_port } => doctor_warning( + DoctorDaemonWarning::DiscoveryCandidateUnreachable { health_endpoint } => doctor_warning( "daemon_discovery_candidate_unreachable", format!( - "The recorded daemon candidate on port {responding_port} did not return fresh enriched STATUS." + "The recorded daemon candidate at {} did not return fresh enriched STATUS.", + bounded_string(health_endpoint) ), ), DoctorDaemonWarning::StaleOrReplacedDiscovery { mismatches } => { @@ -1345,19 +1359,22 @@ fn daemon_doctor_warning(warning: &DoctorDaemonWarning) -> DoctorWarning { format!("Fresh daemon STATUS disagreed with the discovery record: {facts}."), ) } - DoctorDaemonWarning::StatusHealthPortMismatch { - responding_port, - reported_port, + DoctorDaemonWarning::StatusHealthEndpointMismatch { + recorded_endpoint, + reported_endpoint, } => doctor_warning( - "daemon_status_health_port_mismatch", + "daemon_status_health_endpoint_mismatch", format!( - "STATUS from port {responding_port} reported health port {reported_port}; the candidate was rejected." + "STATUS from {} reported health endpoint {}; the candidate was rejected.", + bounded_string(recorded_endpoint), + bounded_string(reported_endpoint) ), ), - DoctorDaemonWarning::MalformedStatus { responding_port } => doctor_warning( + DoctorDaemonWarning::MalformedStatus { health_endpoint } => doctor_warning( "daemon_status_malformed", format!( - "Port {responding_port} returned malformed or unenriched STATUS; the candidate was rejected." + "{} returned malformed or unenriched STATUS; the candidate was rejected.", + bounded_string(health_endpoint) ), ), } @@ -1466,7 +1483,15 @@ const fn daemon_state_label(state: DoctorDaemonState) -> &'static str { DoctorDaemonState::Oversized => "oversized", DoctorDaemonState::ParsedUnreachable => "parsed_unreachable", DoctorDaemonState::LiveFromDiscovery => "live_from_discovery", - DoctorDaemonState::LiveFromScan => "live_from_scan", + } +} + +const fn lock_state_label(state: DoctorLockState) -> &'static str { + match state { + DoctorLockState::Absent => "absent", + DoctorLockState::Held => "held", + DoctorLockState::Free => "free", + DoctorLockState::Unknown => "unknown", } } @@ -1684,7 +1709,7 @@ pub(crate) fn real_network_test_guard() -> std::sync::MutexGuard<'static, ()> { mod tests { use std::cell::RefCell; use std::ffi::OsStr; - use std::io; + use std::io::{self, Read as _, Write as _}; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -1695,17 +1720,20 @@ mod tests { use serde_json::{Value, json}; use tempfile::TempDir; + use crate::daemon::control::{ControlListener, ControlStream, HealthEndpoint}; use crate::daemon::discovery::{ - DaemonBuildIdentity, DaemonInfo, DaemonRecord, DiscoveryBytes, PortScan, RawDiscoveryRead, + DaemonBuildIdentity, DaemonInfo, DaemonRecord, DiscoveryBytes, RawDiscoveryRead, read_discovery_file_raw, }; use crate::daemon::health::{DaemonState, HealthListener}; + use crate::daemon::lock::{DaemonLock, LOCK_FILE_NAME}; use super::{ DiscoveryFactMismatch, DoctorCollectRequest, DoctorCollectorDependencies, - DoctorDaemonState, DoctorDaemonWarning, DoctorDeadline, DoctorMoment, DoctorScanCandidate, - DoctorScanRequest, DoctorStatusProbe, ReportedPath, collect_doctor_daemon, - collect_real_doctor_daemon, real_network_test_guard, + DoctorDaemonState, DoctorDaemonWarning, DoctorDeadline, DoctorLockState, DoctorMoment, + DoctorStatusProbe, MAX_STATUS_RESPONSE_BYTES, ReportedPath, collect_doctor_daemon, + collect_real_doctor_daemon, probe_daemon_lock, real_network_test_guard, + send_doctor_command, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -1742,11 +1770,11 @@ mod tests { } } - fn enriched_status(pid: u32, health_port: u16, build: &str, executable: &str) -> Value { + fn enriched_status(pid: u32, health_endpoint: &str, build: &str, executable: &str) -> Value { json!({ "pid": pid, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": health_port, + "health_endpoint": health_endpoint, "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0", "identity": { @@ -1756,455 +1784,135 @@ mod tests { }) } - fn listener_daemon_info(pid: u32, health_port: u16) -> DaemonInfo { + fn listener_daemon_info(pid: u32, health_endpoint: &HealthEndpoint) -> DaemonInfo { DaemonInfo { pid, hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port, + health_endpoint: health_endpoint.as_str().to_owned(), started_at: "2026-08-13T12:34:56Z".to_string(), version: "0.7.0".to_string(), } } - fn adjacent_fake_and_health_listeners() -> (std::net::TcpListener, HealthListener) { - for _ in 0..128 { - let fake = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); - let fake_port = fake.local_addr().unwrap().port(); - let Some(health_port) = fake_port.checked_add(1) else { - continue; - }; - if let Ok(health) = HealthListener::bind(health_port) { - return (fake, health); - } - } - panic!("could not reserve adjacent OS-selected loopback listeners after 128 attempts"); + /// The endpoint a daemon with state directory `dir` would serve. + fn endpoint_in(dir: &TempDir) -> HealthEndpoint { + HealthEndpoint::for_new_daemon(dir.path()).expect("health endpoint for the test state dir") } - fn health_listener_with_adjacent_followers( - follower_count: u16, - ) -> (HealthListener, Vec) { - for _ in 0..128 { - let health = HealthListener::bind(0).unwrap(); - let Some(last_port) = health.port.checked_add(follower_count) else { - continue; - }; - let mut followers = Vec::with_capacity(usize::from(follower_count)); - for port in health.port + 1..=last_port { - let Ok(listener) = std::net::TcpListener::bind(("127.0.0.1", port)) else { - break; - }; - followers.push(listener); - } - if followers.len() == usize::from(follower_count) { - return (health, followers); - } - } - panic!("could not reserve a contiguous OS-selected loopback range"); + /// Write `status` as the state directory's `daemon.json` and return its path. + fn write_discovery(dir: &TempDir, status: &Value) -> std::path::PathBuf { + let path = dir.path().join("daemon.json"); + std::fs::write(&path, serde_json::to_vec(status).unwrap()).unwrap(); + path } - fn run_invalid_status_then_later_daemon_attempt() -> Vec { - use std::io::{BufRead as _, BufReader, Write as _}; - - let tmp = TempDir::new().unwrap(); - let (fake_listener, health_listener) = adjacent_fake_and_health_listeners(); - let fake_port = fake_listener.local_addr().unwrap().port(); - let health_port = health_listener.port; - fake_listener.set_nonblocking(true).unwrap(); - - let stop_fake = Arc::new(AtomicBool::new(false)); - let fake_stop = Arc::clone(&stop_fake); - let served_ping = Arc::new(AtomicUsize::new(0)); - let fake_served_ping = Arc::clone(&served_ping); - let served_status = Arc::new(AtomicUsize::new(0)); - let fake_served_status = Arc::clone(&served_status); - let (fake_ready_sender, fake_ready_receiver) = mpsc::sync_channel(1); - let wrong_port_status = enriched_status( - 9_090, - health_port, - "0.7.0.rwrong-port", - "/opt/hyperdb/wrong-port-daemon", - ) - .to_string(); - let fake_server = std::thread::spawn(move || -> Result<(), String> { - fake_ready_sender - .send(()) - .map_err(|_| "fake candidate readiness receiver closed".to_string())?; - let deadline = Instant::now() + Duration::from_secs(2); - while !fake_stop.load(Ordering::Acquire) && Instant::now() < deadline { - let mut stream = match fake_listener.accept() { - Ok((stream, _)) => stream, - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(error) => return Err(format!("fake candidate accept failed: {error}")), - }; - stream - .set_read_timeout(Some(Duration::from_millis(200))) - .map_err(|error| error.to_string())?; - let mut command = String::new(); - let read_result = - BufReader::new(stream.try_clone().map_err(|error| error.to_string())?) - .read_line(&mut command); - match read_result { - Ok(0) if command.is_empty() => continue, - Err(error) - if command.is_empty() - && matches!( - error.kind(), - io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock - ) => - { - continue; - } - Ok(_) => {} - Err(error) => { - return Err(format!( - "fake candidate command read failed after {} bytes: {error}", - command.len() - )); - } - } - let (response, served) = match command.trim() { - "PING" => ("PONG hyperdb-mcp 0.7.0\n".to_string(), &fake_served_ping), - "STATUS" => (format!("{wrong_port_status}\n"), &fake_served_status), - other => return Err(format!("fake candidate received {other:?}")), - }; - stream - .write_all(response.as_bytes()) - .map_err(|error| error.to_string())?; - served.fetch_add(1, Ordering::AcqRel); + /// Accept one client on a fake peer, giving up after `window` or once + /// `stop` is set. + fn accept_client( + listener: &mut ControlListener, + window: Duration, + stop: &AtomicBool, + ) -> Result, String> { + let deadline = Instant::now() + window; + while !stop.load(Ordering::Acquire) && Instant::now() < deadline { + match listener.accept_timeout(Duration::from_millis(20)) { + Ok(Some(stream)) => return Ok(Some(stream)), + Ok(None) => {} + Err(error) => return Err(format!("fake peer accept failed: {error}")), } - Ok(()) - }); - - let health_state = Arc::new(DaemonState::new()); - let health_info = Arc::new(Mutex::new(listener_daemon_info(9_191, health_port))); - let run_state = Arc::clone(&health_state); - let run_info = Arc::clone(&health_info); - let (health_ready_sender, health_ready_receiver) = mpsc::sync_channel(1); - let health_server = std::thread::spawn(move || { - let _ = health_ready_sender.send(()); - health_listener.run(run_state, run_info); - }); - - let mut failures = Vec::new(); - let fake_ready = fake_ready_receiver - .recv_timeout(Duration::from_millis(500)) - .is_ok(); - if !fake_ready { - failures.push("fake candidate did not signal readiness within 500ms".to_string()); - } - let health_ready = health_ready_receiver - .recv_timeout(Duration::from_millis(500)) - .is_ok(); - if !health_ready { - failures.push("later HealthListener did not signal readiness within 500ms".to_string()); } - - let report = if fake_ready && health_ready { - if let Ok(report) = catch_unwind(AssertUnwindSafe(|| { - collect_real_doctor_daemon( - Some(&tmp.path().join("missing-daemon.json")), - None, - PortScan { - base: fake_port, - span: 2, - }, - ) - })) { - Some(report) - } else { - failures.push("doctor collector panicked during adjacent scan".to_string()); - None + Ok(None) + } + + /// Read the one command line a doctor probe sends. + fn read_command(stream: &mut ControlStream) -> Result { + stream + .set_io_timeout(Duration::from_secs(2)) + .map_err(|error| format!("fake peer set timeout failed: {error}"))?; + let mut command = Vec::new(); + let mut byte = [0_u8; 1]; + loop { + match stream.read(&mut byte) { + Ok(0) => break, + Ok(_) if byte[0] == b'\n' => break, + Ok(_) => command.push(byte[0]), + Err(error) => return Err(format!("fake peer command read failed: {error}")), } - } else { - None - }; - - stop_fake.store(true, Ordering::Release); - health_state.request_shutdown(); - match fake_server.join() { - Ok(Ok(())) => {} - Ok(Err(error)) => failures.push(error), - Err(_) => failures.push("fake candidate server panicked".to_string()), - } - if health_server.join().is_err() { - failures.push("later HealthListener server panicked".to_string()); } - - let ping_count = served_ping.load(Ordering::Acquire); - let status_count = served_status.load(Ordering::Acquire); - if ping_count == 0 || status_count == 0 { - failures.push(format!( - "fake candidate served PING {ping_count} time(s) and STATUS {status_count} time(s); both must be demonstrated" - )); - } - if let Some(report) = report { - if report.state != DoctorDaemonState::LiveFromScan { - failures.push(format!( - "adjacent scan state was {:?}, expected LiveFromScan", - report.state - )); - } - match report.verified.as_ref() { - Some(verified) - if verified.responding_port == health_port - && verified.record.info().health_port == health_port - && verified.record.info().pid == 9_191 => {} - other => failures.push(format!( - "later real HealthListener did not supply exact fresh facts: {other:?}" - )), - } - if !report.warnings.iter().any(|warning| { - matches!( - warning, - DoctorDaemonWarning::StatusHealthPortMismatch { - responding_port, - reported_port, - } if *responding_port == fake_port && *reported_port == health_port - ) - }) { - failures.push(format!( - "first candidate's wrong-port STATUS was not retained as a warning: {:?}", - report.warnings - )); - } - } - - failures + Ok(String::from_utf8_lossy(&command).into_owned()) } - #[test] - fn real_scan_skips_invalid_status_candidate_and_finds_later_daemon() { - const MAX_SCENARIO_ATTEMPTS: usize = 3; - - let _network_guard = real_network_test_guard(); - let mut attempt_failures = Vec::new(); - for attempt in 1..=MAX_SCENARIO_ATTEMPTS { - let failures = run_invalid_status_then_later_daemon_attempt(); - if failures.is_empty() { - return; - } - attempt_failures.push(format!("attempt {attempt}:\n{}", failures.join("\n"))); + /// Keep a fake peer's end open until the client hangs up (or a bounded + /// wait elapses). Closing first can make a later `set_io_timeout` on the + /// client fail on macOS, which would mask the behavior under test. + fn hold_until_client_closes(stream: &mut ControlStream) { + if stream.set_io_timeout(Duration::from_secs(1)).is_err() { + return; } - - panic!( - "adjacent candidate scan failed all {MAX_SCENARIO_ATTEMPTS} bounded attempts:\n{}", - attempt_failures.join("\n") - ); + let mut scratch = [0_u8; 256]; + while matches!(stream.read(&mut scratch), Ok(count) if count > 0) {} } - #[test] - fn real_scan_verifies_early_daemon_before_slow_later_ports() { - use std::io::{BufRead as _, BufReader, Write as _}; - - const FOLLOWER_COUNT: u16 = 4; - - let _network_guard = real_network_test_guard(); - let (health_listener, slow_listeners) = - health_listener_with_adjacent_followers(FOLLOWER_COUNT); - let health_port = health_listener.port; - - let health_state = Arc::new(DaemonState::new()); - let health_info = Arc::new(Mutex::new(listener_daemon_info(7_171, health_port))); - let run_state = Arc::clone(&health_state); - let run_info = Arc::clone(&health_info); - let health_server = std::thread::spawn(move || health_listener.run(run_state, run_info)); - - let stop_slow_peers = Arc::new(AtomicBool::new(false)); - let slow_peer_commands = Arc::new(AtomicUsize::new(0)); - let mut slow_servers = Vec::new(); - for listener in slow_listeners { - listener.set_nonblocking(true).unwrap(); - let stop = Arc::clone(&stop_slow_peers); - let command_count = Arc::clone(&slow_peer_commands); - slow_servers.push(std::thread::spawn(move || -> Result<(), String> { - let deadline = Instant::now() + Duration::from_secs(2); - while !stop.load(Ordering::Acquire) && Instant::now() < deadline { - let mut stream = match listener.accept() { - Ok((stream, _)) => stream, - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(error) => { - return Err(format!("slow follower accept failed: {error}")); - } - }; - stream - .set_read_timeout(Some(Duration::from_millis(200))) - .map_err(|error| error.to_string())?; - let mut command = String::new(); - let read_result = - BufReader::new(stream.try_clone().map_err(|error| error.to_string())?) - .read_line(&mut command); - match read_result { - Ok(0) if command.is_empty() => continue, - Err(error) - if command.is_empty() - && matches!( - error.kind(), - io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock - ) => - { - continue; - } - Ok(_) => {} - Err(error) => { - return Err(format!( - "slow follower command read failed after {} bytes: {error}", - command.len() - )); - } - } - if command.trim() != "PING" { - return Err(format!( - "slow follower received unexpected command {command:?}" - )); - } - command_count.fetch_add(1, Ordering::AcqRel); - - let response_at = Instant::now() + Duration::from_millis(110); - while !stop.load(Ordering::Acquire) && Instant::now() < response_at { - std::thread::sleep(Duration::from_millis(2)); - } - if stop.load(Ordering::Acquire) { - break; - } - match stream.write_all(b"PONG hyperdb-mcp 0.7.0\n") { - Ok(()) => {} - Err(error) - if matches!( - error.kind(), - io::ErrorKind::BrokenPipe - | io::ErrorKind::ConnectionReset - | io::ErrorKind::NotConnected - ) => {} - Err(error) => { - return Err(format!("slow follower PONG failed: {error}")); - } - } - } - Ok(()) - })); - } - - let tmp = TempDir::new().unwrap(); - let missing_discovery = tmp.path().join("missing-daemon.json"); - let (result_sender, result_receiver) = mpsc::channel(); - let collector = std::thread::spawn(move || { - let started = Instant::now(); - let report = collect_real_doctor_daemon( - Some(&missing_discovery), - None, - PortScan { - base: health_port, - span: FOLLOWER_COUNT + 1, - }, - ); - let _ = result_sender.send((report, started.elapsed())); - }); - - let bounded_result = result_receiver.recv_timeout(Duration::from_millis(650)); - stop_slow_peers.store(true, Ordering::Release); - health_state.request_shutdown(); - let slow_results = slow_servers - .into_iter() - .map(|server| server.join().unwrap()) - .collect::>(); - health_server.join().unwrap(); - collector.join().unwrap(); - - let mut failures = slow_results - .into_iter() - .filter_map(Result::err) - .collect::>(); - match bounded_result { - Ok((report, elapsed)) => { - if elapsed > Duration::from_millis(650) { - failures.push(format!( - "early-daemon scan completed after its 650ms watchdog: {elapsed:?}" - )); - } - if report.state != DoctorDaemonState::LiveFromScan { - failures.push(format!( - "early-daemon scan state was {:?}, expected LiveFromScan", - report.state - )); - } - match report.verified { - Some(verified) - if verified.responding_port == health_port - && verified.record.info().health_port == health_port - && verified.record.info().pid == 7_171 => {} - other => failures.push(format!( - "healthy first port did not supply exact fresh daemon facts: {other:?}" - )), - } - } - Err(mpsc::RecvTimeoutError::Timeout) => failures.push( - "later identified peers starved the healthy first port past 650ms".to_string(), - ), - Err(mpsc::RecvTimeoutError::Disconnected) => { - failures.push("early-daemon collector disconnected without a report".to_string()); - } - } - let later_commands = slow_peer_commands.load(Ordering::Acquire); - if later_commands != 0 { - failures.push(format!( - "{later_commands} later peer command(s) ran after the healthy first port" - )); - } - - assert!( - failures.is_empty(), - "early daemon scan failures:\n{}", - failures.join("\n") - ); + /// A fake STATUS peer: accepts one client, records its command, and + /// answers with `reply` followed by a newline. + fn spawn_status_peer( + mut listener: ControlListener, + reply: String, + ) -> std::thread::JoinHandle> { + std::thread::spawn(move || { + let stop = AtomicBool::new(false); + let Some(mut stream) = accept_client(&mut listener, Duration::from_secs(3), &stop)? + else { + return Err("status peer never received a connection".to_string()); + }; + let command = read_command(&mut stream)?; + stream + .write_all(format!("{reply}\n").as_bytes()) + .map_err(|error| format!("status peer write failed: {error}"))?; + hold_until_client_closes(&mut stream); + Ok(command) + }) } #[test] - fn real_health_listener_accept_cadence_fits_doctor_budget() { + fn real_health_listener_answers_doctor_within_budget() { let _network_guard = real_network_test_guard(); - let listener = HealthListener::bind(0).unwrap(); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(listener_daemon_info(8_181, port))); + let dir = TempDir::new().unwrap(); + let endpoint = endpoint_in(&dir); + let listener = HealthListener::bind(&endpoint).unwrap(); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(listener_daemon_info(8_181, &endpoint))); let run_state = Arc::clone(&state); let run_info = Arc::clone(&info); + // Start serving late so the doctor's connection waits in the accept + // backlog, pinning that the listener's accept cadence fits the budget. let listener_thread = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(75)); listener.run(run_state, run_info); }); - let tmp = TempDir::new().unwrap(); - let missing_discovery = tmp.path().join("missing-daemon.json"); - let started = Instant::now(); - let report = collect_real_doctor_daemon( - Some(&missing_discovery), - None, - PortScan { - base: port, - span: 1, - }, + let discovery = write_discovery( + &dir, + &enriched_status(8_181, endpoint.as_str(), "0.7.0.rrecorded", "/opt/recorded"), ); + let started = Instant::now(); + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(dir.path())); let elapsed = started.elapsed(); state.request_shutdown(); listener_thread.join().unwrap(); let mut failures = Vec::new(); - if report.state != DoctorDaemonState::LiveFromScan { + if report.state != DoctorDaemonState::LiveFromDiscovery { failures.push(format!( - "real HealthListener state was {:?}, expected LiveFromScan", + "real HealthListener state was {:?}, expected LiveFromDiscovery", report.state )); } match report.verified { Some(verified) - if verified.responding_port == port - && verified.record.info().health_port == port + if verified.health_endpoint == endpoint.as_str() + && verified.record.info().health_endpoint == endpoint.as_str() && verified.record.info().pid == 8_181 => {} other => failures.push(format!( "real HealthListener did not yield exact fresh daemon facts: {other:?}" @@ -2224,47 +1932,133 @@ mod tests { } #[test] - fn real_doctor_collector_enforces_global_deadline_against_slow_drip() { - use std::io::{BufRead as _, BufReader, Write as _}; - use std::net::{Shutdown, TcpListener}; + fn real_doctor_reports_a_mismatched_status_endpoint_and_discards_the_peer() { + let _network_guard = real_network_test_guard(); + let dir = TempDir::new().unwrap(); + let endpoint = endpoint_in(&dir); + let listener = ControlListener::bind(&endpoint).unwrap(); + let reported = "/somewhere/else/daemon.sock"; + let peer = spawn_status_peer( + listener, + enriched_status(707, reported, "0.7.0.rwrong", "/opt/hyperdb/wrong-daemon").to_string(), + ); + let discovery = write_discovery( + &dir, + &enriched_status( + 707, + endpoint.as_str(), + "0.7.0.rwrong", + "/opt/hyperdb/wrong-daemon", + ), + ); + + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(dir.path())); + let observed_command = peer.join().unwrap(); + + assert_eq!(observed_command.as_deref(), Ok("STATUS")); + assert!( + report.verified.is_none(), + "a mismatched peer must not verify" + ); + assert_eq!(report.state, DoctorDaemonState::ParsedUnreachable); + assert_eq!( + report.warnings, + vec![DoctorDaemonWarning::StatusHealthEndpointMismatch { + recorded_endpoint: endpoint.as_str().to_owned(), + reported_endpoint: reported.to_owned(), + }] + ); + } + + #[test] + fn real_doctor_reports_an_oversized_status_as_malformed() { + let _network_guard = real_network_test_guard(); + let dir = TempDir::new().unwrap(); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).unwrap(); + let server = std::thread::spawn(move || -> Result<(), String> { + let stop = AtomicBool::new(false); + let Some(mut stream) = accept_client(&mut listener, Duration::from_secs(3), &stop)? + else { + return Err("oversized peer never received a connection".to_string()); + }; + read_command(&mut stream)?; + // More than the fixed cap and never a newline; the client closing + // early surfaces as a write error, which is the expected outcome. + let flood = vec![b'a'; MAX_STATUS_RESPONSE_BYTES + 4096]; + let _ = stream.write_all(&flood); + hold_until_client_closes(&mut stream); + Ok(()) + }); + let discovery = write_discovery( + &dir, + &enriched_status( + 909, + endpoint.as_str(), + "0.7.0.rbig", + "/opt/hyperdb/big-daemon", + ), + ); + + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(dir.path())); + server.join().unwrap().unwrap(); + + assert!(report.verified.is_none()); + assert_eq!( + report.warnings, + vec![DoctorDaemonWarning::MalformedStatus { + health_endpoint: endpoint.as_str().to_owned(), + }] + ); + } + + #[test] + fn send_doctor_command_rejects_a_response_past_the_size_cap() { + let _network_guard = real_network_test_guard(); + let dir = TempDir::new().unwrap(); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).unwrap(); + let server = std::thread::spawn(move || -> Result<(), String> { + let stop = AtomicBool::new(false); + let Some(mut stream) = accept_client(&mut listener, Duration::from_secs(3), &stop)? + else { + return Err("size-cap peer never received a connection".to_string()); + }; + read_command(&mut stream)?; + let flood = vec![b'a'; MAX_STATUS_RESPONSE_BYTES + 4096]; + let _ = stream.write_all(&flood); + hold_until_client_closes(&mut stream); + Ok(()) + }); + + let origin = Instant::now(); + let error = send_doctor_command(&endpoint, "STATUS", origin, DoctorDeadline(2_000)) + .expect_err("a response past the cap must be rejected"); + server.join().unwrap().unwrap(); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + #[test] + fn real_doctor_collector_enforces_global_deadline_against_slow_drip() { let _network_guard = real_network_test_guard(); - let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); - let port = listener.local_addr().unwrap().port(); - listener.set_nonblocking(true).unwrap(); + let dir = TempDir::new().unwrap(); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).unwrap(); let stop_writer = Arc::new(AtomicBool::new(false)); let server_stop = Arc::clone(&stop_writer); let server = std::thread::spawn(move || -> Result<(), String> { - let accept_deadline = Instant::now() + Duration::from_secs(2); - let mut stream = loop { - match listener.accept() { - Ok((stream, _)) => break stream, - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { - if server_stop.load(Ordering::Acquire) || Instant::now() >= accept_deadline - { - return Err("slow-drip peer never received a connection".to_string()); - } - std::thread::sleep(Duration::from_millis(5)); - } - Err(error) => return Err(format!("slow-drip accept failed: {error}")), - } + let Some(mut stream) = + accept_client(&mut listener, Duration::from_secs(2), &server_stop)? + else { + return Err("slow-drip peer never received a connection".to_string()); }; - stream - .set_nodelay(true) - .map_err(|error| error.to_string())?; - stream - .set_read_timeout(Some(Duration::from_millis(200))) - .map_err(|error| error.to_string())?; - let mut command = String::new(); - BufReader::new(stream.try_clone().map_err(|error| error.to_string())?) - .read_line(&mut command) - .map_err(|error| error.to_string())?; - if command.trim() != "PING" { + let command = read_command(&mut stream)?; + if command != "STATUS" { return Err(format!( "slow-drip peer received unexpected command {command:?}" )); } - let write_deadline = Instant::now() + Duration::from_secs(2); while !server_stop.load(Ordering::Acquire) && Instant::now() < write_deadline { if stream.write_all(b"x").is_err() || stream.flush().is_err() { @@ -2272,23 +2066,24 @@ mod tests { } std::thread::sleep(Duration::from_millis(10)); } - let _ = stream.shutdown(Shutdown::Both); + let _ = stream.finish(); Ok(()) }); - let tmp = TempDir::new().unwrap(); - let missing_discovery = tmp.path().join("missing-daemon.json"); + let discovery = write_discovery( + &dir, + &enriched_status( + 616, + endpoint.as_str(), + "0.7.0.rdrip", + "/opt/hyperdb/drip-daemon", + ), + ); + let state_dir = dir.path().to_path_buf(); let (result_sender, result_receiver) = mpsc::channel(); let collector = std::thread::spawn(move || { let started = Instant::now(); - let report = collect_real_doctor_daemon( - Some(&missing_discovery), - None, - PortScan { - base: port, - span: 1, - }, - ); + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(&state_dir)); let _ = result_sender.send((report, started.elapsed())); }); @@ -2310,7 +2105,7 @@ mod tests { } } Err(mpsc::RecvTimeoutError::Timeout) => failures.push( - "real collector exceeded 650ms because each drip reset its socket read timeout" + "real collector exceeded 650ms because each drip reset its read timeout" .to_string(), ), Err(mpsc::RecvTimeoutError::Disconnected) => { @@ -2337,52 +2132,32 @@ mod tests { /// so the candidate is reported unreachable instead. #[test] fn real_doctor_reports_a_silent_peer_close_as_unreachable_not_malformed() { - use std::io::{BufRead as _, BufReader}; - use std::net::TcpListener; - let _network_guard = real_network_test_guard(); - let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); - let port = listener.local_addr().unwrap().port(); + let dir = TempDir::new().unwrap(); + let endpoint = endpoint_in(&dir); + let mut listener = ControlListener::bind(&endpoint).unwrap(); let server = std::thread::spawn(move || -> Result { - let (stream, _) = listener - .accept() - .map_err(|error| format!("silent-close peer accept failed: {error}"))?; - stream - .set_read_timeout(Some(Duration::from_secs(2))) - .map_err(|error| error.to_string())?; - let mut command = String::new(); - BufReader::new(&stream) - .read_line(&mut command) - .map_err(|error| format!("silent-close peer read failed: {error}"))?; + let stop = AtomicBool::new(false); + let Some(mut stream) = accept_client(&mut listener, Duration::from_secs(3), &stop)? + else { + return Err("silent-close peer never received a connection".to_string()); + }; + let command = read_command(&mut stream)?; // Close having written nothing at all. drop(stream); - Ok(command.trim().to_string()) + Ok(command) }); - let tmp = TempDir::new().unwrap(); - let discovery_path = tmp.path().join("daemon.json"); - std::fs::write( - &discovery_path, - serde_json::to_vec(&enriched_status( + let discovery = write_discovery( + &dir, + &enriched_status( 4_242, - port, + endpoint.as_str(), "0.7.0.rsilent", "/opt/hyperdb/bin/hyperdb-mcp", - )) - .unwrap(), - ) - .unwrap(); - - // `span: 0` leaves the port scan with no candidates, so the report - // reflects the discovery candidate's outcome alone. - let report = collect_real_doctor_daemon( - Some(&discovery_path), - None, - PortScan { - base: port, - span: 0, - }, + ), ); + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(dir.path())); let observed_command = server.join().unwrap(); let mut failures = Vec::new(); @@ -2404,7 +2179,7 @@ mod tests { } if report.warnings != vec![DoctorDaemonWarning::DiscoveryCandidateUnreachable { - responding_port: port, + health_endpoint: endpoint.as_str().to_owned(), }] { failures.push(format!( @@ -2421,30 +2196,202 @@ mod tests { ); } + /// The record is untrusted input: an endpoint other than the one the state + /// directory implies must never be connected to, even when a live peer is + /// listening there. + #[cfg(unix)] + #[test] + fn real_doctor_never_connects_to_a_recorded_endpoint_outside_the_state_dir() { + let _network_guard = real_network_test_guard(); + let state = TempDir::new().unwrap(); + let elsewhere = TempDir::new().unwrap(); + let foreign_endpoint = endpoint_in(&elsewhere); + let mut listener = ControlListener::bind(&foreign_endpoint).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let peer_stop = Arc::clone(&stop); + let connections = Arc::new(AtomicUsize::new(0)); + let peer_connections = Arc::clone(&connections); + let peer = std::thread::spawn(move || -> Result<(), String> { + while let Some(stream) = + accept_client(&mut listener, Duration::from_secs(2), &peer_stop)? + { + peer_connections.fetch_add(1, Ordering::AcqRel); + drop(stream); + } + Ok(()) + }); + + let discovery = write_discovery( + &state, + &enriched_status( + 1_313, + foreign_endpoint.as_str(), + "0.7.0.rforeign", + "/opt/hyperdb/foreign-daemon", + ), + ); + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(state.path())); + // Give a wrongly-issued connection time to arrive before stopping. + std::thread::sleep(Duration::from_millis(100)); + stop.store(true, Ordering::Release); + peer.join().unwrap().unwrap(); + + assert_eq!(connections.load(Ordering::Acquire), 0); + assert!(report.verified.is_none()); + assert_eq!(report.state, DoctorDaemonState::ParsedUnreachable); + assert_eq!( + report.warnings, + vec![DoctorDaemonWarning::DiscoveryCandidateUnreachable { + health_endpoint: foreign_endpoint.as_str().to_owned(), + }] + ); + } + + #[cfg(unix)] + #[test] + fn real_doctor_rejects_a_recorded_path_that_is_not_the_state_dirs_socket() { + let state = TempDir::new().unwrap(); + let record = state.path().join("other.sock"); + let discovery = write_discovery( + &state, + &enriched_status( + 1_414, + &record.to_string_lossy(), + "0.7.0.rother", + "/opt/hyperdb/other-daemon", + ), + ); + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(state.path())); + assert_eq!(report.state, DoctorDaemonState::ParsedUnreachable); + assert!(report.verified.is_none()); + assert_eq!( + report.warnings, + vec![DoctorDaemonWarning::DiscoveryCandidateUnreachable { + health_endpoint: record.to_string_lossy().into_owned(), + }] + ); + + // Without a resolvable state directory nothing is ever connected to. + let report = collect_real_doctor_daemon(Some(&discovery), None, None); + assert_eq!(report.state, DoctorDaemonState::ParsedUnreachable); + assert!(report.verified.is_none()); + } + + /// A state directory another account can write to is never read from, + /// connected through or locked: a planted socket sees no connection and + /// no lock file appears. + #[cfg(unix)] + #[test] + fn real_doctor_ignores_an_untrusted_state_dir() { + use std::os::unix::fs::PermissionsExt as _; + + let _network_guard = real_network_test_guard(); + let state = TempDir::new().unwrap(); + let endpoint = endpoint_in(&state); + let mut listener = ControlListener::bind(&endpoint).unwrap(); + let discovery = write_discovery( + &state, + &enriched_status(1_515, endpoint.as_str(), "0.7.0.rplanted", "/opt/planted"), + ); + std::fs::set_permissions(state.path(), std::fs::Permissions::from_mode(0o770)).unwrap(); + + let stop = Arc::new(AtomicBool::new(false)); + let peer_stop = Arc::clone(&stop); + let connections = Arc::new(AtomicUsize::new(0)); + let peer_connections = Arc::clone(&connections); + let peer = std::thread::spawn(move || -> Result<(), String> { + while let Some(stream) = + accept_client(&mut listener, Duration::from_secs(2), &peer_stop)? + { + peer_connections.fetch_add(1, Ordering::AcqRel); + drop(stream); + } + Ok(()) + }); + + let report = collect_real_doctor_daemon(Some(&discovery), None, Some(state.path())); + let lock = probe_daemon_lock(state.path()); + std::thread::sleep(Duration::from_millis(100)); + stop.store(true, Ordering::Release); + peer.join().unwrap().unwrap(); + + assert_eq!(connections.load(Ordering::Acquire), 0); + assert!(report.verified.is_none()); + assert_eq!(report.state, DoctorDaemonState::Unreadable); + assert_eq!(lock, DoctorLockState::Unknown); + assert!( + !state + .path() + .join(crate::daemon::lock::LOCK_FILE_NAME) + .exists(), + "doctor must not create the lock file" + ); + assert!(crate::diagnostics::untrusted_state_dir_error(state.path()).is_some()); + } + + #[test] + fn lock_probe_reports_an_absent_lock_without_creating_it() { + let dir = TempDir::new().unwrap(); + assert_eq!(probe_daemon_lock(dir.path()), DoctorLockState::Absent); + assert!( + !dir.path().join(LOCK_FILE_NAME).exists(), + "probing must not create the lock file" + ); + } + + #[test] + fn lock_probe_reports_a_free_lock_once_its_holder_is_gone() { + let dir = TempDir::new().unwrap(); + drop( + DaemonLock::try_acquire(dir.path()) + .unwrap() + .expect("first acquire"), + ); + assert_eq!(probe_daemon_lock(dir.path()), DoctorLockState::Free); + // The probe took the lock for an instant and must have released it. + assert!( + DaemonLock::try_acquire(dir.path()).unwrap().is_some(), + "the probe must not leave the lock held" + ); + } + + #[test] + fn lock_probe_reports_a_held_lock() { + let dir = TempDir::new().unwrap(); + let _held = DaemonLock::try_acquire(dir.path()) + .unwrap() + .expect("acquire"); + assert_eq!(probe_daemon_lock(dir.path()), DoctorLockState::Held); + } + + #[test] + fn lock_probe_reports_a_non_file_lock_path_as_unknown() { + let dir = TempDir::new().unwrap(); + std::fs::create_dir(dir.path().join(LOCK_FILE_NAME)).unwrap(); + assert_eq!(probe_daemon_lock(dir.path()), DoctorLockState::Unknown); + } + #[test] fn collect_doctor_state_matrix_is_pure() { struct Case { name: &'static str, raw: RawFixture, - scan_ports: Vec, - status_responses: Vec<(u16, Value)>, + status_responses: Vec<(&'static str, Value)>, expected_state: DoctorDaemonState, - expected_live: Option<(u32, u16, &'static str)>, + expected_live: Option<(u32, &'static str, &'static str)>, expected_warnings: Vec, } let discovery_live = enriched_status( 4_242, - 7_486, + "/virtual/live/daemon.sock", "0.7.0.rdiscovery", "/opt/hyperdb/discovery-daemon", ); - let scan_live = enriched_status(5_151, 7_487, "0.7.0.rscan", "/opt/hyperdb/scanned-daemon"); let cases = vec![ Case { name: "missing", raw: RawFixture::Missing, - scan_ports: vec![], status_responses: vec![], expected_state: DoctorDaemonState::Missing, expected_live: None, @@ -2453,7 +2400,6 @@ mod tests { Case { name: "unreadable", raw: RawFixture::Unreadable(io::ErrorKind::PermissionDenied), - scan_ports: vec![], status_responses: vec![], expected_state: DoctorDaemonState::Unreadable, expected_live: None, @@ -2464,7 +2410,6 @@ mod tests { Case { name: "malformed", raw: RawFixture::Malformed, - scan_ports: vec![], status_responses: vec![], expected_state: DoctorDaemonState::Malformed, expected_live: None, @@ -2473,7 +2418,6 @@ mod tests { Case { name: "oversized", raw: RawFixture::Oversized, - scan_ports: vec![], status_responses: vec![], expected_state: DoctorDaemonState::Oversized, expected_live: None, @@ -2483,34 +2427,23 @@ mod tests { name: "parsed-unreachable", raw: RawFixture::Parsed(enriched_status( 4_040, - 7_485, + "/virtual/stale/daemon.sock", "0.7.0.rstale", "/opt/hyperdb/stale-daemon", )), - scan_ports: vec![], status_responses: vec![], expected_state: DoctorDaemonState::ParsedUnreachable, expected_live: None, expected_warnings: vec![DoctorDaemonWarning::DiscoveryCandidateUnreachable { - responding_port: 7_485, + health_endpoint: "/virtual/stale/daemon.sock".to_string(), }], }, Case { name: "live-from-discovery", raw: RawFixture::Parsed(discovery_live.clone()), - scan_ports: vec![], - status_responses: vec![(7_486, discovery_live)], + status_responses: vec![("/virtual/live/daemon.sock", discovery_live)], expected_state: DoctorDaemonState::LiveFromDiscovery, - expected_live: Some((4_242, 7_486, "0.7.0.rdiscovery")), - expected_warnings: vec![], - }, - Case { - name: "live-from-scan", - raw: RawFixture::Missing, - scan_ports: vec![7_487], - status_responses: vec![(7_487, scan_live)], - expected_state: DoctorDaemonState::LiveFromScan, - expected_live: Some((5_151, 7_487, "0.7.0.rscan")), + expected_live: Some((4_242, "/virtual/live/daemon.sock", "0.7.0.rdiscovery")), expected_warnings: vec![], }, ]; @@ -2523,28 +2456,17 @@ mod tests { operations.borrow_mut().push("raw-reader".to_string()); case.raw.read() }; - let probe_enriched_status = |port: u16, deadline: DoctorDeadline| { + let probe_enriched_status = |endpoint: &str, deadline: DoctorDeadline| { operations .borrow_mut() - .push(format!("status-prober:{port}:{}", deadline.0)); + .push(format!("status-prober:{endpoint}:{}", deadline.0)); case.status_responses .iter() - .find(|(candidate, _)| *candidate == port) + .find(|(candidate, _)| *candidate == endpoint) .map_or(DoctorStatusProbe::Unreachable, |(_, response)| { DoctorStatusProbe::Response(response.to_string()) }) }; - let scan_candidates = |request: DoctorScanRequest| { - operations.borrow_mut().push(format!( - "bounded-scanner:{}:{}:{}", - request.ports.base, request.ports.span, request.deadline.0 - )); - case.scan_ports - .iter() - .copied() - .map(|responding_port| DoctorScanCandidate { responding_port }) - .collect() - }; let now = || { operations.borrow_mut().push("clock".to_string()); DoctorMoment(10_000) @@ -2562,15 +2484,10 @@ mod tests { let dependencies = DoctorCollectorDependencies { read_raw_discovery: &read_raw_discovery, probe_enriched_status: &probe_enriched_status, - scan_candidates: &scan_candidates, now: &now, deadline_after: &deadline_after, }; let request = DoctorCollectRequest { - ports: PortScan { - base: 7_485, - span: 4, - }, timeout: Duration::from_millis(275), }; @@ -2586,10 +2503,10 @@ mod tests { } match (report.verified.as_ref(), case.expected_live) { (None, None) => {} - (Some(verified), Some((pid, port, build))) => { - if verified.responding_port != port + (Some(verified), Some((pid, endpoint, build))) => { + if verified.health_endpoint != endpoint || verified.record.info().pid != pid - || verified.record.info().health_port != port + || verified.record.info().health_endpoint != endpoint || verified .record .identity() @@ -2632,7 +2549,6 @@ mod tests { .filter(|operation| { !operation.starts_with("raw-reader") && !operation.starts_with("status-prober") - && !operation.starts_with("bounded-scanner") && !operation.starts_with("clock") && !operation.starts_with("deadline") }) @@ -2664,30 +2580,36 @@ mod tests { struct Case { name: &'static str, raw: RawFixture, - scan_ports: Vec, - probes: Vec<(u16, ProbeFixture)>, + probes: Vec<(&'static str, ProbeFixture)>, expected_state: DoctorDaemonState, expected_fresh_pid: Option, - expected_probed_ports: Vec, - expected_scan: bool, + expected_probed: Vec<&'static str>, expected_warnings: Vec, } let recorded_executable = ReportedPath::from_os_str(OsStr::new("/opt/hyperdb/recorded-daemon")); let fresh_executable = ReportedPath::from_os_str(OsStr::new("/opt/hyperdb/fresh-daemon")); - let recorded = enriched_status(101, 8_000, "0.7.0.rrecorded", &recorded_executable.display); - let fresh = enriched_status(202, 8_000, "0.7.0.rfresh", &fresh_executable.display); + let recorded = enriched_status( + 101, + "/virtual/a/daemon.sock", + "0.7.0.rrecorded", + &recorded_executable.display, + ); + let fresh = enriched_status( + 202, + "/virtual/a/daemon.sock", + "0.7.0.rfresh", + &fresh_executable.display, + ); let cases = vec![ Case { name: "discovery-is-refetched-and-fresh-facts-win", raw: RawFixture::Parsed(recorded), - scan_ports: vec![], - probes: vec![(8_000, ProbeFixture::Response(fresh))], + probes: vec![("/virtual/a/daemon.sock", ProbeFixture::Response(fresh))], expected_state: DoctorDaemonState::LiveFromDiscovery, expected_fresh_pid: Some(202), - expected_probed_ports: vec![8_000], - expected_scan: false, + expected_probed: vec!["/virtual/a/daemon.sock"], expected_warnings: vec![DoctorDaemonWarning::StaleOrReplacedDiscovery { mismatches: vec![ DiscoveryFactMismatch::Pid { @@ -2706,105 +2628,68 @@ mod tests { }], }, Case { - name: "discovery-status-health-port-must-match-responder", + name: "discovery-status-health-endpoint-must-match-responder", raw: RawFixture::Parsed(enriched_status( 303, - 8_001, + "/virtual/b/daemon.sock", "0.7.0.rrecorded", "/opt/hyperdb/discovery-candidate", )), - scan_ports: vec![], probes: vec![( - 8_001, + "/virtual/b/daemon.sock", ProbeFixture::Response(enriched_status( 404, - 9_001, + "/virtual/other/daemon.sock", "0.7.0.rfresh", "/opt/hyperdb/other-daemon", )), )], expected_state: DoctorDaemonState::ParsedUnreachable, expected_fresh_pid: None, - expected_probed_ports: vec![8_001], - expected_scan: true, - expected_warnings: vec![DoctorDaemonWarning::StatusHealthPortMismatch { - responding_port: 8_001, - reported_port: 9_001, + expected_probed: vec!["/virtual/b/daemon.sock"], + expected_warnings: vec![DoctorDaemonWarning::StatusHealthEndpointMismatch { + recorded_endpoint: "/virtual/b/daemon.sock".to_string(), + reported_endpoint: "/virtual/other/daemon.sock".to_string(), }], }, Case { name: "malformed-discovery-status-is-not-live-evidence", raw: RawFixture::Parsed(enriched_status( 505, - 8_002, + "/virtual/c/daemon.sock", "0.7.0.rrecorded", "/opt/hyperdb/discovery-candidate", )), - scan_ports: vec![], - probes: vec![(8_002, ProbeFixture::Malformed)], + probes: vec![("/virtual/c/daemon.sock", ProbeFixture::Malformed)], expected_state: DoctorDaemonState::ParsedUnreachable, expected_fresh_pid: None, - expected_probed_ports: vec![8_002], - expected_scan: true, + expected_probed: vec!["/virtual/c/daemon.sock"], expected_warnings: vec![DoctorDaemonWarning::MalformedStatus { - responding_port: 8_002, + health_endpoint: "/virtual/c/daemon.sock".to_string(), }], }, Case { - name: "scan-hit-is-refetched-before-becoming-live", - raw: RawFixture::Missing, - scan_ports: vec![8_003], - probes: vec![( - 8_003, - ProbeFixture::Response(enriched_status( - 606, - 8_003, - "0.7.0.rscan-fresh", - "/opt/hyperdb/scan-fresh-daemon", - )), - )], - expected_state: DoctorDaemonState::LiveFromScan, - expected_fresh_pid: Some(606), - expected_probed_ports: vec![8_003], - expected_scan: true, - expected_warnings: vec![], - }, - Case { - name: "scan-status-health-port-must-match-responder", + name: "missing-discovery-probes-nothing", raw: RawFixture::Missing, - scan_ports: vec![8_004], - probes: vec![( - 8_004, - ProbeFixture::Response(enriched_status( - 707, - 9_004, - "0.7.0.rwrong-port", - "/opt/hyperdb/wrong-port-daemon", - )), - )], + probes: vec![], expected_state: DoctorDaemonState::Missing, expected_fresh_pid: None, - expected_probed_ports: vec![8_004], - expected_scan: true, - expected_warnings: vec![DoctorDaemonWarning::StatusHealthPortMismatch { - responding_port: 8_004, - reported_port: 9_004, - }], + expected_probed: vec![], + expected_warnings: vec![], }, ]; let mut failures = Vec::new(); for case in cases { let raw_before = case.raw.clone(); - let probed_ports = RefCell::new(Vec::new()); - let scan_requests = RefCell::new(Vec::new()); + let probed = RefCell::new(Vec::new()); let read_raw_discovery = || case.raw.read(); - let probe_enriched_status = |port: u16, deadline: DoctorDeadline| { - probed_ports.borrow_mut().push((port, deadline)); + let probe_enriched_status = |endpoint: &str, deadline: DoctorDeadline| { + probed.borrow_mut().push((endpoint.to_owned(), deadline)); match case .probes .iter() - .find(|(candidate, _)| *candidate == port) + .find(|(candidate, _)| *candidate == endpoint) .map(|(_, response)| response) { Some(ProbeFixture::Response(response)) => { @@ -2816,14 +2701,6 @@ mod tests { None => DoctorStatusProbe::Unreachable, } }; - let scan_candidates = |request: DoctorScanRequest| { - scan_requests.borrow_mut().push(request); - case.scan_ports - .iter() - .copied() - .map(|responding_port| DoctorScanCandidate { responding_port }) - .collect() - }; let now = || DoctorMoment(60_000); let deadline_after = |now: DoctorMoment, timeout: Duration| { DoctorDeadline( @@ -2835,15 +2712,10 @@ mod tests { let dependencies = DoctorCollectorDependencies { read_raw_discovery: &read_raw_discovery, probe_enriched_status: &probe_enriched_status, - scan_candidates: &scan_candidates, now: &now, deadline_after: &deadline_after, }; let request = DoctorCollectRequest { - ports: PortScan { - base: 8_000, - span: 5, - }, timeout: Duration::from_millis(125), }; @@ -2880,18 +2752,18 @@ mod tests { )), } - let actual_ports = probed_ports + let actual_probed = probed .borrow() .iter() - .map(|(port, _)| *port) + .map(|(endpoint, _)| endpoint.clone()) .collect::>(); - if actual_ports != case.expected_probed_ports { + if actual_probed != case.expected_probed { failures.push(format!( - "{}: STATUS probes were {actual_ports:?}, expected {:?}", - case.name, case.expected_probed_ports + "{}: STATUS probes were {actual_probed:?}, expected {:?}", + case.name, case.expected_probed )); } - if probed_ports + if probed .borrow() .iter() .any(|(_, deadline)| *deadline != DoctorDeadline(60_125)) @@ -2901,29 +2773,6 @@ mod tests { case.name )); } - - let expected_scan_requests = usize::from(case.expected_scan); - if scan_requests.borrow().len() != expected_scan_requests { - failures.push(format!( - "{}: bounded scanner call count was {}, expected {expected_scan_requests}", - case.name, - scan_requests.borrow().len() - )); - } - for scan_request in scan_requests.borrow().iter() { - if scan_request.ports - != (PortScan { - base: 8_000, - span: 5, - }) - || scan_request.deadline != DoctorDeadline(60_125) - { - failures.push(format!( - "{}: scan request was not bounded to five ports and 125ms: {scan_request:?}", - case.name - )); - } - } if case.raw != raw_before { failures.push(format!( "{}: candidate verification mutated the raw record", @@ -2976,7 +2825,7 @@ mod tests { let info = record.info(); assert_eq!(info.pid, 4242); assert_eq!(info.hyperd_endpoint, "127.0.0.1:54321"); - assert_eq!(info.health_port, 7485); + assert_eq!(info.health_endpoint, "/virtual/state/daemon.sock"); assert_eq!(info.started_at, "2026-08-13T12:34:56Z"); assert_eq!(info.version, "0.7.0"); @@ -2997,7 +2846,7 @@ mod tests { let old_wire = json!({ "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7485, + "health_endpoint": "/virtual/state/daemon.sock", "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0" }); @@ -3008,7 +2857,7 @@ mod tests { let enriched_wire = json!({ "pid": 4242, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7485, + "health_endpoint": "/virtual/state/daemon.sock", "started_at": "2026-08-13T12:34:56Z", "version": "0.7.0", "identity": { diff --git a/hyperdb-mcp/src/engine.rs b/hyperdb-mcp/src/engine.rs index 77605915..17af9492 100644 --- a/hyperdb-mcp/src/engine.rs +++ b/hyperdb-mcp/src/engine.rs @@ -419,10 +419,10 @@ pub struct Engine { hyper: Option, /// Stored endpoint for daemon mode (the daemon advertises this). daemon_endpoint: Option, - /// The daemon's health port, if connected via daemon mode. `None` in local mode. - /// Used by the server's heartbeat logic to target the correct port (not a re-resolve, - /// which would break when scanning is enabled). - daemon_health_port: Option, + /// The daemon's health endpoint (Unix socket path or named pipe), if + /// connected via daemon mode. `None` in local mode. Used by the server's + /// heartbeat logic to reach the daemon this engine actually connected to. + daemon_health_endpoint: Option, connection: Connection, /// The primary database for this session. Lives in a temp dir and is /// deleted on `Drop`. @@ -574,7 +574,7 @@ impl Engine { Ok(Self { hyper: Some(hyper), daemon_endpoint: None, - daemon_health_port: None, + daemon_health_endpoint: None, connection, ephemeral_path, persistent_path, @@ -607,13 +607,22 @@ impl Engine { persistent_path: Option, log_dir: &Path, ) -> Result, McpError> { - let info = match daemon::spawn::ensure_daemon(daemon::discovery::resolve_port_scan()) { + let info = match daemon::spawn::ensure_daemon() { Ok(info) => info, Err(e) => { tracing::debug!(error = %e, "daemon unavailable, falling back to local mode"); return Ok(None); } }; + // Discovery only returns a record whose endpoint it validated against + // this state directory, so a failure here means the directory moved + // underneath us; local mode is the safe answer. + let Some(health_endpoint) = daemon::discovery::state_dir().ok().and_then(|dir| { + daemon::control::HealthEndpoint::from_record(&info.health_endpoint, &dir) + }) else { + tracing::debug!("daemon health endpoint is not valid here, falling back to local mode"); + return Ok(None); + }; let endpoint = &info.hyperd_endpoint; // CreateAndReplace: same rationale as the local path — a per-pid @@ -627,7 +636,7 @@ impl Engine { // The daemon's discovery file points at this endpoint but we can't // reach it — hyperd is likely dead. Tell the daemon so it can // restart it on its next monitor tick. - daemon::health::report_hyperd_error_to_daemon(info.health_port); + daemon::health::report_hyperd_error_to_daemon(&health_endpoint); McpError::new( ErrorCode::InternalError, format!("Failed to connect to daemon hyperd at {endpoint}: {e}"), @@ -637,7 +646,7 @@ impl Engine { bootstrap_public_schema(&connection)?; // Send heartbeat so daemon knows we're active - let _ = daemon::health::send_command(info.health_port, "HEARTBEAT"); + let _ = daemon::health::send_command(&health_endpoint, "HEARTBEAT"); let primary_db_name = path_stem(ephemeral_path); let persistent_was_created = Self::attach_persistent_if_present( @@ -649,7 +658,7 @@ impl Engine { Ok(Some(Self { hyper: None, daemon_endpoint: Some(info.hyperd_endpoint), - daemon_health_port: Some(info.health_port), + daemon_health_endpoint: Some(health_endpoint), connection, ephemeral_path: ephemeral_path.to_path_buf(), persistent_path, @@ -666,15 +675,15 @@ impl Engine { /// whichever transport it names (a Unix domain socket, a named pipe, or /// TCP) — the same endpoint queries run against. This reflects *current* /// liveness of the resource the engine actually depends on, and is robust - /// to two failure modes the health-port PING is not: - /// - the health port being unreachable (stale `daemon.json`, - /// port-scan-adopted daemon, firewall) while the libpq endpoint serves; + /// to two failure modes the health-endpoint PING is not: + /// - the health endpoint being unreachable (stale `daemon.json`) while + /// the libpq endpoint serves; /// - the daemon restarting `hyperd` at a new endpoint, leaving the cached /// one stale (the probe then correctly reports `false`); the Unix socket /// path is stable across restarts, so there the same probe instead /// observes the live replacement. /// - /// Falls back to discovery (`daemon.json` + health-port PING) only when no + /// Falls back to discovery (`daemon.json` + health-endpoint PING) only when no /// endpoint has been cached yet (before the first connection attempt). pub fn is_running(&self) -> bool { if let Some(ref hyper) = self.hyper { @@ -706,10 +715,18 @@ impl Engine { .map_err(|e| McpError::new(ErrorCode::InternalError, e.to_string())) } - /// The daemon's health port, if this engine is connected via daemon mode. - /// Returns `None` in local mode (when this engine owns a private `HyperProcess`). - pub fn daemon_health_port(&self) -> Option { - self.daemon_health_port + /// The daemon's health endpoint (a Unix socket path or a named-pipe name), + /// if this engine is connected via daemon mode. Returns `None` in local mode + /// (when this engine owns a private `HyperProcess`). + pub fn daemon_health_endpoint(&self) -> Option<&str> { + self.daemon_health_endpoint + .as_ref() + .map(daemon::control::HealthEndpoint::as_str) + } + + /// The typed form of [`Self::daemon_health_endpoint`], for sending commands. + pub(crate) fn daemon_health_endpoint_handle(&self) -> Option<&daemon::control::HealthEndpoint> { + self.daemon_health_endpoint.as_ref() } /// Absolute path to the ephemeral primary `.hyper` file on disk. @@ -1806,18 +1823,18 @@ impl Engine { }); // Connection details for the backing `hyperd`. In daemon mode the - // endpoint and health port come from the shared daemon's discovery + // endpoint and health endpoint come from the shared daemon's discovery // file; in local mode (`--no-daemon`) this engine owns a private - // `hyperd` and there is no health port. `hyperd_endpoint()` only errors + // `hyperd` and there is no health endpoint. `hyperd_endpoint()` only errors // if no endpoint is available at all, which `is_running` already // reflects — surface it as null rather than failing the whole status. let in_daemon_mode = self.daemon_endpoint.is_some(); let endpoint = self.hyperd_endpoint().ok(); let connection_value = endpoint.as_deref().map_or(Value::Null, describe_endpoint); let endpoint_value = endpoint.map_or(Value::Null, Value::String); - let health_port_value = self - .daemon_health_port - .map_or(Value::Null, |p| Value::Number(p.into())); + let health_endpoint_value = self + .daemon_health_endpoint() + .map_or(Value::Null, |endpoint| Value::String(endpoint.to_string())); Ok(json!({ "hyperd_running": self.is_running(), @@ -1828,15 +1845,15 @@ impl Engine { "total_rows": total_rows, "disk_usage_bytes": disk_bytes, // Where this engine is talking to hyperd. `hyperd_endpoint` is the - // libpq endpoint queries run against; `daemon_health_port` is the - // shared daemon's control/lock port (null in local mode); + // libpq endpoint queries run against; `daemon_health_endpoint` is the + // shared daemon's control socket or pipe (null in local mode); // `connection` decomposes the endpoint into transport, host/port // (TCP only), socket path (IPC only), and the scheme-qualified // descriptor another Hyper client can connect with. "engine": { "mode": if in_daemon_mode { "daemon" } else { "local" }, "hyperd_endpoint": endpoint_value, - "daemon_health_port": health_port_value, + "daemon_health_endpoint": health_endpoint_value, "connection": connection_value, }, // The MCP server and the `hyperdb-api` crate it's built on live in diff --git a/hyperdb-mcp/src/main.rs b/hyperdb-mcp/src/main.rs index 96c3adec..e3596475 100644 --- a/hyperdb-mcp/src/main.rs +++ b/hyperdb-mcp/src/main.rs @@ -127,23 +127,13 @@ enum Commands { json: bool, }, - /// Run a foreground daemon managing shared hyperd. Auto-spawn scans before - /// launching; foreground startup binds its configured/base port exactly. + /// Run a foreground daemon managing shared hyperd. One daemon runs per + /// state directory (`~/.hyperdb`, or `HYPERDB_STATE_DIR`); it listens on a + /// per-user Unix socket or named pipe there, not on a TCP port. Daemon { #[command(subcommand)] action: Option, - /// Exact TCP health/lock port for foreground startup. Without `--port`, - /// the foreground daemon binds the configured/base port exactly - /// (`HYPERDB_DAEMON_PORT` when valid, otherwise 7485) and does not scan. - /// Auto-spawn performs bounded discovery from its configured base before - /// launching. For stop/status, omitting the port uses discovery plus - /// scanning. Must be 1-65535: port 0 asks the OS for an *ephemeral* - /// port, which contradicts the exact bind promised here and leaves a - /// daemon on a port no client's scan can ever find. - #[arg(long, global = true, value_parser = clap::value_parser!(u16).range(1..))] - port: Option, - /// Idle timeout in seconds before the daemon shuts down #[arg(long)] idle_timeout: Option, @@ -167,29 +157,22 @@ async fn main() -> Result<(), Box> { Some(Commands::Doctor { json }) => run_doctor_mode(&cli, json), Some(Commands::Daemon { action: Some(DaemonAction::Stop), - port, .. }) => { - daemon_stop(port); + daemon_stop(); Ok(()) } Some(Commands::Daemon { action: Some(DaemonAction::Status), - port, .. }) => { - daemon_status(port); + daemon_status(); Ok(()) } Some(Commands::Daemon { action: None, - port, idle_timeout, - }) => { - // Resolve the effective port for daemon startup - let effective_port = port.unwrap_or_else(|| discovery::resolve_port_scan().base); - run_daemon_mode(effective_port, idle_timeout).await - } + }) => run_daemon_mode(idle_timeout).await, None => run_mcp_mode(cli).await, } } @@ -214,10 +197,7 @@ fn run_doctor_mode(cli: &Cli, json: bool) -> Result<(), Box, -) -> Result<(), Box> { +async fn run_daemon_mode(idle_timeout: Option) -> Result<(), Box> { // Daemon logs go to ~/.hyperdb/logs/. They record the hyperd endpoint, so // both the state directory and the log directory inside it are restricted // to the owning user, as `daemon.json` is. @@ -238,7 +218,7 @@ async fn run_daemon_mode( .with(fmt::layer().with_writer(file_writer).with_ansi(false)) .init(); - let config = DaemonConfig::from_args(port, idle_timeout); + let config = DaemonConfig::from_args(idle_timeout); daemon::run::run_daemon(config).await } @@ -303,57 +283,104 @@ async fn run_mcp_mode(cli: Cli) -> Result<(), Box> { Ok(()) } -fn daemon_stop(port: Option) { - let target_port = match port { - Some(p) => p, - None => { - // No explicit port — discover the running daemon - if let Some(info) = discovery::find_running_daemon() { - info.health_port - } else { - eprintln!("No daemon is currently running."); - std::process::exit(1); - } - } +/// Find the running daemon and the endpoint to reach it on, or exit 1. +/// +/// The record in `daemon.json` is the normal locator. A live daemon whose +/// record is missing or unusable is still found through the daemon lock: when +/// it is held, the daemon's well-known socket (Unix) is probed with an +/// identified `PING`. +fn running_daemon_endpoint() -> daemon::control::HealthEndpoint { + use daemon::control::HealthEndpoint; + + let Ok(dir) = discovery::state_dir() else { + eprintln!("No daemon is currently running."); + std::process::exit(1); }; + if let Some(info) = discovery::discover_in(&dir) + && let Some(endpoint) = HealthEndpoint::from_record(&info.health_endpoint, &dir) + { + return endpoint; + } + + match daemon::state_perms::verify_state_dir_trusted(&dir) { + Ok(()) => {} + // No state directory at all: nothing has ever run here. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + eprintln!("No daemon is currently running."); + std::process::exit(1); + } + Err(e) => { + eprintln!("The daemon state directory cannot be trusted: {e}"); + std::process::exit(1); + } + } + match daemon::lock::DaemonLock::try_acquire(&dir) { + Ok(Some(_free)) => { + eprintln!("No daemon is currently running."); + std::process::exit(1); + } + Ok(None) => {} + Err(e) => { + eprintln!("Cannot check the daemon lock in {}: {e}", dir.display()); + std::process::exit(1); + } + } + // The lock is held, so a daemon is running or starting. A Windows pipe + // name is random per daemon and only the record carries it, so only Unix + // can probe the fixed socket path. + #[cfg(unix)] + if let Ok(endpoint) = HealthEndpoint::for_new_daemon(&dir) + && health::ping_identified( + &endpoint, + std::time::Duration::from_millis(500), + std::time::Duration::from_millis(500), + ) + .is_some() + { + return endpoint; + } + eprintln!( + "The daemon lock is held but no usable record was found, and nothing answered on the \ + health channel in {}. A daemon may still be starting; try again shortly.", + dir.display() + ); + std::process::exit(1); +} - match health::send_command(target_port, "STOP") { +fn daemon_stop() { + let endpoint = running_daemon_endpoint(); + match health::send_command(&endpoint, "STOP") { Ok(response) => { println!("Daemon responded: {}", response.trim()); } Err(e) => { - eprintln!("No daemon running on port {target_port} (or cannot connect): {e}"); + eprintln!("No daemon running at {endpoint} (or cannot connect): {e}"); std::process::exit(1); } } } -fn daemon_status(port: Option) { - let info = if let Some(port) = port { - match health::send_command(port, "STATUS") { - Ok(response) => match serde_json::from_str::(response.trim()) { - Ok(info) => info, - Err(e) => { - eprintln!("Daemon on port {port} returned invalid status: {e}"); - std::process::exit(1); - } - }, +fn daemon_status() { + let endpoint = running_daemon_endpoint(); + // Ask the daemon itself, so the report is its own view of its endpoints. + let info = match health::send_command(&endpoint, "STATUS") { + Ok(response) => match serde_json::from_str::(response.trim()) { + Ok(info) => info, Err(e) => { - eprintln!("No daemon running on port {port} (or cannot connect): {e}"); + eprintln!("Daemon at {endpoint} returned invalid status: {e}"); std::process::exit(1); } + }, + Err(e) => { + eprintln!("No daemon running at {endpoint} (or cannot connect): {e}"); + std::process::exit(1); } - } else if let Some(info) = discovery::find_running_daemon() { - info - } else { - eprintln!("No daemon is currently running."); - std::process::exit(1); }; println!("Daemon is running:"); println!(" PID: {}", info.pid); println!(" Hyperd endpoint: {}", info.hyperd_endpoint); - println!(" Health port: {}", info.health_port); + println!(" Health endpoint: {}", info.health_endpoint); println!(" Started: {}", info.started_at); println!(" Version: {}", info.version); } diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index e02dc71b..92ae0757 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -1645,7 +1645,7 @@ impl HyperMcpServer { where F: FnOnce(&mut Engine) -> Result, { - let (result, daemon_health_port, connection_lost) = { + let (result, daemon_health_endpoint, connection_lost) = { let mut guard = self.ensure_engine()?; // `&mut` because transactional paths need // `Engine::execute_in_transaction`, whose RAII guard borrows the @@ -1653,7 +1653,7 @@ impl HyperMcpServer { // engine already lives behind this exclusive `Mutex`, so no // caller was ever sharing it concurrently. let engine = guard.as_mut().expect("ensure_engine guarantees Some"); - let daemon_health_port = engine.daemon_health_port(); + let daemon_health_endpoint = engine.daemon_health_endpoint_handle().cloned(); // Bootstrap the catalog exactly once per engine. Intentionally // runs *inside* `with_engine` (not `ensure_engine`) so the // catalog SQL can see errors classified via the normal error @@ -1684,31 +1684,34 @@ impl HyperMcpServer { } } drop(guard); - (result, daemon_health_port, connection_lost) + (result, daemon_health_endpoint, connection_lost) }; - // Health-plane TCP I/O must not hold the engine mutex. The captured - // port is authoritative for the daemon this engine actually uses; + // Health-plane I/O must not hold the engine mutex. The captured + // endpoint is authoritative for the daemon this engine actually uses; // `None` means local fallback and therefore no daemon report. if connection_lost { - if let Some(port) = daemon_health_port { - crate::daemon::health::report_hyperd_error_to_daemon(port); + if let Some(endpoint) = &daemon_health_endpoint { + crate::daemon::health::report_hyperd_error_to_daemon(endpoint); } } else { - self.maybe_send_heartbeat(daemon_health_port); + self.maybe_send_heartbeat(daemon_health_endpoint.as_ref()); } result } /// Best-effort heartbeat to keep the daemon alive while this client is active. /// Debounced: only sends if more than 60 seconds have elapsed since the last heartbeat, - /// avoiding a new TCP connection on every tool call. + /// avoiding a new connection on every tool call. /// - /// Accepts the daemon health port captured while the engine was locked so + /// Accepts the daemon health endpoint captured while the engine was locked so /// this method never needs to re-lock `self.engine` after guard release. - fn maybe_send_heartbeat(&self, daemon_health_port: Option) { + fn maybe_send_heartbeat( + &self, + daemon_health_endpoint: Option<&crate::daemon::control::HealthEndpoint>, + ) { const HEARTBEAT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60); - let Some(port) = daemon_health_port else { + let Some(endpoint) = daemon_health_endpoint else { return; }; @@ -1723,7 +1726,7 @@ impl HyperMcpServer { true }); if should_send { - let _ = crate::daemon::health::send_command(port, "HEARTBEAT"); + let _ = crate::daemon::health::send_command(endpoint, "HEARTBEAT"); } } @@ -1732,7 +1735,7 @@ impl HyperMcpServer { /// the mutex — so diagnostics never hang behind a stalled data-plane op. /// /// Includes everything answerable from `self` fields + a fast daemon-health - /// check (read `daemon.json` + one PING to the known health port, max + /// check (read `daemon.json` + one PING to the known health endpoint, max /// ~300ms). Omits `table_count`, `total_rows`, `disk_usage_bytes`, /// `ephemeral_path`, and `logs` (which require the engine / SQL against /// hyperd). @@ -1740,10 +1743,9 @@ impl HyperMcpServer { /// Clients should check `engine_busy: true` and retry `status` later if /// they need the full stats, or wait for the in-progress operation to finish. fn status_degraded(&self) -> Result { - // Use discover() — NOT find_running_daemon(). discover() reads the - // daemon.json file + one PING to the known health port (~1ms if alive, - // 300ms timeout if dead). find_running_daemon() adds a 16-port scan on - // failure (up to 4.8s), which would defeat the "instant response" goal. + // discover() reads the daemon.json file + one PING to the recorded + // health endpoint (~1ms if alive, 300ms timeout if dead), which keeps + // the "instant response" goal. let (hyperd_running, engine_block) = if let Some(info) = crate::daemon::discovery::discover() { ( @@ -1751,7 +1753,7 @@ impl HyperMcpServer { json!({ "mode": "daemon", "hyperd_endpoint": info.hyperd_endpoint, - "daemon_health_port": info.health_port, + "daemon_health_endpoint": info.health_endpoint, "connection": crate::engine::describe_endpoint(&info.hyperd_endpoint), }), ) @@ -1762,7 +1764,7 @@ impl HyperMcpServer { json!({ "mode": "local", "hyperd_endpoint": null, - "daemon_health_port": null, + "daemon_health_endpoint": null, "connection": null, }), ) @@ -1772,7 +1774,7 @@ impl HyperMcpServer { json!({ "mode": "daemon", "hyperd_endpoint": null, - "daemon_health_port": null, + "daemon_health_endpoint": null, "connection": null, }), ) @@ -5858,36 +5860,42 @@ mod kv_value_path_size_tests { #[cfg(test)] mod heartbeat_debounce_tests { use std::io::{BufRead, BufReader, Write}; - use std::net::{TcpListener, TcpStream}; use std::sync::mpsc::{self, Receiver, Sender}; use std::sync::{Arc, Barrier}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; + use crate::daemon::control::{ControlListener, ControlStream, HealthEndpoint}; + use super::HyperMcpServer; + /// A per-test state directory and the (not yet bound) endpoint inside it. + /// Kept short so the socket path stays under the platform's `sun_path` + /// limit. + fn test_endpoint() -> (tempfile::TempDir, HealthEndpoint) { + let dir = tempfile::tempdir().expect("create heartbeat test state dir"); + let endpoint = HealthEndpoint::for_new_daemon(dir.path()).expect("heartbeat endpoint"); + (dir, endpoint) + } + struct HeldHeartbeatPeer { - port: u16, + _dir: tempfile::TempDir, + endpoint: HealthEndpoint, release: Option>, handle: Option, String>>>, } impl HeldHeartbeatPeer { fn spawn() -> Self { + let (dir, endpoint) = test_endpoint(); let listener = - TcpListener::bind(("127.0.0.1", 0)).expect("bind controlled heartbeat listener"); - listener - .set_nonblocking(true) - .expect("make controlled heartbeat listener nonblocking"); - let port = listener - .local_addr() - .expect("controlled heartbeat listener address") - .port(); + ControlListener::bind(&endpoint).expect("bind controlled heartbeat listener"); let (release_tx, release_rx) = mpsc::channel(); let handle = - std::thread::spawn(move || hold_heartbeat_responses(&listener, &release_rx)); + std::thread::spawn(move || hold_heartbeat_responses(listener, &release_rx)); Self { - port, + _dir: dir, + endpoint, release: Some(release_tx), handle: Some(handle), } @@ -5917,19 +5925,17 @@ mod heartbeat_debounce_tests { } fn hold_heartbeat_responses( - listener: &TcpListener, + mut listener: ControlListener, release: &Receiver<()>, ) -> Result, String> { let hard_deadline = Instant::now() + Duration::from_secs(5); let mut streams = Vec::new(); loop { - loop { - match listener.accept() { - Ok((stream, _)) => streams.push(stream), - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, - Err(error) => { - return Err(format!("controlled heartbeat accept failed: {error}")); - } + match listener.accept_timeout(Duration::from_millis(2)) { + Ok(Some(stream)) => streams.push(stream), + Ok(None) => {} + Err(error) => { + return Err(format!("controlled heartbeat accept failed: {error}")); } } @@ -5940,25 +5946,22 @@ mod heartbeat_debounce_tests { if Instant::now() >= hard_deadline { return Err("controlled heartbeat listener was never released".to_string()); } - std::thread::sleep(Duration::from_millis(2)); } // Accept any connections already queued when the release signal won - // the race with the nonblocking accept above. + // the race with the accept above. let drain_deadline = Instant::now() + Duration::from_millis(50); while Instant::now() < drain_deadline { - match listener.accept() { - Ok((stream, _)) => streams.push(stream), - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(1)); - } + match listener.accept_timeout(Duration::from_millis(5)) { + Ok(Some(stream)) => streams.push(stream), + Ok(None) => {} Err(error) => return Err(format!("heartbeat queue drain failed: {error}")), } } let mut commands = Vec::with_capacity(streams.len()); for mut stream in streams { - commands.push(read_heartbeat_command(&stream)?); + commands.push(read_heartbeat_command(&mut stream)?); stream .write_all(b"OK\n") .map_err(|error| format!("acknowledge held heartbeat: {error}"))?; @@ -5966,43 +5969,32 @@ mod heartbeat_debounce_tests { Ok(commands) } - fn read_heartbeat_command(stream: &TcpStream) -> Result { + fn read_heartbeat_command(stream: &mut ControlStream) -> Result { stream - .set_read_timeout(Some(Duration::from_secs(1))) - .map_err(|error| format!("set heartbeat read timeout: {error}"))?; - stream - .set_write_timeout(Some(Duration::from_secs(1))) - .map_err(|error| format!("set heartbeat write timeout: {error}"))?; - let reader = stream - .try_clone() - .map_err(|error| format!("clone heartbeat stream: {error}"))?; + .set_io_timeout(Duration::from_secs(1)) + .map_err(|error| format!("set heartbeat I/O timeout: {error}"))?; let mut command = String::new(); - BufReader::new(reader) + BufReader::new(&mut *stream) .read_line(&mut command) .map_err(|error| format!("read heartbeat command: {error}"))?; Ok(command) } fn collect_immediate_heartbeats( - listener: &TcpListener, + mut listener: ControlListener, window: Duration, ) -> Result, String> { - listener - .set_nonblocking(true) - .map_err(|error| format!("make follow-up listener nonblocking: {error}"))?; let deadline = Instant::now() + window; let mut commands = Vec::new(); while Instant::now() < deadline { - match listener.accept() { - Ok((mut stream, _)) => { - commands.push(read_heartbeat_command(&stream)?); + match listener.accept_timeout(Duration::from_millis(2)) { + Ok(Some(mut stream)) => { + commands.push(read_heartbeat_command(&mut stream)?); stream .write_all(b"OK\n") .map_err(|error| format!("acknowledge follow-up heartbeat: {error}"))?; } - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(2)); - } + Ok(None) => {} Err(error) => return Err(format!("follow-up heartbeat accept failed: {error}")), } } @@ -6030,10 +6022,10 @@ mod heartbeat_debounce_tests { let server = Arc::clone(&server); let barrier = Arc::clone(&barrier); let completed = completed_tx.clone(); - let port = peer.port; + let endpoint = peer.endpoint.clone(); callers.push(std::thread::spawn(move || { barrier.wait(); - server.maybe_send_heartbeat(Some(port)); + server.maybe_send_heartbeat(Some(&endpoint)); let _ = completed.send(()); })); } @@ -6064,27 +6056,22 @@ mod heartbeat_debounce_tests { // Preserve the existing best-effort rule: even a refused connection // consumes the debounce interval, so an immediate retry does not // create a heartbeat storm while the daemon is unavailable. - let reservation = - TcpListener::bind(("127.0.0.1", 0)).expect("reserve a closed heartbeat port"); - let failed_port = reservation - .local_addr() - .expect("closed heartbeat port address") - .port(); - drop(reservation); + // An endpoint nothing is bound to refuses the connection. + let (_failed_dir, failed_endpoint) = test_endpoint(); *server .last_heartbeat .lock() .expect("heartbeat timestamp mutex") = Instant::now() .checked_sub(HEARTBEAT_INTERVAL) .expect("60-second heartbeat interval fits before current instant"); - server.maybe_send_heartbeat(Some(failed_port)); + server.maybe_send_heartbeat(Some(&failed_endpoint)); - let follow_up_listener = TcpListener::bind(("127.0.0.1", failed_port)) - .expect("bind follow-up listener on refused heartbeat port"); + let follow_up_listener = ControlListener::bind(&failed_endpoint) + .expect("bind follow-up listener on refused heartbeat endpoint"); let follow_up = std::thread::spawn(move || { - collect_immediate_heartbeats(&follow_up_listener, Duration::from_millis(300)) + collect_immediate_heartbeats(follow_up_listener, Duration::from_millis(300)) }); - server.maybe_send_heartbeat(Some(failed_port)); + server.maybe_send_heartbeat(Some(&failed_endpoint)); let follow_up_commands = follow_up.join(); let mut failures = Vec::new(); diff --git a/hyperdb-mcp/tests/daemon_tests.rs b/hyperdb-mcp/tests/daemon_tests.rs index 91c2c9d3..05d7c9e6 100644 --- a/hyperdb-mcp/tests/daemon_tests.rs +++ b/hyperdb-mcp/tests/daemon_tests.rs @@ -2,20 +2,31 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT //! Tests for the single-instance daemon: discovery file, health protocol, -//! idle timeout, and full lifecycle integration with a real `hyperd`. +//! idle timeout, the daemon lock, and full lifecycle integration with a real +//! `hyperd`. //! -//! Many tests mutate process-global environment variables (`HYPERDB_STATE_DIR`, -//! `HYPERDB_DAEMON_PORT`) to isolate their state directories. Because env vars +//! A daemon is identified by its state directory: the lock is `daemon.lock`, +//! the health channel is the per-user socket `daemon.sock`, and `daemon.json` +//! points at both. Every fake peer here therefore lives in its own short +//! per-test state directory (a Unix socket path must fit in `sun_path`, 104 +//! bytes on macOS). +//! +//! Many tests mutate the process-global environment variable +//! `HYPERDB_STATE_DIR` to isolate their state directories. Because env vars //! are process-global, these tests MUST run sequentially. We enforce this via a //! shared mutex — every test that touches env vars acquires `ENV_LOCK` first. -use std::net::TcpListener; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; use std::process::{Output, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use hyperdb_mcp::daemon::discovery::{self, DaemonInfo, PortScan}; +use hyperdb_mcp::daemon::control::{self, ControlListener, ControlStream, HealthEndpoint}; +use hyperdb_mcp::daemon::discovery::{self, DaemonInfo}; use hyperdb_mcp::daemon::health::{self, DaemonState, HealthListener}; +use hyperdb_mcp::daemon::lock::DaemonLock; use tempfile::TempDir; /// Process-wide lock for tests that mutate environment variables. @@ -23,7 +34,7 @@ use tempfile::TempDir; /// We recover from poison to prevent one test's panic from cascading. static ENV_LOCK: Mutex<()> = Mutex::new(()); const ENGINE_REPORT_CHILD_ENV: &str = "HYPERDB_MCP_ENGINE_REPORT_CHILD"; -const ENGINE_REPORT_TEST_NAME: &str = "report_hyperd_error_targets_discovered_health_port"; +const ENGINE_REPORT_TEST_NAME: &str = "report_hyperd_error_targets_discovered_health_endpoint"; /// Seconds a crash-recovery test waits for the daemon to bring a fresh `hyperd` /// back to a reachable state after the running engine is killed. @@ -60,9 +71,16 @@ fn acquire_env_lock() -> std::sync::MutexGuard<'static, ()> { // ─── Unit tests: DaemonState (no env vars, safe to run in parallel) ─────────── +/// A `DaemonState` for a daemon whose endpoint lives in a fresh state +/// directory. The directory must outlive the state. +fn new_daemon_state() -> (TempDir, DaemonState) { + let (dir, endpoint) = test_endpoint(); + (dir, DaemonState::new(endpoint)) +} + #[test] fn daemon_state_touch_resets_idle_duration() { - let state = DaemonState::new(); + let (_dir, state) = new_daemon_state(); std::thread::sleep(Duration::from_millis(50)); assert!(state.idle_duration() >= Duration::from_millis(50)); @@ -72,7 +90,7 @@ fn daemon_state_touch_resets_idle_duration() { #[test] fn daemon_state_shutdown_flag() { - let state = DaemonState::new(); + let (_dir, state) = new_daemon_state(); assert!(!state.should_shutdown()); state.request_shutdown(); @@ -80,25 +98,14 @@ fn daemon_state_shutdown_flag() { } #[test] -fn daemon_state_default_is_equivalent_to_new() { - let default_state = DaemonState::default(); - assert!(!default_state.should_shutdown()); - assert!(default_state.idle_duration() < Duration::from_millis(100)); -} - -#[test] -fn daemon_state_default_initializes_restart_flag_false() { - // Guard against future regressions where Default and new() diverge — - // both must initialize restart_requested to false. - let default_state = DaemonState::default(); - let new_state = DaemonState::new(); - assert!(!default_state.consume_restart_request()); - assert!(!new_state.consume_restart_request()); +fn daemon_state_new_initializes_restart_flag_false() { + let (_dir, state) = new_daemon_state(); + assert!(!state.consume_restart_request()); } #[test] fn daemon_state_restart_request_consume_round_trip() { - let state = DaemonState::new(); + let (_dir, state) = new_daemon_state(); assert!(!state.consume_restart_request(), "initially clear"); state.request_restart(); @@ -196,7 +203,7 @@ fn daemon_config_from_args_none_when_unset() { let _lock = acquire_env_lock(); let _guard = EnvGuard::remove("HYPERDB_DAEMON_IDLE_TIMEOUT"); - let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(0, None); + let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(None); assert!( config.idle_timeout.is_none(), "idle_timeout should be None when neither flag nor env is set" @@ -208,7 +215,7 @@ fn daemon_config_from_args_some_when_flag() { let _lock = acquire_env_lock(); let _guard = EnvGuard::remove("HYPERDB_DAEMON_IDLE_TIMEOUT"); - let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(0, Some(120)); + let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(Some(120)); assert_eq!( config.idle_timeout, Some(Duration::from_secs(120)), @@ -221,7 +228,7 @@ fn daemon_config_from_args_some_when_env() { let _lock = acquire_env_lock(); let _guard = EnvGuard::set("HYPERDB_DAEMON_IDLE_TIMEOUT", "90"); - let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(0, None); + let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(None); assert_eq!( config.idle_timeout, Some(Duration::from_secs(90)), @@ -234,7 +241,7 @@ fn daemon_config_from_args_flag_takes_precedence() { let _lock = acquire_env_lock(); let _guard = EnvGuard::set("HYPERDB_DAEMON_IDLE_TIMEOUT", "90"); - let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(0, Some(120)); + let config = hyperdb_mcp::daemon::run::DaemonConfig::from_args(Some(120)); assert_eq!( config.idle_timeout, Some(Duration::from_secs(120)), @@ -245,57 +252,93 @@ fn daemon_config_from_args_flag_takes_precedence() { // ─── Unit tests: Health protocol (no env vars, safe to run in parallel) ─────── #[test] -fn health_listener_bind_succeeds_on_free_port() { - let listener = HealthListener::bind(0).unwrap(); - assert_ne!(listener.port, 0); +fn health_listener_bind_succeeds_on_a_free_endpoint() { + let (_dir, endpoint) = test_endpoint(); + let listener = HealthListener::bind(&endpoint).unwrap(); + assert_eq!(listener.endpoint(), &endpoint); } #[test] -fn health_listener_second_bind_same_port_fails() { - let listener = HealthListener::bind(0).unwrap(); - let port = listener.port; +fn health_listener_second_bind_on_a_live_endpoint_fails() { + let (_dir, endpoint) = test_endpoint(); + let _first = HealthListener::bind(&endpoint).unwrap(); - let result = HealthListener::bind(port); - assert!(result.is_err()); - assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::AddrInUse); + let second = HealthListener::bind(&endpoint); + assert!( + second.is_err(), + "a live listener must not be displaced by a second bind" + ); +} + +/// The lock, not the socket, is what keeps a state directory to one daemon: +/// it is exclusive per state directory (also within one process, which the +/// in-process test daemons rely on), independent across directories, and +/// released on drop. +#[test] +fn daemon_lock_is_exclusive_per_state_dir_and_released_on_drop() { + let first_dir = TempDir::new().unwrap(); + let second_dir = TempDir::new().unwrap(); + + let held = DaemonLock::try_acquire(first_dir.path()) + .unwrap() + .expect("a free state directory yields the lock"); + assert!( + DaemonLock::try_acquire(first_dir.path()).unwrap().is_none(), + "a second holder in the same state directory must be refused" + ); + assert!( + DaemonLock::try_acquire(second_dir.path()) + .unwrap() + .is_some(), + "another state directory has its own lock" + ); + + drop(held); + // A descriptor can linger briefly in a sibling test's freshly forked child + // (it is close-on-exec, so only until that child's exec), hence the short + // bounded wait rather than a single probe. + let deadline = Instant::now() + Duration::from_secs(5); + let released = loop { + if DaemonLock::try_acquire(first_dir.path()).unwrap().is_some() { + break true; + } + if Instant::now() >= deadline { + break false; + } + std::thread::sleep(Duration::from_millis(20)); + }; + assert!( + released, + "dropping the lock must free it for the next holder" + ); } #[test] fn health_listener_waits_for_command_after_accept() { - use std::io::{BufRead, BufReader, Write}; - use std::net::{Shutdown, SocketAddr, TcpStream}; use std::sync::mpsc; const CLIENT_IO_TIMEOUT: Duration = Duration::from_secs(1); const OVERALL_WATCHDOG: Duration = Duration::from_secs(3); - let listener = HealthListener::bind(0).expect("bind OS-assigned health-listener port"); - let port = listener.port; - let info = DaemonInfo { - pid: 12345, - hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: port, - started_at: "2026-05-20T10:30:00Z".to_string(), - version: "0.1.3".to_string(), - }; - let mut listener = OwnedHealthListener::start(listener, info); + let (_dir, mut health) = start_health_listener(); + let endpoint = health.endpoint.clone(); let (outcome_tx, outcome_rx) = mpsc::sync_channel(1); let started = Instant::now(); let client = std::thread::spawn(move || { - struct ShutdownOnDrop(TcpStream); - - impl ShutdownOnDrop { - fn connect(port: u16, timeout: Duration, label: &str) -> Result { - let address = SocketAddr::from(([127, 0, 0, 1], port)); - let stream = TcpStream::connect_timeout(&address, timeout) + struct IdleClient(ControlStream); + + impl IdleClient { + fn connect( + endpoint: &HealthEndpoint, + timeout: Duration, + label: &str, + ) -> Result { + let mut stream = control::connect(endpoint, timeout) .map_err(|error| format!("connect {label} raw health client: {error}"))?; stream - .set_read_timeout(Some(timeout)) - .map_err(|error| format!("set {label} client read timeout: {error}"))?; - stream - .set_write_timeout(Some(timeout)) - .map_err(|error| format!("set {label} client write timeout: {error}"))?; + .set_io_timeout(timeout) + .map_err(|error| format!("set {label} client I/O timeout: {error}"))?; Ok(Self(stream)) } @@ -304,12 +347,11 @@ fn health_listener_waits_for_command_after_accept() { format!("{label} PING write failed ({:?}): {error}", error.kind()) })?; - let mut response = String::new(); - match BufReader::new(&self.0).read_line(&mut response) { - Ok(0) => Err(format!( + match read_line_from(&mut self.0) { + Ok(response) if response.is_empty() => Err(format!( "health listener returned early EOF for {label} PING" )), - Ok(_) => Ok(response), + Ok(response) => Ok(response), Err(error) => Err(format!( "{label} PING read failed ({:?}): {error}", error.kind() @@ -318,21 +360,15 @@ fn health_listener_waits_for_command_after_accept() { } } - impl Drop for ShutdownOnDrop { - fn drop(&mut self) { - let _ = self.0.shutdown(Shutdown::Both); - } - } - let outcome = (|| -> Result { - let expected_pong = format!("PONG hyperdb-mcp {}\n", hyperdb_mcp::version::MCP_VERSION); - let mut idle_client = ShutdownOnDrop::connect(port, CLIENT_IO_TIMEOUT, "first idle")?; + let expected_pong = pong_line(); + let mut idle_client = IdleClient::connect(&endpoint, CLIENT_IO_TIMEOUT, "first idle")?; // A PONG from a later connection proves the listener has accepted // and serviced connections while the first one remains byte-empty. { let mut progress_probe = - ShutdownOnDrop::connect(port, CLIENT_IO_TIMEOUT, "progress probe")?; + IdleClient::connect(&endpoint, CLIENT_IO_TIMEOUT, "progress probe")?; let probe_response = progress_probe.ping("progress-probe")?; if probe_response != expected_pong { return Err(format!( @@ -341,9 +377,9 @@ fn health_listener_waits_for_command_after_accept() { } } - // An accepted socket must not inherit the listener's + // An accepted connection must not inherit the listener's // non-blocking mode: a non-blocking read would fail with - // WouldBlock at once. Keep the first socket idle for 350 ms, + // WouldBlock at once. Keep the first connection idle for 350 ms, // well inside the handler's 5 s read timeout, before sending // its first command. std::thread::sleep(Duration::from_millis(350)); @@ -353,7 +389,7 @@ fn health_listener_waits_for_command_after_accept() { }); let outcome = outcome_rx.recv_timeout(OVERALL_WATCHDOG); - listener.stop_and_join(); + health.stop_and_join(); let client_join = client.join(); assert!( @@ -365,10 +401,7 @@ fn health_listener_waits_for_command_after_accept() { }); assert_eq!( response, - Ok(format!( - "PONG hyperdb-mcp {}\n", - hyperdb_mcp::version::MCP_VERSION - )), + Ok(pong_line()), "HealthListener must keep an accepted connection open until its first command; elapsed={:?}", started.elapsed() ); @@ -376,45 +409,45 @@ fn health_listener_waits_for_command_after_accept() { #[test] fn health_protocol_ping_pong() { - let (port, _handle, _state) = start_health_listener(); + let (_dir, health) = start_health_listener(); - let response = health::send_command(port, "PING").unwrap(); + let response = health::send_command(&health.endpoint, "PING").unwrap(); assert!(response.trim().starts_with("PONG hyperdb-mcp ")); } #[test] fn health_protocol_heartbeat_resets_idle() { - let (port, _handle, state) = start_health_listener(); + let (_dir, health) = start_health_listener(); std::thread::sleep(Duration::from_millis(50)); - assert!(state.idle_duration() >= Duration::from_millis(50)); + assert!(health.state.idle_duration() >= Duration::from_millis(50)); - let response = health::send_command(port, "HEARTBEAT").unwrap(); + let response = health::send_command(&health.endpoint, "HEARTBEAT").unwrap(); assert_eq!(response.trim(), "OK"); - assert!(state.idle_duration() < Duration::from_millis(30)); + assert!(health.state.idle_duration() < Duration::from_millis(30)); } #[test] fn health_protocol_stop_triggers_shutdown() { - let (port, handle, state) = start_health_listener(); + let (_dir, mut health) = start_health_listener(); - assert!(!state.should_shutdown()); + assert!(!health.state.should_shutdown()); - let response = health::send_command(port, "STOP").unwrap(); + let response = health::send_command(&health.endpoint, "STOP").unwrap(); assert_eq!(response.trim(), "STOPPING"); - assert!(state.should_shutdown()); + assert!(health.state.should_shutdown()); // Health listener should exit its loop - handle.join().unwrap(); + health.join(); } #[test] fn health_protocol_status_returns_json() { - let (port, _handle, _state) = start_health_listener(); + let (_dir, health) = start_health_listener(); - let response = health::send_command(port, "STATUS").unwrap(); + let response = health::send_command(&health.endpoint, "STATUS").unwrap(); let parsed: serde_json::Value = serde_json::from_str(response.trim()).unwrap(); assert_eq!(parsed["pid"], 12345); assert_eq!(parsed["hyperd_endpoint"], "127.0.0.1:54321"); @@ -422,370 +455,151 @@ fn health_protocol_status_returns_json() { #[test] fn health_protocol_unknown_command_returns_error() { - let (port, _handle, _state) = start_health_listener(); + let (_dir, health) = start_health_listener(); - let response = health::send_command(port, "INVALID").unwrap(); + let response = health::send_command(&health.endpoint, "INVALID").unwrap(); assert!(response.contains("ERR")); } #[test] fn health_protocol_report_hyperd_error_sets_flag() { - let (port, _handle, state) = start_health_listener(); + let (_dir, health) = start_health_listener(); - assert!(!state.consume_restart_request(), "flag starts clear"); + assert!(!health.state.consume_restart_request(), "flag starts clear"); - let response = health::send_command(port, "REPORT_HYPERD_ERROR").unwrap(); + let response = health::send_command(&health.endpoint, "REPORT_HYPERD_ERROR").unwrap(); assert_eq!(response.trim(), "OK"); // The handler ran on a different thread. The flag is set before the OK // reply is written, so it is observable as soon as send_command returns. assert!( - state.consume_restart_request(), + health.state.consume_restart_request(), "REPORT_HYPERD_ERROR must set the restart-requested flag" ); } #[test] fn health_protocol_multi_command_session() { - let (port, _handle, _state) = start_health_listener(); + let (_dir, health) = start_health_listener(); - let response1 = health::send_command(port, "PING").unwrap(); + let response1 = health::send_command(&health.endpoint, "PING").unwrap(); assert!(response1.trim().starts_with("PONG hyperdb-mcp ")); - let response2 = health::send_command(port, "STATUS").unwrap(); + let response2 = health::send_command(&health.endpoint, "STATUS").unwrap(); let parsed: serde_json::Value = serde_json::from_str(response2.trim()).unwrap(); - assert_eq!(parsed["health_port"], port); + assert_eq!(parsed["health_endpoint"], health.endpoint.as_str()); - let response3 = health::send_command(port, "HEARTBEAT").unwrap(); + let response3 = health::send_command(&health.endpoint, "HEARTBEAT").unwrap(); assert_eq!(response3.trim(), "OK"); } #[test] fn health_protocol_ping_identity_accept() { - let (port, _handle, _state) = start_health_listener(); + let (_dir, health) = start_health_listener(); - let version = - health::ping_identified(port, Duration::from_millis(300), Duration::from_millis(300)) - .expect("should return Some for a valid hyperdb-mcp daemon"); + let version = health::ping_identified( + &health.endpoint, + Duration::from_millis(300), + Duration::from_millis(300), + ) + .expect("should return Some for a valid hyperdb-mcp daemon"); assert_eq!(version, hyperdb_mcp::version::MCP_VERSION); } #[test] fn health_protocol_ping_identity_reject_foreign() { - use std::io::Write; - - // Bind a raw listener that returns a bare "PONG\n" without the identifying token - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let port = listener.local_addr().unwrap().port(); - - std::thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - let _ = stream.write_all(b"PONG\n"); - } - }); - - // Give the thread a moment to start accepting - std::thread::sleep(Duration::from_millis(50)); - - let result = - health::ping_identified(port, Duration::from_millis(300), Duration::from_millis(300)); + // A peer that returns a bare "PONG\n" without the identifying token. + let dir = TempDir::new().unwrap(); + let mut peer = FakeHealthPeer::start_in(dir.path(), |_| b"PONG\n".to_vec()); + + let result = health::ping_identified( + &peer.endpoint, + Duration::from_millis(300), + Duration::from_millis(300), + ); assert_eq!(result, None, "should reject foreign PONG without token"); + assert_eq!( + peer.stop_and_join(), + ["PING"], + "the peer must actually have been asked, or the rejection proves nothing" + ); } #[test] fn health_protocol_ping_identity_reject_token_lookalike() { - use std::io::Write; - // A foreign service whose token *starts with* "hyperdb-mcp" must NOT pass. // Guards against a naive `starts_with("PONG hyperdb-mcp")` prefix check. - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let port = listener.local_addr().unwrap().port(); - - std::thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - let _ = stream.write_all(b"PONG hyperdb-mcpEVIL 9.9.9\n"); - } - }); - - std::thread::sleep(Duration::from_millis(50)); - - let result = - health::ping_identified(port, Duration::from_millis(300), Duration::from_millis(300)); + let dir = TempDir::new().unwrap(); + let mut peer = + FakeHealthPeer::start_in(dir.path(), |_| b"PONG hyperdb-mcpEVIL 9.9.9\n".to_vec()); + + let result = health::ping_identified( + &peer.endpoint, + Duration::from_millis(300), + Duration::from_millis(300), + ); assert_eq!( result, None, "must reject a token that only shares a prefix" ); + assert_eq!(peer.stop_and_join(), ["PING"]); } #[test] fn health_protocol_ping_identity_reject_refused() { - // Bind a listener to get a port, then drop it so the port is closed - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let port = listener.local_addr().unwrap().port(); + // Bind a listener, then drop it so nothing serves the endpoint any more. + let (_dir, endpoint) = test_endpoint(); + let listener = ControlListener::bind(&endpoint).unwrap(); drop(listener); - // Brief sleep to reduce the race (the OS may not have fully released the port) - std::thread::sleep(Duration::from_millis(10)); - - let result = - health::ping_identified(port, Duration::from_millis(300), Duration::from_millis(300)); - assert_eq!(result, None, "should return None for connection refused"); -} - -#[test] -fn resolve_port_scan_pins_when_env_set() { - let _lock = acquire_env_lock(); - let _guard = EnvGuard::set("HYPERDB_DAEMON_PORT", "9001"); - assert_eq!( - discovery::resolve_port_scan(), - PortScan { - base: 9001, - span: 1 - } + let result = health::ping_identified( + &endpoint, + Duration::from_millis(300), + Duration::from_millis(300), ); + assert_eq!(result, None, "should return None when nothing serves it"); } -#[test] -fn resolve_port_scan_scans_when_env_unset() { - let _lock = acquire_env_lock(); - let _guard = EnvGuard::remove("HYPERDB_DAEMON_PORT"); - let scan = discovery::resolve_port_scan(); - assert_eq!(scan.base, hyperdb_mcp::daemon::DEFAULT_DAEMON_BASE_PORT); - assert_eq!(scan.span, hyperdb_mcp::daemon::DAEMON_PORT_SCAN_SPAN); -} - -/// `"0".parse::()` succeeds, so port 0 slipped through the env chain's -/// validity filter and pinned `PortScan { base: 0, span: 1 }` — a -/// configuration nothing can satisfy. `bind` would take an OS-assigned -/// ephemeral port instead of port 0, while every client's scan kept probing -/// port 0 and read `ProbeResult::Refused` as "free", so each client that -/// missed the discovery fast path spawned another daemon-and-`hyperd` pair on -/// another unfindable port. It must fall back to the scanning default, exactly -/// as unparseable input already does. -#[test] -fn resolve_port_scan_rejects_zero_and_falls_back_to_the_default_scan() { - let _lock = acquire_env_lock(); - let _guard = EnvGuard::set("HYPERDB_DAEMON_PORT", "0"); - let scan = discovery::resolve_port_scan(); - assert_eq!( - scan, - PortScan { - base: hyperdb_mcp::daemon::DEFAULT_DAEMON_BASE_PORT, - span: hyperdb_mcp::daemon::DAEMON_PORT_SCAN_SPAN, - }, - "HYPERDB_DAEMON_PORT=0 must fall back to the default scan, not pin an \ - unsatisfiable base: 0 / span: 1" - ); -} - -/// The same rejection at the other entry point. Driven through the real -/// binary because `Cli` lives in `main.rs` and is not reachable from a test -/// crate; that also verifies the clap `value_parser` produces a usage error -/// (exit 2) rather than something the daemon has to defend against later. -/// -/// Two safety properties this test needs, both learned the hard way by -/// running it against the unfixed binary: +/// The removed `--port` flag. Driven through the real binary because `Cli` +/// lives in `main.rs` and is not reachable from a test crate. clap must reject +/// it as a usage error (exit 2) rather than a daemon quietly accepting it. /// -/// - **It must not hang.** Without the `value_parser`, `daemon --port 0` is -/// *accepted* and runs a foreground daemon forever, so a plain -/// `Command::output()` never returns. Each child is therefore waited on -/// against a deadline and killed if it outlives it — a reverted fix fails -/// with a message instead of wedging the suite. -/// - **It must not touch the real state directory.** That same accepted -/// daemon writes `daemon.json`, which overwrote the developer's live -/// discovery record. `HOME`/`USERPROFILE`/`HYPERDB_STATE_DIR` are pinned -/// into a temp dir so the blast radius of a regression stays inside the -/// test. -#[test] -fn daemon_cli_rejects_port_zero() { - /// Long enough that a loaded machine cannot mistake startup for a hang, - /// short enough that a regression reports promptly. A rejected port exits - /// in milliseconds; an accepted one never exits at all. - const CHILD_DEADLINE: Duration = Duration::from_secs(20); - - let sandbox = TempDir::new().expect("temp state dir for port-0 CLI test"); - let mut failures = Vec::new(); - - let run_bounded = |args: &[&str]| -> Result<(Option, String), String> { - let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_hyperdb-mcp")); - command.env_clear(); - preserve_child_runtime_environment(&mut command); - command - .args(args) - .current_dir(sandbox.path()) - .env("HOME", sandbox.path()) - .env("USERPROFILE", sandbox.path()) - .env("HYPERDB_STATE_DIR", sandbox.path()) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - if let Some(hyperd) = std::env::var_os("HYPERD_PATH") { - command.env("HYPERD_PATH", hyperd); - } - - let mut child = command.spawn().map_err(|error| error.to_string())?; - let deadline = Instant::now() + CHILD_DEADLINE; - loop { - match child.try_wait().map_err(|error| error.to_string())? { - Some(status) => { - let output = child - .wait_with_output() - .map_err(|error| error.to_string())?; - return Ok(( - status.code(), - String::from_utf8_lossy(&output.stderr).into(), - )); - } - None if Instant::now() >= deadline => { - let _ = child.kill(); - let _ = child.wait(); - return Err(format!( - "`hyperdb-mcp {}` was still running after {CHILD_DEADLINE:?}; \ - port 0 was accepted and started a foreground daemon instead of \ - being rejected", - args.join(" ") - )); - } - None => std::thread::sleep(Duration::from_millis(50)), - } - } - }; +/// Each child is waited on against a deadline and killed if it outlives it: +/// had `--port` still been accepted, `daemon --port 1234` would run a +/// foreground daemon forever and a plain `Command::output()` would never +/// return. +#[test] +fn daemon_cli_rejects_the_removed_port_flag() { + let sandbox = TempDir::new().expect("temp state dir for the port-flag CLI test"); for args in [ + ["daemon", "--port", "1234"].as_slice(), ["daemon", "--port", "0"].as_slice(), - ["daemon", "stop", "--port", "0"].as_slice(), - ["daemon", "status", "--port", "0"].as_slice(), + ["daemon", "stop", "--port", "1234"].as_slice(), + ["daemon", "status", "--port", "1234"].as_slice(), ] { - match run_bounded(args) { - Ok((code, stderr)) => { - if code != Some(2) { - failures.push(format!( - "{args:?}: expected clap's usage-error exit 2, got {code:?} (stderr {stderr:?})" - )); - } - if !stderr.contains("not in") { - failures.push(format!( - "{args:?}: stderr should name the rejected range, got {stderr:?}" - )); - } - } - Err(message) => failures.push(format!("{args:?}: {message}")), - } - } - - // A valid port must still parse. `--help` short-circuits before any bind, - // so this stays hermetic: no daemon is started and no port is taken. - match run_bounded(&["daemon", "--port", "7485", "--help"]) { - Ok((Some(0), _)) => {} - Ok((code, stderr)) => failures.push(format!( - "a valid --port must still be accepted; exit {code:?}, stderr {stderr:?}" - )), - Err(message) => failures.push(format!("valid --port: {message}")), + let output = run_cli_bounded(args, sandbox.path(), Duration::from_secs(20)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(2), + "{args:?}: expected clap's usage-error exit 2\nstderr:\n{stderr}" + ); + assert!( + stderr.contains("unexpected argument"), + "{args:?}: stderr should say the argument is unexpected, got {stderr:?}" + ); } assert!( !sandbox.path().join("daemon.json").exists(), "no invocation in this test may publish a discovery record; \ - port 0 was accepted and a daemon actually started" - ); - assert!( - failures.is_empty(), - "port-0 rejection failures:\n{}", - failures.join("\n") - ); -} - -#[test] -fn daemon_status_post_action_port_targets_explicit_listener() { - let _lock = acquire_env_lock(); - let temp_dir = TempDir::new().expect("create isolated daemon-status state root"); - let state_dir = temp_dir.path().join("state"); - std::fs::create_dir_all(&state_dir).expect("create isolated daemon-status state directory"); - let discovery_path = state_dir.join("daemon.json"); - let stale_discovery = br#"{ - "pid": 424242, - "hyperd_endpoint": "127.0.0.1:1", - "health_port": 0, - "started_at": "stale-discovery-sentinel", - "version": "0.0.0-stale" -}"#; - std::fs::write(&discovery_path, stale_discovery) - .expect("write stale discovery sentinel for explicit-port bypass proof"); - - let mut explicit_listener = CliStatusListener::start(); - let reserved_base = TcpListener::bind(("127.0.0.1", 0)) - .expect("reserve an isolated discovery base port different from the explicit listener"); - let base_port = reserved_base - .local_addr() - .expect("read reserved discovery base address") - .port(); - assert_ne!( - base_port, explicit_listener.port, - "isolated discovery base must differ from the explicit target" - ); - - let spellings = [ - vec![ - "daemon".to_string(), - "status".to_string(), - "--port".to_string(), - explicit_listener.port.to_string(), - ], - vec![ - "daemon".to_string(), - "--port".to_string(), - explicit_listener.port.to_string(), - "status".to_string(), - ], - ]; - let mut failures = Vec::new(); - - for args in &spellings { - let output = run_cli_child_with_watchdog(args, &state_dir, base_port); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - failures.push(format!( - "`hyperdb-mcp {}` exited {}\nstdout:\n{stdout}\nstderr:\n{stderr}", - args.join(" "), - output.status - )); - } - let expected_port_line = format!("Health port: {}", explicit_listener.port); - if !stdout.contains(&expected_port_line) { - failures.push(format!( - "`hyperdb-mcp {}` did not report the explicit listener {expected_port_line:?}\nstdout:\n{stdout}\nstderr:\n{stderr}", - args.join(" ") - )); - } - } - - let received_commands = explicit_listener.stop_and_join(); - if received_commands != ["STATUS", "STATUS"] { - failures.push(format!( - "explicit listener received {received_commands:?}, expected one STATUS from each literal spelling" - )); - } - match std::fs::read(&discovery_path) { - Ok(contents) if contents == stale_discovery => {} - Ok(contents) => failures.push(format!( - "explicit status mutated stale discovery sentinel: {:?}", - String::from_utf8_lossy(&contents) - )), - Err(error) => failures.push(format!( - "explicit status removed or made stale discovery unreadable: {error}" - )), - } - - assert!( - failures.is_empty(), - "explicit daemon status contract failures:\n{}", - failures.join("\n\n") + a rejected flag must not have started a daemon" ); } #[test] -fn report_hyperd_error_targets_discovered_health_port() { +fn report_hyperd_error_targets_discovered_health_endpoint() { let _lock = acquire_env_lock(); if std::env::var_os(ENGINE_REPORT_CHILD_ENV).is_some() { run_engine_report_child(); @@ -800,12 +614,6 @@ fn run_bounded_engine_report_child() { let process_temp_dir = temp_dir.path().join("tmp"); std::fs::create_dir_all(&state_dir).expect("create isolated engine-construction state"); std::fs::create_dir_all(&process_temp_dir).expect("create isolated process temp directory"); - let configured_base = - TcpListener::bind(("127.0.0.1", 0)).expect("reserve configured daemon base port"); - let configured_base_port = configured_base - .local_addr() - .expect("read configured daemon base port") - .port(); let mut command = std::process::Command::new( std::env::current_exe().expect("locate daemon integration-test binary"), @@ -819,7 +627,6 @@ fn run_bounded_engine_report_child() { .env("HOME", &state_dir) .env("USERPROFILE", &state_dir) .env("HYPERDB_STATE_DIR", &state_dir) - .env("HYPERDB_DAEMON_PORT", configured_base_port.to_string()) .env("TMPDIR", &process_temp_dir) .env("TEMP", &process_temp_dir) .env("TMP", &process_temp_dir) @@ -877,40 +684,33 @@ fn run_bounded_engine_report_child() { } fn run_engine_report_child() { - let configured_base_port = std::env::var("HYPERDB_DAEMON_PORT") - .expect("parent must configure a distinct daemon base port") - .parse::() - .expect("configured daemon base port must be a u16"); - - let health_listener = HealthListener::bind(0).expect("bind discovered health listener"); - let health_port = health_listener.port; - assert_ne!( - health_port, configured_base_port, - "discovered health port must differ from the configured base" - ); - let daemon_info = DaemonInfo { + let state_dir = discovery::state_dir().expect("child state directory"); + let mut health = RunningHealth::start_with_info(&state_dir, |endpoint| DaemonInfo { pid: 616_161, // Port zero is never a reachable TCP server endpoint. It makes the // public Engine constructor enter the daemon connection-error branch // without racing another process for a recently released port. hyperd_endpoint: "127.0.0.1:0".to_string(), - health_port, + health_endpoint: endpoint.as_str().to_string(), started_at: "2026-08-14T12:34:56Z".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), - }; - let mut health_listener = OwnedHealthListener::start(health_listener, daemon_info.clone()); - discovery::write_discovery_file(&daemon_info) - .expect("write isolated non-base daemon discovery record"); + }); + let daemon_info = health + .info + .lock() + .expect("daemon info mutex must not be poisoned") + .clone(); + discovery::write_discovery_file(&daemon_info).expect("write isolated daemon discovery record"); assert!( - !health_listener.state.consume_restart_request(), + !health.state.consume_restart_request(), "restart flag must begin clear" ); let error = hyperdb_mcp::engine::Engine::new(None) .expect_err("unreachable discovered Hyper endpoint must fail Engine construction"); - let restart_requested = health_listener.state.consume_restart_request(); - health_listener.stop_and_join(); + let restart_requested = health.state.consume_restart_request(); + health.stop_and_join(); assert!( error .message @@ -920,7 +720,7 @@ fn run_engine_report_child() { ); assert!( restart_requested, - "Engine::try_daemon_mode must report through the non-base health port from discovery" + "Engine::try_daemon_mode must report through the health endpoint from discovery" ); } @@ -941,8 +741,8 @@ fn daemon_idle_timeout_shuts_down_daemon() { // long this thread took to get from `new()` to `Instant::now()` rather // than on the timeout under test. let start = Instant::now(); - let state = Arc::new(DaemonState::new()); - + let (_dir, state) = new_daemon_state(); + let state = Arc::new(state); let monitor_state = Arc::clone(&state); let monitor = std::thread::spawn(move || { loop { @@ -984,8 +784,8 @@ fn daemon_idle_timeout_shuts_down_daemon() { /// readiness floor is removed: `run` then never returns and the join hangs /// forever, which is what the daemon would do in production. /// -/// Needs no state-directory isolation: the listener takes an ephemeral port -/// rather than the daemon port, and nothing here writes a discovery record or +/// Needs no state-directory isolation beyond its own tempdir: the listener +/// binds a socket there, and nothing here writes a discovery record or /// otherwise touches `$HOME`. #[test] fn idle_timeout_stops_a_real_health_listener() { @@ -995,16 +795,10 @@ fn idle_timeout_stops_a_real_health_listener() { // the unbounded wait a broken accept loop would produce. const SHUTDOWN_DEADLINE: Duration = Duration::from_secs(15); - let listener = HealthListener::bind(0).expect("bind real health listener"); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(DaemonInfo { - pid: std::process::id(), - hyperd_endpoint: "127.0.0.1:0".to_string(), - health_port: port, - started_at: "2026-09-06T00:00:00Z".to_string(), - version: "0.0.0-test".to_string(), - })); + let (_dir, endpoint) = test_endpoint(); + let listener = HealthListener::bind(&endpoint).expect("bind real health listener"); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(test_daemon_info(&endpoint))); let run_state = Arc::clone(&state); let (finished_tx, finished_rx) = std::sync::mpsc::channel(); @@ -1016,9 +810,9 @@ fn idle_timeout_stops_a_real_health_listener() { // PING rather than HEARTBEAT: it proves the listener is really serving // before the clock matters, without resetting the idle timer the test is // about. - let pong = health::send_command(port, "PING").expect("real health listener must answer PING"); + let pong = + health::send_command(&endpoint, "PING").expect("real health listener must answer PING"); assert!(pong.starts_with("PONG"), "unexpected PING reply: {pong:?}"); - // The same decision `run.rs`'s `idle_monitor` makes, at a tick that suits a // test: no activity for `IDLE_TIMEOUT` ⇒ request shutdown. let monitor_state = Arc::clone(&state); @@ -1060,7 +854,8 @@ fn idle_timeout_stops_a_real_health_listener() { #[test] fn daemon_heartbeat_prevents_idle_shutdown() { - let state = Arc::new(DaemonState::new()); + let (_dir, state) = new_daemon_state(); + let state = Arc::new(state); let idle_timeout = Duration::from_secs(1); let monitor_state = Arc::clone(&state); @@ -1109,11 +904,11 @@ fn daemon_heartbeat_prevents_idle_shutdown() { // ─── Unit tests: Discovery file (require ENV_LOCK) ──────────────────────────── #[test] -fn legacy_daemon_info_literal_is_source_compatible() { +fn daemon_info_literal_serializes_the_health_endpoint_field() { let info = DaemonInfo { pid: 12345, hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: 7484, + health_endpoint: "/tmp/state/daemon.sock".to_string(), started_at: "2026-05-20T10:30:00Z".to_string(), version: "0.1.3".to_string(), }; @@ -1123,7 +918,7 @@ fn legacy_daemon_info_literal_is_source_compatible() { serde_json::json!({ "pid": 12345, "hyperd_endpoint": "127.0.0.1:54321", - "health_port": 7484, + "health_endpoint": "/tmp/state/daemon.sock", "started_at": "2026-05-20T10:30:00Z", "version": "0.1.3" }) @@ -1139,7 +934,7 @@ fn discovery_file_write_and_read() { let info = DaemonInfo { pid: 12345, hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: 7484, + health_endpoint: "/tmp/state/daemon.sock".to_string(), started_at: "2026-05-20T10:30:00Z".to_string(), version: "0.1.3".to_string(), }; @@ -1153,7 +948,7 @@ fn discovery_file_write_and_read() { let read_back: DaemonInfo = serde_json::from_str(&contents).unwrap(); assert_eq!(read_back.pid, 12345); assert_eq!(read_back.hyperd_endpoint, "127.0.0.1:54321"); - assert_eq!(read_back.health_port, 7484); + assert_eq!(read_back.health_endpoint, "/tmp/state/daemon.sock"); assert_eq!(read_back.version, "0.1.3"); } @@ -1166,7 +961,7 @@ fn discovery_file_overwrite_replaces_content() { let info1 = DaemonInfo { pid: 100, hyperd_endpoint: "127.0.0.1:1111".to_string(), - health_port: 7484, + health_endpoint: "/tmp/state/daemon.sock".to_string(), started_at: "2026-01-01T00:00:00Z".to_string(), version: "0.1.0".to_string(), }; @@ -1175,7 +970,7 @@ fn discovery_file_overwrite_replaces_content() { let info2 = DaemonInfo { pid: 200, hyperd_endpoint: "127.0.0.1:2222".to_string(), - health_port: 7485, + health_endpoint: "/tmp/other/daemon.sock".to_string(), started_at: "2026-02-02T00:00:00Z".to_string(), version: "0.2.0".to_string(), }; @@ -1186,6 +981,7 @@ fn discovery_file_overwrite_replaces_content() { let read_back: DaemonInfo = serde_json::from_str(&contents).unwrap(); assert_eq!(read_back.pid, 200); assert_eq!(read_back.hyperd_endpoint, "127.0.0.1:2222"); + assert_eq!(read_back.health_endpoint, "/tmp/other/daemon.sock"); } #[test] @@ -1197,7 +993,7 @@ fn remove_discovery_file_deletes_it() { let info = DaemonInfo { pid: 1, hyperd_endpoint: "127.0.0.1:1".to_string(), - health_port: 7484, + health_endpoint: "/tmp/state/daemon.sock".to_string(), started_at: "2026-01-01T00:00:00Z".to_string(), version: "0.0.1".to_string(), }; @@ -1218,226 +1014,150 @@ fn discover_returns_none_when_no_file_exists() { assert!(discovery::discover().is_none()); } +/// A record whose endpoint nobody serves is not a daemon. Unlike the previous +/// `discover`, it is also *not deleted* here: removing a stale record is the +/// job of whoever holds the daemon lock, because only a held lock proves no +/// daemon is starting. #[test] -fn discover_returns_none_for_stale_file() { - let _lock = acquire_env_lock(); +fn discover_returns_none_for_stale_record_and_leaves_it() { let tmp = TempDir::new().unwrap(); - let _guard = EnvGuard::set("HYPERDB_STATE_DIR", tmp.path().to_str().unwrap()); + let endpoint = HealthEndpoint::for_new_daemon(tmp.path()).unwrap(); + let record = write_record(tmp.path(), &test_daemon_info(&endpoint)); - let info = DaemonInfo { - pid: 99999, - hyperd_endpoint: "127.0.0.1:1".to_string(), - health_port: 1, - started_at: "2026-01-01T00:00:00Z".to_string(), - version: "0.0.1".to_string(), - }; - discovery::write_discovery_file(&info).unwrap(); + assert!(discovery::discover_in(tmp.path()).is_none()); - assert!(discovery::discover().is_none()); - - let path = tmp.path().join("daemon.json"); - assert!(!path.exists()); -} - -#[test] -fn resolve_port_uses_env_var() { - let _lock = acquire_env_lock(); - let _guard = EnvGuard::set("HYPERDB_DAEMON_PORT", "9999"); - assert_eq!(discovery::resolve_port(), 9999); -} - -#[test] -fn resolve_port_uses_default_when_env_unset() { - let _lock = acquire_env_lock(); - let _guard = EnvGuard::remove("HYPERDB_DAEMON_PORT"); assert_eq!( - discovery::resolve_port(), - hyperdb_mcp::daemon::DEFAULT_DAEMON_BASE_PORT + std::fs::read(tmp.path().join("daemon.json")).unwrap(), + record, + "discovery must be side-effect free" ); } -// ─── Unit tests: Port scanning (require ENV_LOCK + sandbox OFF) ────────────── - #[test] -fn scan_finds_our_daemon_via_status() { - let _lock = acquire_env_lock(); - let (port, _handle, _state) = start_health_listener(); - - let scan = PortScan { - base: port, - span: 1, - }; - match discovery::scan_for_daemon(scan) { - discovery::ScanOutcome::Found(info) => { - assert_eq!(info.hyperd_endpoint, "127.0.0.1:54321"); - } - other => panic!("expected Found, got {other:?}"), - } -} - -#[test] -fn scan_skips_camped_returns_free() { +fn discover_finds_live_daemon() { let _lock = acquire_env_lock(); + let tmp = TempDir::new().unwrap(); + let _guard = EnvGuard::set("HYPERDB_STATE_DIR", tmp.path().to_str().unwrap()); - // Find a camped port `base` whose immediate successor `base + 1` is free, - // so the scan range is exactly the two adjacent ports {base, base+1}. - // - // TOCTOU mitigation: other tests' `start_health_listener` helpers leak - // identity-answering listeners on random high ports for the test-process - // lifetime. If one steals `base+1` between our probe-drop and the scan, - // the scan returns `Found` instead of `FreePort`. We retry with a fresh - // port pair up to 5 times to tolerate this race on busy CI runners. - for _attempt in 0..5 { - let (camped_listener, base) = loop { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let port = listener.local_addr().unwrap().port(); - if port < u16::MAX - && let Ok(probe) = TcpListener::bind(("127.0.0.1", port + 1)) - { - drop(probe); - break (listener, port); - } - drop(listener); - }; - let expected_free = base + 1; - - // Spawn a thread that keeps the camped listener alive and accepts - // connections, answering with non-protocol garbage so the identity - // check classifies it as `Camped`, not `OurDaemon`. - std::thread::spawn(move || { - loop { - if let Ok((mut stream, _)) = camped_listener.accept() { - use std::io::Write; - let _ = stream.write_all(b"NOPE\n"); - } - } - }); - - // Give the thread a moment to start accepting. - std::thread::sleep(Duration::from_millis(50)); + let health = start_health_listener_in(tmp.path()); - // Scan exactly {base (camped), base+1 (free)}. - let scan = PortScan { base, span: 2 }; + let info = test_daemon_info(&health.endpoint); + discovery::write_discovery_file(&info).unwrap(); - match discovery::scan_for_daemon(scan) { - discovery::ScanOutcome::FreePort(port) => { - assert_eq!( - port, expected_free, - "scan should skip the camped base port and return base+1" - ); - return; // Success - } - discovery::ScanOutcome::Found(_) => { - // Another test's leaked health listener stole base+1 — retry. - } - other => panic!("expected FreePort, got {other:?}"), - } - } - panic!("scan_skips_camped_returns_free: failed after 5 attempts (port stolen each time)"); + let discovered = discovery::discover().expect("should discover live daemon"); + assert_eq!(discovered.pid, 12345); + assert_eq!(discovered.health_endpoint, health.endpoint.as_str()); } +/// A record naming a live socket that is not this state directory's own +/// `daemon.sock` must never be connected to: the record is attacker-writable +/// input as far as the client is concerned. Unix only, where the endpoint is a +/// path derived from the state directory (a Windows pipe name is random). +#[cfg(unix)] #[test] -fn scan_all_refused_returns_freeport_base() { - // Pick a high port that's almost certainly free. - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let base = listener.local_addr().unwrap().port(); - drop(listener); // Release the port so it's free again. - - std::thread::sleep(Duration::from_millis(10)); +fn discover_never_connects_to_an_endpoint_outside_the_state_dir() { + let state = TempDir::new().unwrap(); + let elsewhere = TempDir::new().unwrap(); + let mut foreign = FakeHealthPeer::start_in(elsewhere.path(), |command| match command { + "PING" => pong_line().into_bytes(), + _ => b"ERR\n".to_vec(), + }); + write_record(state.path(), &test_daemon_info(&foreign.endpoint)); - // Span of 1: probe only the single known-free `base` port. A wider span - // would risk colliding with a leaked health listener from another parallel - // test (they bind random high ports and leak for the test-process lifetime), - // which would be reported as `Found` rather than `FreePort`. - let scan = PortScan { base, span: 1 }; - match discovery::scan_for_daemon(scan) { - discovery::ScanOutcome::FreePort(port) => { - assert_eq!(port, base, "should return the first free port (base)"); - } - other => panic!("expected FreePort(base), got {other:?}"), - } + assert!( + discovery::discover_in(state.path()).is_none(), + "a record pointing at a live foreign socket must not be believed" + ); + assert!( + foreign.stop_and_join().is_empty(), + "the client must not even connect to a socket outside its state directory" + ); } +/// A state directory another user could write to may hold a record steering +/// the client anywhere, so discovery refuses it outright, live daemon or not. +/// Group-writable is testable without root; "owned by someone else" takes +/// the same branch of `verify_state_dir_trusted` but needs a second user. +#[cfg(unix)] #[test] -fn probe_refused_when_closed() { - // Bind a listener to get a port, then drop it so the port is closed. - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let port = listener.local_addr().unwrap().port(); - drop(listener); +fn discover_refuses_a_group_writable_state_dir_even_with_a_live_daemon() { + let tmp = TempDir::new().unwrap(); + let mut peer = FakeHealthPeer::start_in(tmp.path(), |_| pong_line().into_bytes()); + write_record(tmp.path(), &test_daemon_info(&peer.endpoint)); + assert!( + discovery::discover_in(tmp.path()).is_some(), + "control: the same directory is discoverable while private" + ); + peer.stop_and_join(); - std::thread::sleep(Duration::from_millis(10)); + let mut peer = FakeHealthPeer::start_in(tmp.path(), |_| pong_line().into_bytes()); + make_group_writable(tmp.path()); - // Access the private probe_port via the public scan_for_daemon wrapper. - // We know that if the scan returns FreePort, then probe_port returned Refused. - let scan = PortScan { - base: port, - span: 1, - }; - match discovery::scan_for_daemon(scan) { - discovery::ScanOutcome::FreePort(p) => { - assert_eq!( - p, port, - "probe_port should have returned Refused for closed port" - ); - } - other => panic!("expected FreePort (Refused), got {other:?}"), - } + assert!(discovery::discover_in(tmp.path()).is_none()); + assert!( + peer.stop_and_join().is_empty(), + "an untrusted state directory must be rejected before any connection" + ); } +/// `ensure_daemon` on an untrusted state directory fails and spawns nothing +/// (and `Engine` then falls back to a private `hyperd`). Observable without a +/// spawner seam: a spawn would create `daemon.json` / `daemon.lock` / +/// `daemon.sock` here, or at least leave a process behind. +#[cfg(unix)] #[test] -fn discover_finds_live_daemon() { +fn ensure_daemon_refuses_an_untrusted_state_dir_without_spawning() { let _lock = acquire_env_lock(); let tmp = TempDir::new().unwrap(); + let mut peer = FakeHealthPeer::start_in(tmp.path(), |_| pong_line().into_bytes()); + write_record(tmp.path(), &test_daemon_info(&peer.endpoint)); + make_group_writable(tmp.path()); let _guard = EnvGuard::set("HYPERDB_STATE_DIR", tmp.path().to_str().unwrap()); - let (port, _handle, _state) = start_health_listener(); - - let info = DaemonInfo { - pid: 12345, - hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: port, - started_at: "2026-05-20T10:30:00Z".to_string(), - version: "0.1.3".to_string(), - }; - discovery::write_discovery_file(&info).unwrap(); + let started = Instant::now(); + let error = hyperdb_mcp::daemon::spawn::ensure_daemon() + .expect_err("an untrusted state directory must not yield a daemon"); - let discovered = discovery::discover().expect("should discover live daemon"); - assert_eq!(discovered.pid, 12345); - assert_eq!(discovered.health_port, port); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + assert!( + started.elapsed() < Duration::from_secs(5), + "refusal must be immediate, not a spawn wait" + ); + assert!( + !tmp.path().join(control_lock_name()).exists(), + "no daemon lock was ever taken, so nothing was spawned" + ); + assert!( + peer.stop_and_join().is_empty(), + "the untrusted directory's socket must not be contacted" + ); } -// ─── Unit tests: concurrent-spawn dedup (no env vars, safe parallel) ──────────── - +/// The user-visible end of the same rule: with an untrusted state directory +/// the engine does not use the (apparently live) daemon but runs its own +/// private `hyperd`. +#[cfg(unix)] #[test] -fn scan_for_daemon_prefers_lower_port_daemon() { - // Simulates the concurrent-spawn race: two daemons ended up on adjacent ports - // (base and base+1). The client that landed on base+1 should prefer the - // base-port daemon when it re-scans the lower range. - // - // We model this by starting two real health listeners (which answer PING with - // the identifying token), then asserting that scan_for_daemon with a 2-port - // range returns the LOWER port's daemon as Found. - let (lower_port, _lower_handle, _lower_state) = start_health_listener(); - let (higher_port, _higher_handle, _higher_state) = start_health_listener(); - - // Make sure lower_port < higher_port for a predictable result. - let (base, _top) = if lower_port < higher_port { - (lower_port, higher_port) - } else { - (higher_port, lower_port) - }; +fn engine_falls_back_to_local_mode_in_an_untrusted_state_dir() { + let _lock = acquire_env_lock(); + let tmp = TempDir::new().unwrap(); + let mut peer = FakeHealthPeer::start_in(tmp.path(), |_| pong_line().into_bytes()); + write_record(tmp.path(), &test_daemon_info(&peer.endpoint)); + make_group_writable(tmp.path()); + let _guard = EnvGuard::set("HYPERDB_STATE_DIR", tmp.path().to_str().unwrap()); - let scan = PortScan { base, span: 2 }; + let engine = hyperdb_mcp::engine::Engine::new(None) + .expect("an untrusted state directory must fall back to a private hyperd"); - match discovery::scan_for_daemon(scan) { - discovery::ScanOutcome::Found(info) => { - assert_eq!( - info.health_port, base, - "scan should return the LOWEST-port daemon (the first one bound)" - ); - } - other => panic!("expected Found, got {other:?}"), - } + assert!( + engine.daemon_health_endpoint().is_none(), + "the engine must be in local mode, not attached to the daemon" + ); + drop(engine); + assert!( + peer.stop_and_join().is_empty(), + "the untrusted directory's socket must not be contacted" + ); } // ─── Unit tests: Version takeover decision (no env vars, safe parallel) ───────── @@ -1490,6 +1210,303 @@ fn takeover_decision_both_unparseable_reuses() { ); } +// ─── CLI tests: `daemon status` / `daemon stop` / `daemon` against fakes ────── + +/// With no daemon at all, `status` and `stop` say so and exit 1. +#[test] +fn daemon_cli_status_and_stop_without_a_daemon_exit_1() { + let state = TempDir::new().unwrap(); + + for action in ["status", "stop"] { + let output = run_cli_bounded(&["daemon", action], state.path(), Duration::from_secs(5)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "daemon {action}: {stderr}"); + assert!( + stderr.contains("No daemon is currently running."), + "daemon {action} stderr was {stderr:?}" + ); + } +} + +/// A record left behind by a dead daemon (right shape, nobody serving the +/// socket) is not a daemon: the CLI reports none, and does not clean up — +/// cleanup belongs to whoever holds the daemon lock. +#[test] +fn daemon_cli_status_ignores_a_stale_record_and_leaves_it() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let record = write_record(state.path(), &test_daemon_info(&endpoint)); + + let output = run_cli_bounded(&["daemon", "status"], state.path(), Duration::from_secs(5)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "{stderr}"); + assert!( + stderr.contains("No daemon is currently running."), + "{stderr}" + ); + assert_eq!( + std::fs::read(state.path().join("daemon.json")).unwrap(), + record, + "`daemon status` must not delete or rewrite the record" + ); +} + +/// `daemon status` talks to the endpoint recorded in the state directory +/// (selected by `HYPERDB_STATE_DIR`) and prints that daemon's own STATUS. +#[test] +fn daemon_status_cli_queries_the_recorded_endpoint() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let info = DaemonInfo { + pid: 515_151, + ..test_daemon_info(&endpoint) + }; + write_record(state.path(), &info); + let mut peer = FakeHealthPeer::start_at(&endpoint, scripted_daemon(&info)); + + let output = run_cli_bounded(&["daemon", "status"], state.path(), Duration::from_secs(10)); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "`daemon status` failed: {}\nstdout:\n{stdout}\nstderr:\n{stderr}", + output.status + ); + assert!( + stdout.contains(&format!("Health endpoint: {}", endpoint.as_str())), + "status must report the endpoint it asked\nstdout:\n{stdout}" + ); + assert!( + stdout.contains("515151"), + "status must report the pid\n{stdout}" + ); + + let commands = peer.stop_and_join(); + assert_eq!( + commands + .iter() + .filter(|command| *command == "STATUS") + .count(), + 1, + "exactly one STATUS expected, got {commands:?}" + ); +} + +/// `daemon stop` sends STOP to the recorded endpoint and relays the answer. +#[test] +fn daemon_stop_cli_sends_stop_to_the_recorded_endpoint() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let info = test_daemon_info(&endpoint); + write_record(state.path(), &info); + let mut peer = FakeHealthPeer::start_at(&endpoint, scripted_daemon(&info)); + + let output = run_cli_bounded(&["daemon", "stop"], state.path(), Duration::from_secs(10)); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "`daemon stop` failed: {}\nstdout:\n{stdout}\nstderr:\n{stderr}", + output.status + ); + assert!( + stdout.contains("Daemon responded: STOPPING"), + "stdout was {stdout:?}" + ); + assert!( + peer.stop_and_join().iter().any(|command| command == "STOP"), + "the peer never received STOP" + ); +} + +/// A live daemon whose `daemon.json` is gone is still reachable by `daemon +/// stop` and `daemon status`: the held lock plus an identified `PING` on the +/// well-known socket find it. Unix only (a Windows pipe name lives in the record). +#[cfg(unix)] +#[test] +fn daemon_cli_reaches_a_live_daemon_whose_record_is_missing() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let info = test_daemon_info(&endpoint); + // No record is written. The held lock stands in for the live daemon. + let _lock = DaemonLock::try_acquire(state.path()) + .unwrap() + .expect("fresh state directory has a free lock"); + let mut peer = FakeHealthPeer::start_at(&endpoint, scripted_daemon(&info)); + + let status = run_cli_bounded(&["daemon", "status"], state.path(), Duration::from_secs(10)); + assert!( + status.status.success(), + "`daemon status` failed: {}\nstderr:\n{}", + status.status, + String::from_utf8_lossy(&status.stderr) + ); + let stop = run_cli_bounded(&["daemon", "stop"], state.path(), Duration::from_secs(10)); + assert!( + stop.status.success(), + "`daemon stop` failed: {}\nstderr:\n{}", + stop.status, + String::from_utf8_lossy(&stop.stderr) + ); + assert!( + peer.stop_and_join().iter().any(|command| command == "STOP"), + "the peer never received STOP" + ); +} + +/// A held lock with no record and nothing answering gets a clear message, not +/// "No daemon is currently running." +#[test] +fn daemon_cli_explains_a_held_lock_with_no_usable_record() { + let state = TempDir::new().unwrap(); + let _lock = DaemonLock::try_acquire(state.path()) + .unwrap() + .expect("fresh state directory has a free lock"); + + for args in [["daemon", "status"], ["daemon", "stop"]] { + let output = run_cli_bounded(&args, state.path(), Duration::from_secs(10)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "{args:?}: {stderr}"); + assert!( + stderr.contains("daemon lock is held but no usable record was found"), + "{args:?} stderr was {stderr:?}" + ); + assert!( + !stderr.contains("No daemon is currently running"), + "{stderr}" + ); + } +} + +/// A record naming a live socket outside the state directory is rejected by the +/// CLI without connecting to it. Unix only (see +/// `discover_never_connects_to_an_endpoint_outside_the_state_dir`). +#[cfg(unix)] +#[test] +fn daemon_cli_does_not_follow_a_record_to_a_foreign_socket() { + let state = TempDir::new().unwrap(); + let elsewhere = TempDir::new().unwrap(); + let foreign_info = test_daemon_info(&HealthEndpoint::for_new_daemon(elsewhere.path()).unwrap()); + let mut foreign = FakeHealthPeer::start_in(elsewhere.path(), scripted_daemon(&foreign_info)); + write_record(state.path(), &foreign_info); + + let output = run_cli_bounded(&["daemon", "status"], state.path(), Duration::from_secs(5)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "{stderr}"); + assert!( + stderr.contains("No daemon is currently running."), + "{stderr}" + ); + assert!( + foreign.stop_and_join().is_empty(), + "the CLI connected to a socket that is not its state directory's" + ); +} + +/// A group-writable state directory is untrusted: the CLI reports no daemon and +/// never contacts the socket in it, even though a live daemon is answering. +#[cfg(unix)] +#[test] +fn daemon_cli_does_not_trust_a_group_writable_state_dir() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let info = test_daemon_info(&endpoint); + write_record(state.path(), &info); + let mut peer = FakeHealthPeer::start_at(&endpoint, scripted_daemon(&info)); + make_group_writable(state.path()); + + let output = run_cli_bounded(&["daemon", "status"], state.path(), Duration::from_secs(5)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "{stderr}"); + assert!(stderr.contains("cannot be trusted"), "{stderr}"); + assert!( + peer.stop_and_join().is_empty(), + "an untrusted state directory's socket must not be contacted" + ); +} + +// ─── Single-instance tests: the daemon lock, not a port, decides ────────────── + +/// While a daemon holds the lock and answers PING, a second `hyperdb-mcp +/// daemon` in the same state directory is refused promptly with the +/// "already running" message. +/// +/// Replaces `health_listener_second_bind_same_port_fails`. What this proves: +/// the second start sees a held lock and a live, PING-answering daemon for the +/// debounce interval, and bails *before* binding anything or spawning a +/// `hyperd` — the pre-existing record and the live socket are untouched (a +/// second daemon would have rewritten the record), and the live peer only ever +/// saw PINGs. It does not start a real `hyperd`; the real-process variant is +/// `second_daemon_in_the_same_state_dir_is_refused_while_the_first_runs`. +#[test] +fn second_daemon_is_refused_while_the_lock_is_held_by_a_live_daemon() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let info = test_daemon_info(&endpoint); + let held = DaemonLock::try_acquire(state.path()) + .unwrap() + .expect("fresh state directory is lockable"); + let record = write_record(state.path(), &info); + let mut peer = FakeHealthPeer::start_at(&endpoint, scripted_daemon(&info)); + + let started = Instant::now(); + let output = run_cli_bounded(&["daemon"], state.path(), Duration::from_secs(20)); + let elapsed = started.elapsed(); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + !output.status.success(), + "a second daemon must not start; stderr:\n{stderr}" + ); + assert!( + stderr.contains("already running"), + "stderr must say a daemon is already running, got:\n{stderr}" + ); + assert!( + elapsed < hyperdb_mcp::daemon::run::STARTUP_WAIT, + "the refusal took {elapsed:?}; a live daemon must be detected after \ + about a second, not after the {:?} startup wait", + hyperdb_mcp::daemon::run::STARTUP_WAIT + ); + assert_eq!( + std::fs::read(state.path().join("daemon.json")).unwrap(), + record, + "the refused daemon must not touch the running daemon's record" + ); + let commands = peer.stop_and_join(); + assert!( + !commands.is_empty() && commands.iter().all(|command| command == "PING"), + "the live daemon should only have been PINGed, got {commands:?}" + ); + drop(held); +} + +/// A held lock with nothing answering means a daemon that is starting or +/// stopping: a new daemon waits the full startup budget for the lock and then +/// gives up with a lock-specific message instead of racing it. +#[test] +fn second_daemon_waits_out_a_lock_held_without_a_daemon_then_times_out() { + let state = TempDir::new().unwrap(); + let _held = DaemonLock::try_acquire(state.path()) + .unwrap() + .expect("fresh state directory is lockable"); + + let started = Instant::now(); + let output = run_cli_bounded(&["daemon"], state.path(), Duration::from_secs(30)); + let elapsed = started.elapsed(); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!(!output.status.success(), "stderr:\n{stderr}"); + assert!( + stderr.contains("daemon lock"), + "stderr must name the lock, got:\n{stderr}" + ); + assert!( + elapsed + Duration::from_secs(1) >= hyperdb_mcp::daemon::run::STARTUP_WAIT, + "gave up after {elapsed:?}; a starting/stopping daemon deserves the full wait" + ); +} + // ─── Integration tests: full daemon lifecycle with real hyperd ───────────────── #[test] @@ -1535,7 +1552,7 @@ fn daemon_mode_two_engines_share_same_hyperd() { hyperdb_mcp::engine::Engine::new(Some(path2.to_str().unwrap().to_string())).unwrap(); // Both engines should be in daemon mode (connected to the same daemon). - // We verify via the health port rather than the hyperd endpoint, because + // We verify via the health endpoint rather than the hyperd endpoint, because // the daemon's liveness monitor can restart hyperd (changing the endpoint) // between the two Engine::new calls. let ep1 = engine1.hyperd_endpoint().unwrap(); @@ -1544,8 +1561,8 @@ fn daemon_mode_two_engines_share_same_hyperd() { !ep1.is_empty() && !ep2.is_empty(), "both engines must report a daemon endpoint" ); - // Verify the daemon is the one we started (health port reachable) - let status = health::send_command(daemon.info.health_port, "PING").unwrap(); + // Verify the daemon is the one we started (health endpoint reachable) + let status = health::send_command(&daemon.endpoint, "PING").unwrap(); assert!(status.trim().starts_with("PONG hyperdb-mcp ")); engine1.execute_command("CREATE TABLE foo (x INT)").unwrap(); @@ -1615,7 +1632,7 @@ fn daemon_mode_persistent_engine_data_is_queryable() { assert_eq!(rows[0]["name"], "alpha"); assert_eq!(rows[1]["name"], "beta"); - let resp = health::send_command(daemon.info.health_port, "PING").unwrap(); + let resp = health::send_command(&daemon.endpoint, "PING").unwrap(); assert!(resp.trim().starts_with("PONG hyperdb-mcp ")); } @@ -1640,7 +1657,7 @@ fn hyperd_monitor_detects_killed_hyperd_and_restarts() { // (5s monitor tick + cold spawn + slack) so a loaded CI runner doesn't trip // a false timeout; see RESTART_READINESS_BUDGET_SECS. let new_endpoint = wait_for_live_hyperd_after_kill( - daemon.info.health_port, + &daemon.endpoint, &daemon.info.hyperd_endpoint, RESTART_READINESS_BUDGET_SECS, ) @@ -1674,11 +1691,11 @@ fn client_report_triggers_restart_after_kill() { // Even so, the monitor only reacts on its 5s tick, so the worst-case // recovery time is unchanged. This test just verifies the report path // triggers the same restart as detection-via-polling. - let response = health::send_command(daemon.info.health_port, "REPORT_HYPERD_ERROR").unwrap(); + let response = health::send_command(&daemon.endpoint, "REPORT_HYPERD_ERROR").unwrap(); assert_eq!(response.trim(), "OK"); let new_endpoint = wait_for_live_hyperd_after_kill( - daemon.info.health_port, + &daemon.endpoint, &daemon.info.hyperd_endpoint, RESTART_READINESS_BUDGET_SECS, ) @@ -1737,7 +1754,7 @@ fn engine_recovers_after_hyperd_killed() { // it flips what STATUS serves, so a live endpoint from STATUS means the // discovery file `Engine::new` reads below is already committed. wait_for_live_hyperd_after_kill( - daemon.info.health_port, + &daemon.endpoint, &daemon.info.hyperd_endpoint, RESTART_READINESS_BUDGET_SECS, ) @@ -1782,117 +1799,201 @@ fn daemon_mode_ephemeral_database_cleaned_up_on_drop() { ); } +/// The real-process variant of the single-instance guarantee: with a genuine +/// daemon (and its `hyperd`) running, a second `hyperdb-mcp daemon` process in +/// the same state directory is refused with "already running", and the first +/// daemon is left undisturbed — same record, same pid, still answering. +#[test] +#[cfg_attr( + target_os = "macos", + ignore = "flaky on macOS CI — daemon startup exceeds 150s timeout" +)] +fn second_daemon_in_the_same_state_dir_is_refused_while_the_first_runs() { + let _lock = acquire_env_lock(); + let daemon = TestDaemon::start(); + let record = std::fs::read(daemon.state_dir.join("daemon.json")).unwrap(); + + let started = Instant::now(); + let output = run_cli_bounded(&["daemon"], &daemon.state_dir, Duration::from_secs(30)); + let elapsed = started.elapsed(); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + !output.status.success(), + "a second daemon must not start; stderr:\n{stderr}" + ); + assert!(stderr.contains("already running"), "stderr:\n{stderr}"); + assert!( + elapsed < hyperdb_mcp::daemon::run::STARTUP_WAIT, + "refusal took {elapsed:?}" + ); + + assert_eq!( + std::fs::read(daemon.state_dir.join("daemon.json")).unwrap(), + record, + "the refused daemon must not have rewritten the record" + ); + let status = health::send_command(&daemon.endpoint, "STATUS").unwrap(); + let parsed: serde_json::Value = serde_json::from_str(status.trim()).unwrap(); + assert_eq!( + parsed["pid"], + std::process::id(), + "the original (in-process) daemon must still be the one answering" + ); + assert_eq!(parsed["hyperd_endpoint"], daemon.info.hyperd_endpoint); +} + // ─── Test helpers ───────────────────────────────────────────────────────────── -struct OwnedHealthListener { - state: Arc, - handle: Option>, +/// A per-test state directory and the health endpoint inside it. The directory +/// is short (a system temp dir plus a random name) so that +/// `/daemon.sock` stays under the platform's `sun_path` limit. +fn test_endpoint() -> (TempDir, HealthEndpoint) { + let dir = TempDir::new().expect("create short per-test state dir"); + let endpoint = HealthEndpoint::for_new_daemon(dir.path()).expect("derive health endpoint"); + (dir, endpoint) } -impl OwnedHealthListener { - fn start(listener: HealthListener, info: DaemonInfo) -> Self { - let state = Arc::new(DaemonState::new()); - let run_state = Arc::clone(&state); - let info = Arc::new(Mutex::new(info)); - let handle = std::thread::spawn(move || listener.run(run_state, info)); - Self { - state, - handle: Some(handle), - } +/// The reply a genuine daemon gives to `PING`. +fn pong_line() -> String { + format!("PONG hyperdb-mcp {}\n", hyperdb_mcp::version::MCP_VERSION) +} + +/// A `DaemonInfo` for a daemon serving `endpoint`, with a fixed identity. +fn test_daemon_info(endpoint: &HealthEndpoint) -> DaemonInfo { + DaemonInfo { + pid: 12345, + hyperd_endpoint: "127.0.0.1:54321".to_string(), + health_endpoint: endpoint.as_str().to_string(), + started_at: "2026-05-20T10:30:00Z".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), } +} - fn stop_and_join(&mut self) { - self.state.request_shutdown(); - if let Some(handle) = self.handle.take() { - handle - .join() - .expect("owned health listener must shut down cleanly"); - } +/// Write `info` as `daemon.json` in `state_dir` and return the exact bytes, so +/// a test can later prove the record was left alone. +fn write_record(state_dir: &Path, info: &DaemonInfo) -> Vec { + let bytes = serde_json::to_vec_pretty(info).expect("serialize test daemon record"); + std::fs::write(state_dir.join("daemon.json"), &bytes).expect("write test daemon record"); + bytes +} + +/// A reply script for [`FakeHealthPeer`] that behaves like a daemon: an +/// identified `PONG`, the given record as `STATUS`, and `STOPPING`. +fn scripted_daemon(info: &DaemonInfo) -> impl Fn(&str) -> Vec + Send + 'static { + let status = format!( + "{}\n", + serde_json::to_string(info).expect("serialize test status") + ); + move |command| match command { + "PING" => pong_line().into_bytes(), + "STATUS" => status.clone().into_bytes(), + "STOP" => b"STOPPING\n".to_vec(), + _ => b"ERR unknown command\n".to_vec(), } } -impl Drop for OwnedHealthListener { - fn drop(&mut self) { - self.stop_and_join(); +/// Name of the file whose presence proves a daemon lock was taken. +#[cfg(unix)] +fn control_lock_name() -> &'static str { + hyperdb_mcp::daemon::lock::LOCK_FILE_NAME +} + +/// Remove the "private to this user" property from a state directory. +#[cfg(unix)] +fn make_group_writable(dir: &Path) { + use std::os::unix::fs::PermissionsExt as _; + + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o770)) + .expect("make test state dir group-writable"); +} + +/// Read one line (including its newline) a byte at a time, so no buffered +/// reader swallows bytes meant for a later read. An empty string means EOF +/// before any byte. +fn read_line_from(stream: &mut ControlStream) -> std::io::Result { + let mut line = Vec::new(); + let mut byte = [0_u8]; + loop { + if stream.read(&mut byte)? == 0 { + break; + } + line.push(byte[0]); + if byte[0] == b'\n' { + break; + } } + String::from_utf8(line) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error)) } -struct CliStatusListener { - port: u16, +/// A scripted stand-in for a daemon: serves a control endpoint, records every +/// command it is sent, and answers each with whatever the script returns. +/// Drives the code under test without a real daemon or `hyperd`. +struct FakeHealthPeer { + endpoint: HealthEndpoint, + stop: Arc, handle: Option>>, } -impl CliStatusListener { - fn start() -> Self { - use std::io::{BufRead, BufReader, Write}; - - let listener = TcpListener::bind(("127.0.0.1", 0)) - .expect("bind OS-assigned explicit daemon-status listener"); - let port = listener - .local_addr() - .expect("read explicit daemon-status listener address") - .port(); +impl FakeHealthPeer { + /// Serve `/daemon.sock`. + fn start_in(state_dir: &Path, reply: impl Fn(&str) -> Vec + Send + 'static) -> Self { + let endpoint = HealthEndpoint::for_new_daemon(state_dir).expect("derive peer endpoint"); + Self::start_at(&endpoint, reply) + } + + /// Serve an already derived endpoint. + fn start_at( + endpoint: &HealthEndpoint, + reply: impl Fn(&str) -> Vec + Send + 'static, + ) -> Self { + let mut listener = ControlListener::bind(endpoint).expect("bind fake health peer"); + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); let handle = std::thread::spawn(move || { let mut commands = Vec::new(); - loop { - let (mut stream, _) = listener - .accept() - .expect("explicit daemon-status listener accept"); - stream - .set_read_timeout(Some(Duration::from_secs(2))) - .expect("bound explicit daemon-status request read"); - stream - .set_write_timeout(Some(Duration::from_secs(2))) - .expect("bound explicit daemon-status response write"); - let mut line = String::new(); - BufReader::new(&stream) - .read_line(&mut line) - .expect("read explicit daemon-status command"); + while !thread_stop.load(Ordering::SeqCst) { + let mut stream = match listener.accept_timeout(Duration::from_millis(50)) { + Ok(Some(stream)) => stream, + Ok(None) => continue, + Err(error) => panic!("fake health peer accept failed: {error}"), + }; + if stream.set_io_timeout(Duration::from_secs(2)).is_err() { + continue; + } + let Ok(line) = read_line_from(&mut stream) else { + continue; + }; let command = line.trim(); - if command == "TEST_SHUTDOWN" { - break; + // A bare connect-and-hang-up is a wake-up, not a command. + if command.is_empty() { + continue; } + let response = reply(command); commands.push(command.to_string()); - let response = serde_json::json!({ - "pid": 515151, - "hyperd_endpoint": "127.0.0.1:54321", - "health_port": port, - "started_at": "2026-08-14T12:34:56Z", - "version": env!("CARGO_PKG_VERSION") - }); - writeln!(stream, "{response}").expect("write explicit daemon-status response"); + let _ = stream.write_all(&response); + let _ = stream.finish(); } commands }); Self { - port, + endpoint: endpoint.clone(), + stop, handle: Some(handle), } } + /// Stop serving and return every command received, in order. fn stop_and_join(&mut self) -> Vec { - use std::io::Write; - - if self.handle.is_none() { - return Vec::new(); - } - let mut stream = std::net::TcpStream::connect(("127.0.0.1", self.port)) - .expect("connect explicit daemon-status listener shutdown"); - stream - .set_write_timeout(Some(Duration::from_secs(2))) - .expect("bound explicit daemon-status shutdown write"); - stream - .write_all(b"TEST_SHUTDOWN\n") - .expect("signal explicit daemon-status listener shutdown"); - self.handle - .take() - .expect("explicit daemon-status listener handle exists") - .join() - .expect("explicit daemon-status listener must join") + self.stop.store(true, Ordering::SeqCst); + self.handle.take().map_or_else(Vec::new, |handle| { + handle.join().expect("fake health peer must join cleanly") + }) } } -impl Drop for CliStatusListener { +impl Drop for FakeHealthPeer { fn drop(&mut self) { if self.handle.is_some() { let _ = self.stop_and_join(); @@ -1900,47 +2001,110 @@ impl Drop for CliStatusListener { } } -fn run_cli_child_with_watchdog( - args: &[String], - state_dir: &std::path::Path, - base_port: u16, -) -> Output { +/// A real [`HealthListener`] running on its own thread against a real +/// [`DaemonState`], bound inside a caller-owned state directory. +struct RunningHealth { + endpoint: HealthEndpoint, + state: Arc, + info: Arc>, + handle: Option>, +} + +impl RunningHealth { + fn start_with_info( + state_dir: &Path, + make_info: impl FnOnce(&HealthEndpoint) -> DaemonInfo, + ) -> Self { + let endpoint = HealthEndpoint::for_new_daemon(state_dir).expect("derive health endpoint"); + let listener = HealthListener::bind(&endpoint).expect("bind running health listener"); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(make_info(&endpoint))); + let run_state = Arc::clone(&state); + let run_info = Arc::clone(&info); + let handle = std::thread::spawn(move || listener.run(run_state, run_info)); + Self { + endpoint, + state, + info, + handle: Some(handle), + } + } + + /// Wait for the listener thread to return (after a `STOP`, say). + fn join(&mut self) { + if let Some(handle) = self.handle.take() { + handle + .join() + .expect("running health listener must shut down cleanly"); + } + } + + fn stop_and_join(&mut self) { + self.state.request_shutdown(); + self.join(); + } +} + +impl Drop for RunningHealth { + fn drop(&mut self) { + self.stop_and_join(); + } +} + +/// Starts a health listener in `state_dir`. Does NOT touch env vars — safe for +/// parallel use. The caller owns (and must outlive) the directory. +fn start_health_listener_in(state_dir: &Path) -> RunningHealth { + RunningHealth::start_with_info(state_dir, test_daemon_info) +} + +/// [`start_health_listener_in`] in a fresh short state directory. +fn start_health_listener() -> (TempDir, RunningHealth) { + let dir = TempDir::new().expect("create short per-test state dir"); + let health = start_health_listener_in(dir.path()); + (dir, health) +} + +/// Run `hyperdb-mcp ` against `state_dir` with a scrubbed environment and +/// a deadline. The child never sees the developer's real `HOME` or state +/// directory: both are pinned to `state_dir`, so a regression's blast radius +/// stays inside the test. A child that outlives `deadline` is killed and the +/// test fails with its output. +fn run_cli_bounded(args: &[&str], state_dir: &Path, deadline: Duration) -> Output { let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_hyperdb-mcp")); command.env_clear(); preserve_child_runtime_environment(&mut command); command - .current_dir( - state_dir - .parent() - .expect("isolated daemon-status state has a parent"), - ) + .current_dir(state_dir) .env("HOME", state_dir) .env("USERPROFILE", state_dir) .env("HYPERDB_STATE_DIR", state_dir) - .env("HYPERDB_DAEMON_PORT", base_port.to_string()) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .args(args); - let mut child = command.spawn().expect("spawn isolated daemon-status child"); - let deadline = Instant::now() + Duration::from_secs(5); + if let Some(hyperd) = std::env::var_os("HYPERD_PATH") { + command.env("HYPERD_PATH", hyperd); + } + let mut child = command.spawn().expect("spawn hyperdb-mcp CLI child"); + let give_up_at = Instant::now() + deadline; loop { match child.try_wait() { Ok(Some(_)) => { return child .wait_with_output() - .expect("collect completed daemon-status child output"); + .expect("collect completed CLI child output"); } - Ok(None) if Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(10)); + Ok(None) if Instant::now() < give_up_at => { + std::thread::sleep(Duration::from_millis(20)); } Ok(None) => { let kill_error = child.kill().err(); let output = child .wait_with_output() - .expect("wait for timed-out daemon-status child after kill"); + .expect("wait for timed-out CLI child after kill"); panic!( - "daemon-status child exceeded 5s and was killed ({kill_error:?})\nstdout:\n{}\nstderr:\n{}", + "`hyperdb-mcp {}` exceeded {deadline:?} and was killed ({kill_error:?})\nstdout:\n{}\nstderr:\n{}", + args.join(" "), String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); @@ -1949,9 +2113,9 @@ fn run_cli_child_with_watchdog( let kill_error = child.kill().err(); let output = child .wait_with_output() - .expect("wait for daemon-status child after status error"); + .expect("wait for CLI child after status error"); panic!( - "daemon-status child status failed: {error}; kill result: {kill_error:?}\nstdout:\n{}\nstderr:\n{}", + "CLI child status failed: {error}; kill result: {kill_error:?}\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); @@ -1976,67 +2140,34 @@ fn preserve_child_runtime_environment(command: &mut std::process::Command) { } } -/// Starts a health listener on a random port and returns the port, join handle, -/// and shared state. Does NOT touch env vars — safe for parallel use. -fn start_health_listener() -> (u16, std::thread::JoinHandle<()>, Arc) { - let listener = HealthListener::bind(0).unwrap(); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let run_state = Arc::clone(&state); - - let info = Arc::new(Mutex::new(DaemonInfo { - pid: 12345, - hyperd_endpoint: "127.0.0.1:54321".to_string(), - health_port: port, - started_at: "2026-05-20T10:30:00Z".to_string(), - version: "0.1.3".to_string(), - })); - - let handle = std::thread::spawn(move || { - listener.run(run_state, info); - }); - - // Give the listener a moment to start accepting - std::thread::sleep(Duration::from_millis(50)); - - (port, handle, state) -} - /// A real daemon running in a background thread for integration tests. -/// Sets `HYPERDB_STATE_DIR` and `HYPERDB_DAEMON_PORT` to isolated values. +/// Sets `HYPERDB_STATE_DIR` to an isolated, short directory. /// Caller MUST hold `ENV_LOCK` before calling `start()`. struct TestDaemon { info: DaemonInfo, + endpoint: HealthEndpoint, + state_dir: PathBuf, _state_dir_guard: EnvGuard, - _port_guard: EnvGuard, + /// Declared last so the directory outlives every other field's drop. + _tmp: TempDir, } impl TestDaemon { fn start() -> Self { let tmp = TempDir::new().unwrap(); - // Leak the TempDir so it persists for the lifetime of the test. - let tmp = Box::leak(Box::new(tmp)); + let state_dir = tmp.path().to_path_buf(); - let state_dir_guard = EnvGuard::set("HYPERDB_STATE_DIR", tmp.path().to_str().unwrap()); - - // Pass port 0 so the daemon's HealthListener binds an OS-assigned - // free port and reports it back via the discovery file. We avoid - // the find_free_port → set env → daemon binds later TOCTOU race - // where another process could grab the port between pick and bind - // (a real source of flakes on busy CI runners). The - // `HYPERDB_DAEMON_PORT` env var only matters for the - // spawn-daemon-if-missing path; once we've written the discovery - // file, clients read `health_port` directly from it. - let port_guard = EnvGuard::set("HYPERDB_DAEMON_PORT", "0"); + let state_dir_guard = EnvGuard::set("HYPERDB_STATE_DIR", state_dir.to_str().unwrap()); // Start the daemon in a background tokio runtime. Capture errors // via a JoinHandle so a bind/spawn failure fails the test fast - // with a real message instead of a 30s timeout. + // with a real message instead of a 30s timeout. The daemon takes the + // state directory from `HYPERDB_STATE_DIR` and serves + // `/daemon.sock`; there is no port to choose or race for. let daemon_handle = std::thread::spawn(move || -> Result<(), String> { let rt = tokio::runtime::Runtime::new().map_err(|e| format!("rt: {e}"))?; rt.block_on(async { let config = hyperdb_mcp::daemon::run::DaemonConfig { - port: 0, idle_timeout: Some(Duration::from_secs(300)), }; hyperdb_mcp::daemon::run::run_daemon(config) @@ -2060,10 +2191,14 @@ impl TestDaemon { let start = Instant::now(); loop { if let Some(info) = discovery::discover() { + let endpoint = HealthEndpoint::from_record(&info.health_endpoint, &state_dir) + .expect("a discovered record names this state directory's endpoint"); return Self { info, + endpoint, + state_dir, _state_dir_guard: state_dir_guard, - _port_guard: port_guard, + _tmp: tmp, }; } // Fail fast if the daemon thread has already exited (bind error, @@ -2087,15 +2222,33 @@ impl TestDaemon { impl Drop for TestDaemon { fn drop(&mut self) { - let _ = health::send_command(self.info.health_port, "STOP"); - // Wait until the daemon's health port is unreachable, indicating full - // shutdown (HyperProcess Drop can take up to ~5s). 200ms was not - // enough — under load, the next test could find the port still bound. - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline { - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], self.info.health_port)); - if std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(200)).is_err() { - return; + let _ = health::send_command(&self.endpoint, "STOP"); + // The daemon holds the state directory's lock until after its `hyperd` + // has exited, so the lock being free is the one reliable sign of full + // shutdown (`HyperProcess` drop can take ~5s). Waiting on the control + // endpoint instead would return before `hyperd` is gone, and the next + // test could find the previous one's engine still winding down. + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match DaemonLock::try_acquire(&self.state_dir) { + Ok(Some(lock)) => { + drop(lock); + return; + } + // Cannot be checked at all (directory gone): nothing to wait on. + Err(_) => return, + Ok(None) => {} + } + if Instant::now() >= deadline { + let message = format!( + "TestDaemon: the daemon lock in {} was still held 15s after STOP", + self.state_dir.display() + ); + if std::thread::panicking() { + eprintln!("{message}"); + return; + } + panic!("{message}"); } std::thread::sleep(Duration::from_millis(100)); } @@ -2196,7 +2349,7 @@ fn kill_pid(pid: u32) { /// its whole timeout on a perfectly healthy restart. #[cfg(unix)] fn wait_for_live_hyperd_after_kill( - health_port: u16, + health_endpoint: &HealthEndpoint, killed_endpoint: &str, timeout_secs: u64, ) -> Option { @@ -2220,7 +2373,7 @@ fn wait_for_live_hyperd_after_kill( // Phase 2: whatever STATUS advertises now must be reachable. With the old // socket proven down, a successful connect means a live replacement. while Instant::now() < deadline { - if let Ok(response) = health::send_command(health_port, "STATUS") + if let Ok(response) = health::send_command(health_endpoint, "STATUS") && let Ok(parsed) = serde_json::from_str::(response.trim()) && let Some(endpoint) = parsed["hyperd_endpoint"].as_str() && endpoint_accepts_connection(endpoint) diff --git a/hyperdb-mcp/tests/doctor_tests.rs b/hyperdb-mcp/tests/doctor_tests.rs index 8ecc691f..8ea819c3 100644 --- a/hyperdb-mcp/tests/doctor_tests.rs +++ b/hyperdb-mcp/tests/doctor_tests.rs @@ -4,12 +4,12 @@ //! Native CLI contracts for the side-effect-free `doctor` report. use std::ffi::{OsStr, OsString}; -use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; +use hyperdb_mcp::daemon::control::HealthEndpoint; use hyperdb_mcp::daemon::discovery::DaemonInfo; use hyperdb_mcp::daemon::health::{self, DaemonState, HealthListener}; use serde_json::{Value, json}; @@ -48,6 +48,8 @@ enum SnapshotNode { Directory, File(Vec), Symlink(PathBuf), + /// A socket, FIFO or device: recorded by kind, never read. + Special, } #[derive(Debug)] @@ -61,21 +63,22 @@ struct DoctorSandbox { wrapper_package_path: PathBuf, platform_package_path: PathBuf, launcher_executable_path: PathBuf, - isolated_daemon_port: u16, - _isolated_daemon_listener: TcpListener, } impl DoctorSandbox { fn new() -> Self { + // `/daemon.sock` must fit in `sockaddr_un.sun_path` (103 + // bytes on macOS). The default temp root is already ~60 bytes there, + // so on Unix root the sandbox under the short `/tmp` instead. + #[cfg(unix)] + let temp_dir = tempfile::Builder::new() + .prefix("hdbdoc") + .tempdir_in("/tmp") + .expect("create isolated doctor test root"); + #[cfg(not(unix))] let temp_dir = TempDir::new().expect("create isolated doctor test root"); let root = canonicalize_for_test(temp_dir.path()).expect("canonicalize isolated doctor test root"); - let isolated_daemon_listener = TcpListener::bind(("127.0.0.1", 0)) - .expect("reserve an OS-assigned foreign daemon-isolation port"); - let isolated_daemon_port = isolated_daemon_listener - .local_addr() - .expect("read daemon-isolation listener address") - .port(); Self { state_dir: root.join("state-must-not-be-created"), persistent_path: root @@ -86,8 +89,6 @@ impl DoctorSandbox { wrapper_package_path: root.join("npm/wrapper/package.json"), platform_package_path: root.join("npm/platform/package.json"), launcher_executable_path: root.join("npm/platform/hyperdb-mcp"), - isolated_daemon_port, - _isolated_daemon_listener: isolated_daemon_listener, _temp_dir: temp_dir, root, } @@ -122,7 +123,6 @@ impl DoctorSandbox { Some(self.persistent_path.as_os_str()), launcher_metadata, Some(hyperd_path), - self.isolated_daemon_port, ) } @@ -132,14 +132,12 @@ impl DoctorSandbox { persistent_environment: Option<&OsStr>, launcher_metadata: &str, hyperd_path: Option<&OsStr>, - daemon_port: u16, ) -> Output { self.run_with_home_options( args, persistent_environment, launcher_metadata, hyperd_path, - daemon_port, self.home_dir.as_os_str(), ) } @@ -150,7 +148,6 @@ impl DoctorSandbox { persistent_environment: Option<&OsStr>, launcher_metadata: &str, hyperd_path: Option<&OsStr>, - daemon_port: u16, home_profile: &OsStr, ) -> Output { let mut command = Command::new(env!("CARGO_BIN_EXE_hyperdb-mcp")); @@ -174,7 +171,6 @@ impl DoctorSandbox { .env("TEMP", &self.runtime_tmp_dir) .env("HYPERDB_STATE_DIR", &self.state_dir) .env("HYPERDB_MCP_LAUNCHER_INFO", launcher_metadata) - .env("HYPERDB_DAEMON_PORT", daemon_port.to_string()) .env("NO_COLOR", "1") .args(args); if let Some(path) = persistent_environment { @@ -233,29 +229,50 @@ impl DoctorSandbox { #[derive(Debug)] struct RunningHealthListener { - port: u16, + endpoint: HealthEndpoint, info: DaemonInfo, state: Arc, handle: Option>, } impl RunningHealthListener { - fn start() -> Self { - let listener = HealthListener::bind(0).expect("bind OS-assigned health-listener port"); - let port = listener.port; + /// Bind the real health listener on `/daemon.sock`, + /// creating the state directory first (a real daemon owns it). + fn start(sandbox: &DoctorSandbox) -> Self { + Self::start_reporting(sandbox, None) + } + + /// Like [`Self::start`], but the STATUS reply (not the record we return in + /// `info`) carries `reported_endpoint` instead of the bound one, to model a + /// daemon whose status disagrees with its discovery record. + fn start_reporting(sandbox: &DoctorSandbox, reported_endpoint: Option<&str>) -> Self { + std::fs::create_dir_all(&sandbox.state_dir).expect("create daemon state directory"); + let endpoint = HealthEndpoint::for_new_daemon(&sandbox.state_dir) + .expect("derive the health endpoint for the sandbox state directory"); + #[cfg(unix)] + assert_eq!( + Path::new(endpoint.as_str()), + sandbox.state_dir.join("daemon.sock"), + "the health endpoint must be exactly /daemon.sock" + ); + let listener = HealthListener::bind(&endpoint).expect("bind real health listener"); let info = DaemonInfo { pid: std::process::id(), hyperd_endpoint: "127.0.0.1:54321".to_owned(), - health_port: port, + health_endpoint: endpoint.as_str().to_owned(), started_at: "2026-08-14T12:34:56Z".to_owned(), version: env!("CARGO_PKG_VERSION").to_owned(), }; - let state = Arc::new(DaemonState::new()); - let shared_info = Arc::new(Mutex::new(info.clone())); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let mut reported_info = info.clone(); + if let Some(reported) = reported_endpoint { + reported.clone_into(&mut reported_info.health_endpoint); + } + let shared_info = Arc::new(Mutex::new(reported_info)); let run_state = Arc::clone(&state); let handle = std::thread::spawn(move || listener.run(run_state, shared_info)); Self { - port, + endpoint, info, state, handle: Some(handle), @@ -263,7 +280,7 @@ impl RunningHealthListener { } fn prime_listener(&self) { - let response = health::send_command(self.port, "PING") + let response = health::send_command(&self.endpoint, "PING") .expect("real health listener must answer the priming PING"); assert!( response.starts_with("PONG hyperdb-mcp "), @@ -331,6 +348,8 @@ fn snapshot_tree(root: &Path) -> Vec<(PathBuf, SnapshotNode)> { } else if metadata.is_dir() { entries.push((relative, SnapshotNode::Directory)); visit(root, &path, entries); + } else if !metadata.is_file() { + entries.push((relative, SnapshotNode::Special)); } else { let bytes = std::fs::read(&path).unwrap_or_else(|error| { panic!("read snapshot file {}: {error}", path.display()) @@ -608,8 +627,20 @@ fn live_daemon_human_parity_failure(report: &Value, human: &str) -> Option { + let expected = format!(" PID: {pid}"); + if !human.contains(&expected) { + missing.push(format!("human live identity omitted {expected:?}")); + } + } + None => missing.push("daemon.pid missing from live identity".to_owned()), + } + for (key, label) in [ + ("health_endpoint", "Health endpoint"), + ("lock", "Daemon lock"), + ] { + let Some(value) = daemon.get(key).and_then(Value::as_str) else { missing.push(format!("daemon.{key} missing from live identity")); continue; }; @@ -653,7 +684,7 @@ fn live_daemon_fact_failure( daemon.get("state").and_then(Value::as_str), daemon.get("pid").and_then(Value::as_u64), daemon.get("hyperd_endpoint").and_then(Value::as_str), - daemon.get("health_port").and_then(Value::as_u64), + daemon.get("health_endpoint").and_then(Value::as_str), daemon.get("started_at").and_then(Value::as_str), daemon.get("version").and_then(Value::as_str), ); @@ -661,7 +692,7 @@ fn live_daemon_fact_failure( Some(expected_state), Some(u64::from(expected.pid)), Some(expected.hyperd_endpoint.as_str()), - Some(u64::from(expected.health_port)), + Some(expected.health_endpoint.as_str()), Some(expected.started_at.as_str()), Some(expected.version.as_str()), ); @@ -918,7 +949,21 @@ fn doctor_cli_json_and_human_smoke_is_side_effect_free() { .expect("daemon section must contain one typed discovery state"); assert_eq!( daemon_state, "missing", - "OS-assigned pinned isolation port must prevent a resident developer daemon from changing this smoke test" + "the isolated state directory must prevent a resident developer daemon from changing this smoke test" + ); + + assert_eq!( + daemon.get("lock").and_then(Value::as_str), + Some("absent"), + "a state directory without a lock file must report the lock as absent" + ); + assert!( + daemon.get("health_endpoint").is_none_or(Value::is_null), + "a missing daemon has no health endpoint" + ); + assert!( + !daemon.contains_key("health_port"), + "the TCP health port is gone from the typed report" ); let mut reported_paths = Vec::new(); @@ -1054,7 +1099,7 @@ fn doctor_human_output_escapes_and_bounds_reported_paths() { assert_eq!( daemon_state(&report), Some("missing"), - "OS-assigned pinned isolation port must keep the edge-case report daemon-independent" + "the isolated state directory must keep the edge-case report daemon-independent" ); assert_eq!(snapshot_tree(&sandbox.root), before); sandbox.assert_no_artifacts(); @@ -1160,7 +1205,6 @@ fn doctor_persistent_tilde_sources_match_runtime_and_preserve_cli_semantics() { case.persistent_environment.map(OsStr::new), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); if !output.status.success() { failures.push(format!( @@ -1257,7 +1301,6 @@ fn doctor_persistent_tilde_sources_match_runtime_and_preserve_cli_semantics() { Some(OsStr::new("~/ignored-while-disabled.hyper")), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); if disabled.status.success() { match serde_json::from_slice::(&disabled.stdout) { @@ -1338,7 +1381,6 @@ fn doctor_persistent_tilde_sources_match_runtime_and_preserve_cli_semantics() { None, &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); if output.status.code() != Some(2) || !output.stdout.is_empty() @@ -1376,7 +1418,6 @@ fn doctor_relative_persistent_path_normalizes_parent_and_matches_runtime() { Some(OsStr::new("ignored-environment.hyper")), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); let report = parse_json_report( &json_output, @@ -1442,7 +1483,6 @@ fn doctor_relative_persistent_path_normalizes_parent_and_matches_runtime() { Some(OsStr::new("ignored-environment.hyper")), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); assert_success( &human_output, @@ -1496,7 +1536,6 @@ fn doctor_nested_tilde_home_expands_once_and_preserves_raw_input() { Some(OsStr::new("~/ignored-environment.hyper")), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, nested_home.as_os_str(), ); let report = parse_json_report(&json_output, "hyperdb-mcp doctor --json with HOME=~/outer"); @@ -1560,7 +1599,6 @@ fn doctor_nested_tilde_home_expands_once_and_preserves_raw_input() { Some(OsStr::new("~/ignored-environment.hyper")), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, nested_home.as_os_str(), ); assert_success(&human_output, "hyperdb-mcp doctor with HOME=~/outer"); @@ -1627,7 +1665,6 @@ fn doctor_non_utf8_persistent_environment_reports_observed_and_effective_paths() Some(configured.raw.as_os_str()), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); let report = parse_json_report( &output, @@ -1721,7 +1758,6 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(configured.raw.as_os_str()), - sandbox.isolated_daemon_port, ); let report = parse_json_report( &output, @@ -1769,7 +1805,7 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { ); } if daemon_state(&report) != Some("missing") { - failures.push("non-UTF-8 case escaped the pinned daemon port".to_owned()); + failures.push("non-UTF-8 case escaped the isolated daemon state directory".to_owned()); } if snapshot_tree(&sandbox.root) != before { failures.push("non-UTF-8 HYPERD_PATH case changed filesystem bytes".to_owned()); @@ -1790,7 +1826,6 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(configured_directory.as_os_str()), - sandbox.isolated_daemon_port, ); let report = parse_json_report( &output, @@ -1827,7 +1862,9 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { )); } if daemon_state(&report) != Some("missing") { - failures.push("empty-directory case escaped the pinned daemon port".to_owned()); + failures.push( + "empty-directory case escaped the isolated daemon state directory".to_owned(), + ); } if snapshot_tree(&sandbox.root) != before { failures.push("empty-directory HYPERD_PATH case changed filesystem bytes".to_owned()); @@ -1846,7 +1883,6 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(OsStr::new("")), - sandbox.isolated_daemon_port, ); let report = parse_json_report( &output, @@ -1880,7 +1916,8 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { ); } if daemon_state(&report) != Some("missing") { - failures.push("empty UTF-8 case escaped the pinned daemon port".to_owned()); + failures + .push("empty UTF-8 case escaped the isolated daemon state directory".to_owned()); } if snapshot_tree(&sandbox.root) != before { failures.push("empty UTF-8 HYPERD_PATH case changed filesystem bytes".to_owned()); @@ -1904,7 +1941,6 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(configured_stem.as_os_str()), - sandbox.isolated_daemon_port, ); let report = parse_json_report( &output, @@ -1935,7 +1971,8 @@ fn doctor_hyperd_path_diagnostics_match_runtime_resolution() { failures.push("accepted Windows HYPERD_PATH must suppress upward fallback".to_owned()); } if daemon_state(&report) != Some("missing") { - failures.push("Windows .exe case escaped the pinned daemon port".to_owned()); + failures + .push("Windows .exe case escaped the isolated daemon state directory".to_owned()); } if snapshot_tree(&sandbox.root) != before { failures.push("Windows .exe HYPERD_PATH case changed filesystem bytes".to_owned()); @@ -1970,7 +2007,6 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, ); let warm_report = parse_json_report(&warm, "warm hyperdb-mcp doctor --json"); if daemon_state(&warm_report) != Some("missing") { @@ -1984,8 +2020,7 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { )); } - let listener = RunningHealthListener::start(); - std::fs::create_dir_all(&sandbox.state_dir).expect("create discovery fixture directory"); + let listener = RunningHealthListener::start(&sandbox); let discovery_path = sandbox.state_dir.join("daemon.json"); let discovery_bytes = serde_json::to_vec_pretty(&listener.info).expect("serialize discovery fixture"); @@ -1998,7 +2033,6 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(missing_hyperd.as_os_str()), - listener.port, ); let report = parse_json_report( &output, @@ -2010,6 +2044,12 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { { failures.push(format!("attempt {attempt}: {failure}")); } + if report["daemon"]["lock"] != "absent" { + failures.push(format!( + "attempt {attempt}: no lock file exists, so daemon.lock must be absent: {}", + report["daemon"]["lock"] + )); + } if attempt == 1 { listener.prime_listener(); let human_output = sandbox.run_with_options( @@ -2017,7 +2057,6 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(missing_hyperd.as_os_str()), - listener.port, ); assert_success( &human_output, @@ -2056,73 +2095,83 @@ fn doctor_cli_reports_live_from_discovery_via_real_health_listener() { ); } +/// A daemon record that the live listener contradicts must surface the typed +/// `daemon_status_health_endpoint_mismatch` warning, without touching disk. #[test] -fn doctor_cli_reports_live_from_scan_via_real_health_listener() { - const ATTEMPTS: usize = 4; - let mut failures = Vec::new(); +fn doctor_cli_warns_when_status_reports_a_different_health_endpoint() { + let sandbox = DoctorSandbox::new(); + let launcher_metadata = sandbox.launcher_metadata("hyperdb-mcp-test-wrapper"); + let missing_hyperd = sandbox.root.join("missing-hyperd"); + let listener = RunningHealthListener::start_reporting(&sandbox, Some("/elsewhere/daemon.sock")); + let discovery_bytes = + serde_json::to_vec_pretty(&listener.info).expect("serialize discovery fixture"); + std::fs::write(sandbox.state_dir.join("daemon.json"), &discovery_bytes) + .expect("write discovery fixture"); + let before = snapshot_tree(&sandbox.root); - for attempt in 1..=ATTEMPTS { - let sandbox = DoctorSandbox::new(); - let launcher_metadata = sandbox.launcher_metadata("hyperdb-mcp-test-wrapper"); - let missing_hyperd = sandbox.root.join("missing-hyperd"); + listener.prime_listener(); + let output = sandbox.run_with_options( + &["doctor", "--json"], + Some(sandbox.persistent_path.as_os_str()), + &launcher_metadata, + Some(missing_hyperd.as_os_str()), + ); + let report = parse_json_report( + &output, + "hyperdb-mcp doctor --json against mismatching status", + ); + assert_exact_top_level_keys(&report); + assert!( + has_warning_code(&report, "daemon_status_health_endpoint_mismatch"), + "doctor must warn about the status/record endpoint mismatch: {report}" + ); + assert_ne!( + daemon_state(&report), + Some("live_from_discovery"), + "a contradicted record must not be reported as a live daemon" + ); + assert_eq!( + snapshot_tree(&sandbox.root), + before, + "doctor must not change the sandbox" + ); + drop(listener); +} - let warm_before = snapshot_tree(&sandbox.root); - let warm = sandbox.run_with_options( - &["doctor", "--json"], - Some(sandbox.persistent_path.as_os_str()), - &launcher_metadata, - Some(missing_hyperd.as_os_str()), - sandbox.isolated_daemon_port, - ); - let warm_report = parse_json_report(&warm, "warm hyperdb-mcp doctor --json"); - if daemon_state(&warm_report) != Some("missing") { - failures.push(format!( - "attempt {attempt}: warmup escaped pinned missing-daemon isolation" - )); - } - if snapshot_tree(&sandbox.root) != warm_before { - failures.push(format!( - "attempt {attempt}: warmup changed filesystem bytes" - )); - } +/// The lock file is only inspected when it already exists: a held lock is +/// reported as `held`, a released one as `free`, and doctor never alters it. +#[test] +fn doctor_cli_reports_daemon_lock_state_without_changing_it() { + use hyperdb_mcp::daemon::lock::DaemonLock; - let listener = RunningHealthListener::start(); - let before = snapshot_tree(&sandbox.root); - listener.prime_listener(); + let sandbox = DoctorSandbox::new(); + let launcher_metadata = sandbox.launcher_metadata("hyperdb-mcp-test-wrapper"); + let missing_hyperd = sandbox.root.join("missing-hyperd"); + std::fs::create_dir_all(&sandbox.state_dir).expect("create state directory"); + + let held = DaemonLock::try_acquire(&sandbox.state_dir) + .expect("take the daemon lock") + .expect("the lock must be free in a fresh sandbox"); + let before = snapshot_tree(&sandbox.root); + let run = || { let output = sandbox.run_with_options( &["doctor", "--json"], Some(sandbox.persistent_path.as_os_str()), &launcher_metadata, Some(missing_hyperd.as_os_str()), - listener.port, - ); - let report = parse_json_report( - &output, - "hyperdb-mcp doctor --json against scanned HealthListener", ); - if let Some(failure) = live_daemon_fact_failure(&report, "live_from_scan", &listener.info) { - failures.push(format!("attempt {attempt}: {failure}")); - } - if snapshot_tree(&sandbox.root) != before { - failures.push(format!( - "attempt {attempt}: scan doctor changed filesystem bytes" - )); - } - if sandbox.state_dir.exists() { - failures.push(format!( - "attempt {attempt}: scan doctor created a daemon state directory" - )); - } - sandbox.assert_no_artifacts(); - assert_all_strings_bounded(&report); - drop(listener); - } + parse_json_report(&output, "hyperdb-mcp doctor --json with a lock file") + }; - assert!( - failures.is_empty(), - "real scanned HealthListener regressions:\n{}", - failures.join("\n") - ); + let report = run(); + assert_eq!(report["daemon"]["lock"], "held", "{report}"); + assert_eq!(snapshot_tree(&sandbox.root), before); + + drop(held); + let before = snapshot_tree(&sandbox.root); + let report = run(); + assert_eq!(report["daemon"]["lock"], "free", "{report}"); + assert_eq!(snapshot_tree(&sandbox.root), before); } /// `--help` is the recovery surface available even when neither MCP nor @@ -2265,20 +2314,35 @@ fn cli_help_matches_hyperd_and_read_only_contract() { } } - if !(daemon_help.contains("auto-spawn") - && daemon_help.contains("scan") - && daemon_help.contains("foreground") - && daemon_help.contains("exact")) + for stale in ["--port", "hyperdb_daemon_port"] { + if daemon_help.contains(stale) || root_help.contains(stale) { + failures.push(format!( + "help must not mention {stale}: the daemon has no TCP port" + )); + } + if static_cli.contains(stale) { + failures.push(format!( + "README CLI reference must not mention {stale}: the daemon has no TCP port" + )); + } + } + if !(daemon_help.contains("foreground") + && daemon_help.contains("one daemon runs per state directory") + && daemon_help.contains("hyperdb_state_dir") + && daemon_help.contains("unix socket or named pipe") + && daemon_help.contains("not on a tcp port")) { failures.push( - "daemon help must distinguish auto-spawn port scanning from the foreground daemon's exact/base-port bind" + "daemon help must say one foreground daemon runs per state directory (HYPERDB_STATE_DIR) on a Unix socket or named pipe, not a TCP port" .to_owned(), ); } - if daemon_help.contains("daemon scans from the base port to find a free port") { + if !(daemon_help.contains("--idle-timeout") + && daemon_help.contains("stop") + && daemon_help.contains("status")) + { failures.push( - "foreground daemon help must not promise startup scanning that it does not perform" - .to_owned(), + "daemon help must list --idle-timeout and the stop/status subcommands".to_owned(), ); } @@ -2293,13 +2357,12 @@ fn cli_help_matches_hyperd_and_read_only_contract() { .to_owned(), ); } - if !(static_cli.contains("hyperdb_daemon_port") - && static_cli.contains("auto-spawn") - && static_cli.contains("configured/base") - && static_cli.contains("exact")) - { + if !static_cli.contains("hyperdb_state_dir") { + failures.push("static README CLI reference must mention HYPERDB_STATE_DIR".to_owned()); + } + if !(static_cli.contains("socket") && static_cli.contains("named pipe")) { failures.push( - "static README CLI reference must distinguish HYPERDB_DAEMON_PORT auto-spawn discovery from foreground configured/base binding" + "static README CLI reference must describe the per-user socket or named pipe health channel" .to_owned(), ); } diff --git a/hyperdb-mcp/tests/engine_tests.rs b/hyperdb-mcp/tests/engine_tests.rs index ae210b87..ef6dcba5 100644 --- a/hyperdb-mcp/tests/engine_tests.rs +++ b/hyperdb-mcp/tests/engine_tests.rs @@ -363,7 +363,7 @@ fn engine_status() { // Engine connection block. This is a `--no-daemon` engine (TestEngine uses // `Engine::new_no_daemon`), so mode is "local", there's a private hyperd - // endpoint, and no daemon health port. + // endpoint, and no daemon health endpoint. let engine_info = &status["engine"]; assert_eq!(engine_info["mode"], "local"); assert!( @@ -372,8 +372,8 @@ fn engine_status() { engine_info["hyperd_endpoint"] ); assert!( - engine_info["daemon_health_port"].is_null(), - "local engine has no daemon health port" + engine_info["daemon_health_endpoint"].is_null(), + "local engine has no daemon health endpoint" ); // Decomposed connection forms (issue #124). The transport is whatever diff --git a/hyperdb-mcp/tests/health_idle_cost_tests.rs b/hyperdb-mcp/tests/health_idle_cost_tests.rs index 95412863..198cec2c 100644 --- a/hyperdb-mcp/tests/health_idle_cost_tests.rs +++ b/hyperdb-mcp/tests/health_idle_cost_tests.rs @@ -44,6 +44,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use hyperdb_mcp::daemon::control::HealthEndpoint; use hyperdb_mcp::daemon::discovery::DaemonInfo; use hyperdb_mcp::daemon::health::{DaemonState, HealthListener}; @@ -114,29 +115,32 @@ fn timeval_to_duration(value: libc::timeval) -> Duration { Duration::from_secs(seconds) + Duration::from_micros(microseconds) } -fn idle_daemon_info(health_port: u16) -> DaemonInfo { +fn idle_daemon_info(endpoint: &HealthEndpoint) -> DaemonInfo { DaemonInfo { pid: std::process::id(), hyperd_endpoint: "127.0.0.1:0".to_string(), - health_port, + health_endpoint: endpoint.as_str().to_string(), started_at: "2026-09-06T00:00:00Z".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), } } -/// Binds a real listener on an ephemeral port, leaves it completely alone for -/// `IDLE_WINDOW`, and reports what the process paid for the privilege. +/// Binds a real listener on a socket inside a private temporary directory, +/// leaves it completely alone for `IDLE_WINDOW`, and reports what the process +/// paid for the privilege. /// -/// No client ever connects and no discovery record is written — the listener -/// takes an ephemeral port and never touches the state directory — so this +/// No client ever connects and no discovery record is written — the socket +/// lives in a throwaway directory, never the real state directory — so this /// cannot disturb a daemon running on the same machine. #[test] #[ignore = "burns a fixed 5s wall-clock window and reads a process-wide counter; run deliberately"] fn idle_health_listener_costs_about_one_wakeup_per_second() { - let listener = HealthListener::bind(0).expect("bind idle-cost health listener"); - let port = listener.port; - let state = Arc::new(DaemonState::new()); - let info = Arc::new(Mutex::new(idle_daemon_info(port))); + let state_dir = tempfile::tempdir().expect("create idle-cost state dir"); + let endpoint = + HealthEndpoint::for_new_daemon(state_dir.path()).expect("derive idle-cost health endpoint"); + let listener = HealthListener::bind(&endpoint).expect("bind idle-cost health listener"); + let state = Arc::new(DaemonState::new(endpoint.clone())); + let info = Arc::new(Mutex::new(idle_daemon_info(&endpoint))); let run_state = Arc::clone(&state); let server = std::thread::spawn(move || listener.run(run_state, info)); diff --git a/hyperdb-mcp/tests/readme_tests.rs b/hyperdb-mcp/tests/readme_tests.rs index 4ffcf5ff..fa1adb5a 100644 --- a/hyperdb-mcp/tests/readme_tests.rs +++ b/hyperdb-mcp/tests/readme_tests.rs @@ -737,14 +737,25 @@ fn public_docs_database_and_read_only_contract() { .to_owned(), ); } - if !(public_cli.contains("hyperdb_daemon_port") - && public_cli.contains("auto-spawn") - && public_cli.contains("configured/base") - && public_cli.contains("exact") - && contains_any(public_cli, &["pin", "candidate", "discovery"])) - { + for stale in ["--port", "hyperdb_daemon_port"] { + if public_cli.contains(stale) { + failures.push(format!( + "public CLI reference must not mention {stale}: the daemon has no TCP port" + )); + } + } + if !public_cli.contains("hyperdb_state_dir") { + failures.push("public CLI reference must mention HYPERDB_STATE_DIR".to_owned()); + } + if !PUBLIC_README.contains("daemon.sock") { + failures.push( + "README must name the per-user daemon health socket (daemon.sock) in the state directory" + .to_owned(), + ); + } + if !(public_cli.contains("socket") && public_cli.contains("named pipe")) { failures.push( - "public CLI reference must distinguish HYPERDB_DAEMON_PORT auto-spawn discovery from the foreground configured/base exact bind" + "public CLI reference must describe the daemon health channel as a per-user socket or named pipe in the state directory" .to_owned(), ); } diff --git a/hyperdb-mcp/tests/recovery_tests.rs b/hyperdb-mcp/tests/recovery_tests.rs index 0247498d..009f6793 100644 --- a/hyperdb-mcp/tests/recovery_tests.rs +++ b/hyperdb-mcp/tests/recovery_tests.rs @@ -13,8 +13,7 @@ mod common; -use std::io::{BufRead as _, BufReader, Write as _}; -use std::net::TcpListener; +use std::io::{Read as _, Write as _}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Output, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -24,6 +23,7 @@ use std::time::{Duration, Instant}; use common::TestEngine; use hyperdb_api::{HyperProcess, Parameters, TransportMode}; +use hyperdb_mcp::daemon::control::{ControlListener, ControlStream, HealthEndpoint}; use hyperdb_mcp::daemon::discovery::{self, DaemonInfo}; use hyperdb_mcp::daemon::health; use hyperdb_mcp::error::{ErrorCode, is_connection_lost}; @@ -466,23 +466,19 @@ fn run_slow_health_mutex_child() { .expect("parent must provide a Hyper PID report path"); let (hyper, endpoint) = start_reported_hyper_process(&state_dir, &hyper_pid_path); - let listener = - TcpListener::bind(("127.0.0.1", 0)).expect("bind controlled OS-assigned health listener"); - let health_port = listener - .local_addr() - .expect("read controlled health listener address") - .port(); + let health_endpoint = HealthEndpoint::for_new_daemon(&state_dir) + .expect("derive the controlled health endpoint inside the isolated state directory"); + let listener = ControlListener::bind(&health_endpoint).expect("bind controlled health peer"); let daemon_info = DaemonInfo { pid: std::process::id(), hyperd_endpoint: endpoint, - health_port, + health_endpoint: health_endpoint.as_str().to_owned(), started_at: "2026-08-14T00:00:00Z".to_string(), version: hyperdb_mcp::version::MCP_VERSION.to_string(), }; // Discovery is the only routing input: the engine must route - // `REPORT_HYPERD_ERROR` to this effective health port. The child - // intentionally does not mutate the process-global daemon-port setting. + // `REPORT_HYPERD_ERROR` to this recorded health endpoint. discovery::write_discovery_file(&daemon_info).expect("write isolated daemon discovery"); let engine_probe = Arc::new(OnceLock::::new()); @@ -497,7 +493,7 @@ fn run_slow_health_mutex_child() { let peer_worker_finished = Arc::clone(&worker_finished); let peer = thread::spawn(move || { run_controlled_health_peer( - &listener, + listener, &peer_info, &peer_engine_probe, &report_seen_tx, @@ -509,7 +505,7 @@ fn run_slow_health_mutex_child() { // A real protocol round-trip proves the listener is accepting and its // STATUS response is usable; no timing sleep is needed for readiness. let status = health::send_command_with_timeout( - health_port, + &health_endpoint, "STATUS", Duration::from_secs(1), Duration::from_secs(1), @@ -538,9 +534,9 @@ fn run_slow_health_mutex_child() { .expect("inspect warmed engine") .as_ref() .expect("warm-up must install an engine") - .daemon_health_port(), - Some(health_port), - "warmed engine must retain the discovered health port" + .daemon_health_endpoint(), + Some(health_endpoint.as_str()), + "warmed engine must retain the discovered health endpoint" ); hyper @@ -647,7 +643,7 @@ fn run_slow_health_mutex_child() { } let stop_result = health::send_command_with_timeout( - health_port, + &health_endpoint, "STOP", Duration::from_secs(1), Duration::from_secs(1), @@ -714,8 +710,29 @@ fn receive_and_release_report( observation } +/// Read one newline-terminated command a byte at a time. +fn read_controlled_command(stream: &mut ControlStream) -> Result { + let mut request = Vec::new(); + let mut byte = [0_u8]; + loop { + match stream.read(&mut byte) { + Ok(0) => break, + Ok(_) => { + if byte[0] == b'\n' { + break; + } + request.push(byte[0]); + } + Err(error) => return Err(format!("read controlled health command: {error}")), + } + } + String::from_utf8(request) + .map(|command| command.trim().to_string()) + .map_err(|error| format!("controlled health command not UTF-8: {error}")) +} + fn run_controlled_health_peer( - listener: &TcpListener, + mut listener: ControlListener, info: &DaemonInfo, engine_probe: &OnceLock, report_seen_tx: &mpsc::Sender, @@ -724,23 +741,24 @@ fn run_controlled_health_peer( ) -> Result, String> { let mut commands = Vec::new(); let mut report_sequence = 0_usize; + // The peer lives for one scenario; never let a missed STOP hang the test. + let accept_deadline = Instant::now() + Duration::from_secs(60); loop { - let (stream, _) = listener - .accept() - .map_err(|error| format!("accept controlled health connection: {error}"))?; - stream - .set_read_timeout(Some(Duration::from_secs(2))) - .map_err(|error| format!("bound controlled health read: {error}"))?; + let mut stream = match listener + .accept_timeout(Duration::from_millis(50)) + .map_err(|error| format!("accept controlled health connection: {error}"))? + { + Some(stream) => stream, + None if Instant::now() >= accept_deadline => { + return Err("controlled health peer timed out waiting for a command".to_string()); + } + None => continue, + }; stream - .set_write_timeout(Some(Duration::from_secs(2))) - .map_err(|error| format!("bound controlled health write: {error}"))?; - - let mut reader = BufReader::new(&stream); - let mut command = String::new(); - reader - .read_line(&mut command) - .map_err(|error| format!("read controlled health command: {error}"))?; - let command = command.trim().to_string(); + .set_io_timeout(Duration::from_secs(2)) + .map_err(|error| format!("bound controlled health I/O: {error}"))?; + + let command = read_controlled_command(&mut stream)?; commands.push(command.clone()); let (response, should_stop) = match command.as_str() { @@ -808,9 +826,14 @@ fn run_controlled_health_peer( other => (format!("ERR unknown command {other}\n"), false), }; - (&stream) + stream .write_all(response.as_bytes()) .map_err(|error| format!("write controlled health response: {error}"))?; + // Keep the connection open until the client hangs up, as the real + // daemon does: closing first can make the client's next timeout + // update fail (EINVAL) on macOS. + let mut sink = [0_u8; 16]; + while matches!(stream.read(&mut sink), Ok(count) if count > 0) {} if should_stop { return Ok(commands); } From d8f3d0f565d2b50e70bbb1ad860bd118c4dfad6a Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 15:48:59 -0700 Subject: [PATCH 35/61] feat(mcp): retire a pre-1.0 TCP daemon on takeover A client that finds a legacy-shaped discovery record verifies the daemon (identified PING, STATUS pid equals the record pid), stops it, and waits for its port to go quiet before spawning. A mismatched or dead record is removed without sending anything further; an open but unresponsive port keeps the record and the client uses local mode. --- hyperdb-mcp/CHANGELOG.md | 12 + hyperdb-mcp/src/daemon/discovery.rs | 87 +++++++ hyperdb-mcp/src/daemon/health.rs | 8 +- hyperdb-mcp/src/daemon/legacy.rs | 330 +++++++++++++++++++++++++++ hyperdb-mcp/src/daemon/mod.rs | 1 + hyperdb-mcp/src/daemon/spawn.rs | 336 +++++++++++++++++++++++++++- 6 files changed, 767 insertions(+), 7 deletions(-) create mode 100644 hyperdb-mcp/src/daemon/legacy.rs diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index b4ba0d5e..c5a14744 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -39,6 +39,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/). by group or others.** A daemon refuses to start in one, and a client treats it as untrusted and uses local mode rather than connecting to an endpoint it names. +- **A running pre-1.0 (rc.5 and earlier) TCP daemon is stopped and replaced + on first contact.** Before spawning, a new client checks `daemon.json` for the + old `health_port` shape. It sends `STOP` only to a daemon that answers an + identified `PING` and reports the pid the record names, then waits up to 15 s + for the port to go quiet. A record whose port is dead or answered by anything + else is removed with nothing further sent. Only a refused connection counts + as a dead port: if the port is open but unresponsive, or the daemon will not + stop, the record is kept and the client runs in local mode without spawning + (a daemon that ignores `STOP` costs up to 15 s and another `STOP` on every + client start until it exits). An old client meeting a new + daemon cannot parse its record, so it waits out its spawn timeout and runs in + local mode; nothing is destroyed. ### Removed diff --git a/hyperdb-mcp/src/daemon/discovery.rs b/hyperdb-mcp/src/daemon/discovery.rs index 05f145c1..eb37cff6 100644 --- a/hyperdb-mcp/src/daemon/discovery.rs +++ b/hyperdb-mcp/src/daemon/discovery.rs @@ -413,6 +413,28 @@ pub(super) fn remove_stale_record(dir: &Path) { } } +/// The record in `dir`, if it is a pre-1.0 (legacy-shaped) one that names a +/// TCP health port. Reads like doctor does: bounded, and without following a +/// symlink. A current-shape record yields `None`. +pub(super) fn read_legacy_record(dir: &Path) -> Option { + match read_discovery_file_raw(&dir.join("daemon.json")) { + RawDiscoveryRead::Malformed { contents, .. } => { + super::legacy::LegacyRecord::parse(contents.as_slice()) + } + _ => None, + } +} + +/// Remove the legacy record `expected`, if `dir` still holds exactly it. +/// +/// The caller must hold the [`DaemonLock`](super::lock::DaemonLock) for `dir`. +/// The re-read keeps a record that changed since it was judged. +pub(super) fn remove_legacy_record(dir: &Path, expected: super::legacy::LegacyRecord) { + if read_legacy_record(dir) == Some(expected) { + let _ = std::fs::remove_file(dir.join("daemon.json")); + } +} + /// Remove the discovery file (called during graceful shutdown). pub fn remove_discovery_file() { if let Ok(path) = discovery_file_path() { @@ -1411,4 +1433,69 @@ mod tests { } run_discovery_compatibility_child(TEST_NAME, CHILD_SENTINEL_ENV); } + + // ─── Legacy record hook ────────────────────────────────────────────── + + fn write_legacy_file(dir: &Path, pid: u32) -> PathBuf { + let path = dir.join("daemon.json"); + let record = json!({ + "pid": pid, + "hyperd_endpoint": "127.0.0.1:54321", + "health_port": 7485, + "started_at": "2026-08-13T12:34:56Z", + "version": "0.7.0", + }); + std::fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap(); + path + } + + #[test] + fn remove_legacy_record_removes_exactly_the_judged_record() { + let tmp = TempDir::new().unwrap(); + let path = write_legacy_file(tmp.path(), 4_242); + let judged = read_legacy_record(tmp.path()).expect("legacy record"); + assert_eq!(judged.pid, 4_242); + + remove_legacy_record(tmp.path(), judged); + assert!(!path.exists()); + } + + #[test] + fn remove_legacy_record_keeps_a_record_that_changed_since_it_was_read() { + let tmp = TempDir::new().unwrap(); + write_legacy_file(tmp.path(), 4_242); + let judged = read_legacy_record(tmp.path()).expect("legacy record"); + let path = write_legacy_file(tmp.path(), 9_999); + + remove_legacy_record(tmp.path(), judged); + assert!(path.exists(), "a changed record must survive"); + } + + #[test] + fn read_legacy_record_ignores_a_current_shape_record() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("daemon.json"); + std::fs::write( + &path, + json!({"pid": 1, "hyperd_endpoint": "x", "health_endpoint": "y", + "started_at": "t", "version": "1.0.0"}) + .to_string(), + ) + .unwrap(); + assert_eq!(read_legacy_record(tmp.path()), None); + } + + #[cfg(unix)] + #[test] + fn legacy_hooks_do_not_follow_a_symlinked_daemon_json() { + let tmp = TempDir::new().unwrap(); + let target_dir = TempDir::new().unwrap(); + let target = write_legacy_file(target_dir.path(), 4_242); + std::os::unix::fs::symlink(&target, tmp.path().join("daemon.json")).unwrap(); + + assert_eq!(read_legacy_record(tmp.path()), None); + let judged = read_legacy_record(target_dir.path()).unwrap(); + remove_legacy_record(tmp.path(), judged); + assert!(target.exists(), "the symlink target must be left alone"); + } } diff --git a/hyperdb-mcp/src/daemon/health.rs b/hyperdb-mcp/src/daemon/health.rs index ff15fa0e..b31271ca 100644 --- a/hyperdb-mcp/src/daemon/health.rs +++ b/hyperdb-mcp/src/daemon/health.rs @@ -929,16 +929,22 @@ mod tests { /// reply must still be readable. #[test] fn rearming_the_timeout_after_the_peer_closed_keeps_the_reply_readable() { - let peer = spawn_test_peer(|mut stream, _stop| { + let (closed_tx, closed_rx) = mpsc::channel::<()>(); + let peer = spawn_test_peer(move |mut stream, _stop| { read_test_command(&mut stream)?; stream .write_all(b"OK\n") .map_err(|error| format!("write reply: {error}"))?; drop(stream); + let _ = closed_tx.send(()); Ok(()) }); let mut client = control::connect(&peer.endpoint, Duration::from_secs(1)).unwrap(); client.write_all(b"X\n").unwrap(); + // Finishing the peer earlier can stop it before it accepts. + closed_rx + .recv_timeout(Duration::from_secs(5)) + .expect("peer must reply and close"); peer.finish().expect("peer must finish"); std::thread::sleep(Duration::from_millis(100)); diff --git a/hyperdb-mcp/src/daemon/legacy.rs b/hyperdb-mcp/src/daemon/legacy.rs new file mode 100644 index 00000000..ac8461e5 --- /dev/null +++ b/hyperdb-mcp/src/daemon/legacy.rs @@ -0,0 +1,330 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Retiring a pre-1.0 daemon (rc.5 and earlier). Marked for removal after 1.x. +//! +//! Those daemons publish `health_port` in `daemon.json`, serve the health +//! protocol on `127.0.0.1:` over TCP, and hold no daemon lock. A new +//! client that simply spawned would get the lock and then fail its `hyperd` +//! socket pre-flight against the old engine, leaving every client in local mode +//! until the old daemon exits, which by default is never. So, under the daemon +//! lock, the client asks the old daemon to stop and waits for its port to go +//! quiet. +//! +//! The old daemon is only sent `STOP` after it has proved to be the one the +//! record names: an identified `PING`, then a `STATUS` whose pid equals the +//! record's. Anything else is a stale record pointing at a port somebody else +//! may now hold, and nothing further is sent to it. +//! +//! Only a refused connection means "nothing is listening". A timeout, reset or +//! partial reply from an open port is "unknown": the record is kept, nothing +//! more is sent and the client runs in local mode without spawning. A prompt, +//! complete reply that is not an identified `PONG` is a foreign service, so the +//! record is stale. +//! +//! Caveats, accepted: a legacy daemon that ignores `STOP` costs up to +//! [`STOP_WAIT`] and another `STOP` on every client start until it exits; the +//! old daemon removes its own record during its shutdown, which can race this +//! module's removal (harmless, both only delete the same legacy file); each +//! command uses its own connection; a probe costs about a second at worst; and +//! a state directory that is not trusted never reaches this code (the client +//! uses local mode without retiring anything). +//! +//! This module uses only `std` (plus serde/tracing), so it compiles unchanged on +//! every platform. + +use std::io::{self, Read, Write}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream}; +use std::path::Path; +use std::time::{Duration, Instant}; + +use serde::Deserialize; +use tracing::{info, warn}; + +use super::discovery; +use super::health::PONG_TOKEN; + +/// How long to wait, after `STOP`, for the old daemon's port to stop answering. +pub(super) const STOP_WAIT: Duration = Duration::from_secs(15); + +/// Budget for one probe (connect, or the whole request/reply exchange). +const PROBE_TIMEOUT: Duration = Duration::from_millis(500); + +/// Polling interval while waiting for the port to go quiet. +const POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Longest reply read from a legacy port. A `STATUS` line is far smaller; the +/// cap keeps a foreign listener that streams bytes from growing the buffer. +const MAX_REPLY_BYTES: usize = 16 * 1024; + +/// The two fields of a legacy `daemon.json` this module acts on. Unknown +/// fields are ignored, so an `identity` block written by a late rc is accepted. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +pub(super) struct LegacyRecord { + pub(super) pid: u32, + pub(super) health_port: u16, +} + +impl LegacyRecord { + /// Parse a legacy-shaped record. A record that carries `health_endpoint` + /// is the current shape and is never a legacy record, whatever else it has. + pub(super) fn parse(bytes: &[u8]) -> Option { + let value: serde_json::Value = serde_json::from_slice(bytes).ok()?; + if value.get("health_endpoint").is_some() { + return None; + } + let record: Self = serde_json::from_value(value).ok()?; + (record.health_port != 0).then_some(record) + } +} + +/// How a legacy record resolved. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Retirement { + /// The port refuses connections, or what answers there is definitely not + /// the recorded daemon: the record is stale and nothing was sent beyond + /// `PING`/`STATUS`. + Stale, + /// The recorded daemon was told to stop and its port went quiet. + Retired, + /// The port is open but did not answer in a way that identifies the + /// recorded daemon. The record is kept and nothing more was sent. + Unresponsive, + /// The recorded daemon was told to stop and still answered after the wait. + StillRunning, +} + +/// What one probe of a legacy port found. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Probe { + /// The connection was refused: nothing listens on the port. + Refused, + /// A complete (newline-terminated) reply line. + Line(String), + /// Anything else: connect or read timeout, reset, EOF or a reply with no + /// line end, an oversized reply. Something may well be listening, so this + /// is never treated as "dead". + Unknown, +} + +/// Retire a legacy daemon named by `dir`'s record, if there is one. +/// +/// The caller must hold the daemon lock for `dir`. On `Ok` no legacy daemon is +/// running and any legacy record has been removed, so the caller may spawn. The +/// record is removed only here, under the lock, so it can never later delete a +/// new daemon's record. +/// +/// # Errors +/// Returns [`io::ErrorKind::TimedOut`] if the port is open but did not identify +/// the recorded daemon, or the daemon is still answering after `stop_wait`. +/// The record is kept and the caller must not spawn. +pub(super) fn retire_under_lock(dir: &Path, stop_wait: Duration) -> io::Result<()> { + let Some(record) = discovery::read_legacy_record(dir) else { + return Ok(()); + }; + match retire(record, stop_wait) { + Retirement::Stale => { + info!( + port = record.health_port, + "removing a stale pre-1.0 daemon record" + ); + } + Retirement::Retired => { + info!(pid = record.pid, "retired a pre-1.0 daemon"); + } + Retirement::Unresponsive => { + warn!( + port = record.health_port, + "pre-1.0 daemon port is open but did not identify the recorded daemon" + ); + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "the pre-1.0 daemon record names port {}, which is open but did not \ + identify the recorded daemon (pid {}); nothing was sent to it", + record.health_port, record.pid + ), + )); + } + Retirement::StillRunning => { + warn!( + pid = record.pid, + port = record.health_port, + "pre-1.0 daemon did not stop within the timeout" + ); + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "a pre-1.0 daemon (pid {}) is still running on port {} and did not stop \ + within {} seconds; stop it manually or wait for it to exit", + record.pid, + record.health_port, + stop_wait.as_secs() + ), + )); + } + } + discovery::remove_legacy_record(dir, record); + Ok(()) +} + +/// Only a refused connection means "nothing there". A timeout, reset or +/// garbled reply from an open port is "unknown" and keeps the record; a +/// prompt, complete reply that is not an identified `PONG` is a foreign +/// service, so the record is stale. +fn retire(record: LegacyRecord, stop_wait: Duration) -> Retirement { + let port = record.health_port; + match classify_ping(port) { + Ping::Identified => {} + Ping::Dead | Ping::Foreign => return Retirement::Stale, + Ping::Unknown => return Retirement::Unresponsive, + } + // Separate connections for each command: the old daemon may close after + // every reply. + match status_pid(port) { + StatusPid::Pid(pid) if pid == record.pid => {} + StatusPid::Pid(_) | StatusPid::Refused => return Retirement::Stale, + StatusPid::Unknown => return Retirement::Unresponsive, + } + + // Best effort: an error here means the daemon may already be going away, + // which the wait below settles. + let _ = exchange(port, "STOP"); + + // The old daemon removes its own record before it releases the port, so a + // quiet port also means its record is gone (or ours to remove). + let deadline = Instant::now() + stop_wait; + loop { + match classify_ping(port) { + Ping::Dead | Ping::Foreign => return Retirement::Retired, + Ping::Identified | Ping::Unknown => {} + } + if Instant::now() >= deadline { + return Retirement::StillRunning; + } + std::thread::sleep(POLL_INTERVAL); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ping { + /// A complete `PONG hyperdb-mcp ...` line. + Identified, + /// A complete reply line that is anything else. + Foreign, + /// The connection was refused. + Dead, + /// No usable answer from a port that may be open. + Unknown, +} + +fn classify_ping(port: u16) -> Ping { + match exchange(port, "PING") { + Probe::Refused => Ping::Dead, + Probe::Unknown => Ping::Unknown, + Probe::Line(line) => { + let mut tokens = line.split_whitespace(); + if tokens.next() == Some("PONG") && tokens.next() == Some(PONG_TOKEN) { + Ping::Identified + } else { + Ping::Foreign + } + } + } +} + +enum StatusPid { + Pid(u32), + Refused, + Unknown, +} + +/// The pid a legacy daemon reports for itself in `STATUS`. +fn status_pid(port: u16) -> StatusPid { + match exchange(port, "STATUS") { + Probe::Refused => StatusPid::Refused, + Probe::Unknown => StatusPid::Unknown, + Probe::Line(line) => serde_json::from_str::(line.trim()) + .ok() + .and_then(|value| u32::try_from(value.get("pid")?.as_u64()?).ok()) + .map_or(StatusPid::Unknown, StatusPid::Pid), + } +} + +/// Send one command on a fresh connection to `127.0.0.1:port` and return the +/// first reply line. Connecting and the exchange each take at most +/// [`PROBE_TIMEOUT`], so one probe costs about a second at worst. +fn exchange(port: u16, command: &str) -> Probe { + let address = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); + let mut stream = match TcpStream::connect_timeout(&address, PROBE_TIMEOUT) { + Ok(stream) => stream, + Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => return Probe::Refused, + Err(_) => return Probe::Unknown, + }; + let deadline = Instant::now() + PROBE_TIMEOUT; + if stream.set_write_timeout(Some(PROBE_TIMEOUT)).is_err() + || stream.write_all(command.as_bytes()).is_err() + || stream.write_all(b"\n").is_err() + { + return Probe::Unknown; + } + + let mut reply = Vec::new(); + let mut chunk = [0_u8; 512]; + loop { + if let Some(end) = reply.iter().position(|byte| *byte == b'\n') { + return Probe::Line(String::from_utf8_lossy(&reply[..end]).into_owned()); + } + let Some(remaining) = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + else { + return Probe::Unknown; + }; + match stream.set_read_timeout(Some(remaining)) { + // macOS rejects a timeout on a socket the peer already closed + // (EINVAL); the read below then reports the close. + Err(error) if error.kind() != io::ErrorKind::InvalidInput => return Probe::Unknown, + _ => {} + } + match stream.read(&mut chunk) { + // A close before a full line is a garbled reply. + Ok(0) | Err(_) => return Probe::Unknown, + Ok(read) => reply.extend_from_slice(&chunk[..read]), + } + if reply.len() > MAX_REPLY_BYTES { + return Probe::Unknown; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_legacy_record_and_ignores_unknown_fields() { + let bytes = br#"{"pid":42,"hyperd_endpoint":"127.0.0.1:1","health_port":7485, + "started_at":"t","version":"0.7.0","identity":{"anything":1}}"#; + assert_eq!( + LegacyRecord::parse(bytes), + Some(LegacyRecord { + pid: 42, + health_port: 7485 + }) + ); + } + + #[test] + fn rejects_the_current_shape_and_unusable_records() { + let current = br#"{"pid":42,"health_endpoint":"/x/daemon.sock","health_port":7485}"#; + assert_eq!(LegacyRecord::parse(current), None); + assert_eq!(LegacyRecord::parse(br#"{"pid":42,"health_port":0}"#), None); + assert_eq!(LegacyRecord::parse(br#"{"pid":42}"#), None); + assert_eq!(LegacyRecord::parse(b"not json"), None); + assert_eq!( + LegacyRecord::parse(br#"{"pid":42,"health_port":70000}"#), + None + ); + } +} diff --git a/hyperdb-mcp/src/daemon/mod.rs b/hyperdb-mcp/src/daemon/mod.rs index a0ac1340..6da37609 100644 --- a/hyperdb-mcp/src/daemon/mod.rs +++ b/hyperdb-mcp/src/daemon/mod.rs @@ -11,6 +11,7 @@ pub mod control; pub mod discovery; pub mod health; +mod legacy; pub mod lock; pub mod run; pub mod spawn; diff --git a/hyperdb-mcp/src/daemon/spawn.rs b/hyperdb-mcp/src/daemon/spawn.rs index 1c71202f..3d101bca 100644 --- a/hyperdb-mcp/src/daemon/spawn.rs +++ b/hyperdb-mcp/src/daemon/spawn.rs @@ -17,6 +17,7 @@ use std::path::Path; use super::control::HealthEndpoint; use super::discovery::{self, DaemonInfo}; +use super::legacy; use super::lock::DaemonLock; /// Maximum time to wait for the daemon to write its discovery file after spawning. @@ -39,7 +40,7 @@ const POLL_INTERVAL: Duration = Duration::from_millis(100); /// period. pub fn ensure_daemon() -> io::Result { let dir = discovery::state_dir()?; - ensure_daemon_in(&dir, spawn_detached, SPAWN_TIMEOUT) + ensure_daemon_in(&dir, spawn_detached, SPAWN_TIMEOUT, legacy::STOP_WAIT) } /// [`ensure_daemon`] for an explicit state directory, spawner and wait budget. @@ -47,6 +48,7 @@ fn ensure_daemon_in( dir: &Path, spawn: impl FnOnce() -> io::Result<()>, timeout: Duration, + legacy_stop_wait: Duration, ) -> io::Result { match super::state_perms::verify_state_dir_trusted(dir) { Ok(()) => {} @@ -68,9 +70,15 @@ fn ensure_daemon_in( Err(error) => return Err(error), }; if let Some(lock) = lock { - // Nothing holds the lock, so no daemon is running and any record - // left behind is stale. Clean it up while we hold the lock so a - // daemon cannot start between the check and the removal. + // Nothing holds the lock, so no daemon of this release is running + // and any record left behind is stale. Clean it up while we hold + // the lock so a daemon cannot start between the check and the + // removal. + // + // A pre-1.0 daemon holds no lock, so it is retired first; if it + // will not stop, return without spawning (the caller runs in + // local mode). + legacy::retire_under_lock(dir, legacy_stop_wait)?; discovery::remove_stale_record(dir); drop(lock); info!("no running daemon detected, spawning one"); @@ -304,6 +312,7 @@ mod tests { dir.path(), counting_spawner(&calls), Duration::from_millis(300), + NO_LEGACY_WAIT, ) .expect_err("a held lock with no daemon must not succeed"); @@ -326,6 +335,7 @@ mod tests { dir.path(), counting_spawner(&calls), Duration::from_millis(200), + NO_LEGACY_WAIT, ) .expect_err("the fake spawner publishes no daemon"); @@ -341,10 +351,324 @@ mod tests { let dir = parent.path().join("never-created"); let calls = Cell::new(0); - let error = ensure_daemon_in(&dir, counting_spawner(&calls), Duration::from_millis(200)) - .expect_err("the fake spawner publishes no daemon"); + let error = ensure_daemon_in( + &dir, + counting_spawner(&calls), + Duration::from_millis(200), + NO_LEGACY_WAIT, + ) + .expect_err("the fake spawner publishes no daemon"); assert_eq!(error.kind(), io::ErrorKind::TimedOut); assert_eq!(calls.get(), 1, "the spawner must run once"); } + + /// Legacy wait for tests that have no legacy daemon. + const NO_LEGACY_WAIT: Duration = Duration::from_millis(100); + + // ─── Retiring a pre-1.0 TCP daemon ────────────────────────────────── + + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::JoinHandle; + + /// What the fake legacy daemon does on `STOP`. + #[derive(Clone, Copy)] + enum OnStop { + /// Release the port, as an rc.5 daemon does when it shuts down. + ClosePort, + /// Keep answering forever. + KeepAnswering, + /// Keep the port open but stop replying (accept, then stay silent). + GoSilent, + } + + /// How the fake behaves from the first connection. + #[derive(Clone, Copy)] + enum Mode { + /// An rc.5 daemon reporting this pid. + Daemon(u32, OnStop), + /// Accepts, reads the command, never replies. + Silent, + /// A foreign service: answers every command with something else. + Foreign, + } + + /// An rc.5 look-alike on `127.0.0.1:0`: answers `PING`, `STATUS` and + /// `STOP`, and records every command it receives. + struct FakeLegacyDaemon { + port: u16, + received: Arc>>, + quit: Arc, + handle: Option>, + } + + impl FakeLegacyDaemon { + fn start(mode: Mode) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind fake legacy daemon"); + let port = listener.local_addr().expect("local addr").port(); + listener.set_nonblocking(true).expect("nonblocking"); + let received = Arc::new(Mutex::new(Vec::new())); + let quit = Arc::new(AtomicBool::new(false)); + let handle = { + let received = Arc::clone(&received); + let quit = Arc::clone(&quit); + std::thread::spawn(move || { + let mut silent = matches!(mode, Mode::Silent); + while !quit.load(Ordering::SeqCst) { + let Ok((stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(5)); + continue; + }; + stream.set_nonblocking(false).expect("blocking stream"); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("read timeout"); + let mut line = String::new(); + let mut reader = BufReader::new(stream); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + continue; + } + let command = line.trim().to_string(); + received.lock().expect("received").push(command.clone()); + let (reported_pid, on_stop) = match mode { + Mode::Daemon(pid, on_stop) => (pid, Some(on_stop)), + Mode::Silent | Mode::Foreign => (0, None), + }; + if silent { + // Hold the connection open without replying. + for _ in 0..70 { + if quit.load(Ordering::SeqCst) { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + continue; + } + let reply = match (mode, command.as_str()) { + (Mode::Foreign, _) => "SSH-2.0-OpenSSH_9.0\n".to_string(), + (_, "PING") => "PONG hyperdb-mcp 0.7.0\n".to_string(), + (_, "STATUS") => format!("{{\"pid\":{reported_pid}}}\n"), + (_, "STOP") => "STOPPING\n".to_string(), + _ => "ERR unknown command\n".to_string(), + }; + let _ = reader.get_mut().write_all(reply.as_bytes()); + if command == "STOP" { + match on_stop { + Some(OnStop::ClosePort) => break, + Some(OnStop::GoSilent) => silent = true, + _ => {} + } + } + } + // Dropping the listener releases the port. + }) + }; + Self { + port, + received, + quit, + handle: Some(handle), + } + } + + fn received(&self) -> Vec { + self.received.lock().expect("received").clone() + } + } + + impl Drop for FakeLegacyDaemon { + fn drop(&mut self) { + self.quit.store(true, Ordering::SeqCst); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } + } + + fn write_legacy_record(dir: &Path, pid: u32, port: u16) -> std::path::PathBuf { + let path = dir.join("daemon.json"); + let record = serde_json::json!({ + "pid": pid, + "hyperd_endpoint": "127.0.0.1:54321", + "health_port": port, + "started_at": "2026-08-13T12:34:56Z", + "version": "0.7.0", + }); + std::fs::write(&path, serde_json::to_vec(&record).expect("serialize")).expect("write"); + path + } + + fn stops(received: &[String]) -> usize { + received.iter().filter(|command| *command == "STOP").count() + } + + /// The recorded daemon proves its pid, is stopped, its port goes quiet, + /// the legacy record is removed, and the new daemon is spawned. + #[test] + fn a_legacy_daemon_is_retired_and_the_record_removed() { + let dir = tempfile::tempdir().expect("state dir"); + let peer = FakeLegacyDaemon::start(Mode::Daemon(4_242, OnStop::ClosePort)); + let record = write_legacy_record(dir.path(), 4_242, peer.port); + let calls = Cell::new(0); + + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_secs(5), + ) + .expect_err("the fake spawner publishes no daemon"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 1, "a retired legacy daemon is replaced"); + assert!(!record.exists(), "the legacy record must be gone"); + let received = peer.received(); + assert_eq!(&received[..3], ["PING", "STATUS", "STOP"], "{received:?}"); + assert_eq!(stops(&received), 1); + } + + /// A pid that does not match the record means the port belongs to + /// someone else: no `STOP` is sent, the stale record is removed. + #[test] + fn a_legacy_record_with_a_different_pid_gets_no_stop() { + let dir = tempfile::tempdir().expect("state dir"); + let peer = FakeLegacyDaemon::start(Mode::Daemon(9_999, OnStop::ClosePort)); + let record = write_legacy_record(dir.path(), 4_242, peer.port); + let calls = Cell::new(0); + + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_secs(5), + ) + .expect_err("the fake spawner publishes no daemon"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 1); + assert!(!record.exists(), "the stale legacy record must be removed"); + let received = peer.received(); + assert_eq!(received, ["PING", "STATUS"], "nothing beyond STATUS"); + } + + /// A legacy record whose port nobody serves is stale: removed, then spawn. + #[test] + fn a_legacy_record_with_a_dead_port_is_removed_and_spawns() { + let dir = tempfile::tempdir().expect("state dir"); + let port = { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + listener.local_addr().expect("addr").port() + }; + let record = write_legacy_record(dir.path(), 4_242, port); + let calls = Cell::new(0); + + let _ = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_secs(5), + ); + + assert_eq!(calls.get(), 1); + assert!(!record.exists()); + } + + /// A legacy daemon that keeps answering after `STOP`: wait the injected + /// time, never spawn, leave its record, and report local mode's cause. + #[test] + fn a_legacy_daemon_that_never_stops_means_no_spawn() { + let dir = tempfile::tempdir().expect("state dir"); + let peer = FakeLegacyDaemon::start(Mode::Daemon(4_242, OnStop::KeepAnswering)); + let record = write_legacy_record(dir.path(), 4_242, peer.port); + let calls = Cell::new(0); + + let started = Instant::now(); + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_millis(400), + ) + .expect_err("a legacy daemon that will not stop must not succeed"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(error.to_string().contains("pre-1.0"), "{error}"); + assert_eq!(calls.get(), 0, "the spawner must never run"); + assert!(record.exists(), "the running daemon's record is kept"); + assert!(started.elapsed() >= Duration::from_millis(400)); + assert!(started.elapsed() < Duration::from_secs(5)); + assert_eq!(stops(&peer.received()), 1, "STOP is sent once"); + } + + /// An open port that never answers is not proof of a dead daemon: keep + /// the record, send nothing, do not spawn. + #[test] + fn a_legacy_port_that_never_replies_keeps_the_record_and_never_spawns() { + let dir = tempfile::tempdir().expect("state dir"); + let peer = FakeLegacyDaemon::start(Mode::Silent); + let record = write_legacy_record(dir.path(), 4_242, peer.port); + let calls = Cell::new(0); + + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_millis(400), + ) + .expect_err("an unresponsive legacy port must not succeed"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(error.to_string().contains("pre-1.0"), "{error}"); + assert_eq!(calls.get(), 0, "the spawner must never run"); + assert!(record.exists(), "the record must be kept"); + let received = peer.received(); + assert_eq!(received, ["PING"], "nothing beyond the PING"); + } + + /// A foreign service answering something else holds the port: the record + /// is stale, removed, and only the PING was sent. + #[test] + fn a_foreign_service_on_the_legacy_port_gets_only_a_ping() { + let dir = tempfile::tempdir().expect("state dir"); + let peer = FakeLegacyDaemon::start(Mode::Foreign); + let record = write_legacy_record(dir.path(), 4_242, peer.port); + let calls = Cell::new(0); + + let _ = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_secs(5), + ); + + assert_eq!(calls.get(), 1); + assert!(!record.exists(), "the stale record must be removed"); + assert_eq!(peer.received(), ["PING"]); + } + + /// After `STOP`, a port that hangs (open, silent) still counts as alive. + #[test] + fn a_legacy_port_that_hangs_after_stop_counts_as_alive() { + let dir = tempfile::tempdir().expect("state dir"); + let peer = FakeLegacyDaemon::start(Mode::Daemon(4_242, OnStop::GoSilent)); + let record = write_legacy_record(dir.path(), 4_242, peer.port); + let calls = Cell::new(0); + + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_millis(400), + ) + .expect_err("a hung legacy daemon must not succeed"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(error.to_string().contains("still running"), "{error}"); + assert_eq!(calls.get(), 0, "the spawner must never run"); + assert!(record.exists()); + assert_eq!(stops(&peer.received()), 1); + } } From dd8daf7cb6263a92e7b66dba94d99afc37a2a10b Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 16:14:47 -0700 Subject: [PATCH 36/61] fix(mcp): version takeover starts the new daemon reliably STOP now wakes the daemon immediately, and takeover and `daemon stop` wait for the old daemon to release its lock before continuing. A takeover that times out falls back to local mode instead of reusing the dying daemon. Daemon failures are now logged. --- .../plans/2026-10-10-mcp-security.md | 4 +- .../specs/2026-10-10-mcp-security-design.md | 5 +- hyperdb-mcp/CHANGELOG.md | 15 + hyperdb-mcp/Cargo.toml | 2 +- hyperdb-mcp/DEVELOPMENT.md | 22 +- hyperdb-mcp/README.md | 2 +- hyperdb-mcp/src/daemon/health.rs | 13 + hyperdb-mcp/src/daemon/legacy.rs | 5 +- hyperdb-mcp/src/daemon/run.rs | 29 +- hyperdb-mcp/src/daemon/spawn.rs | 284 +++++++++++++++--- hyperdb-mcp/src/engine.rs | 2 +- hyperdb-mcp/src/main.rs | 26 +- hyperdb-mcp/tests/daemon_tests.rs | 240 ++++++++++++++- 13 files changed, 571 insertions(+), 78 deletions(-) diff --git a/docs/superpowers/plans/2026-10-10-mcp-security.md b/docs/superpowers/plans/2026-10-10-mcp-security.md index d7f6416f..ed180d36 100644 --- a/docs/superpowers/plans/2026-10-10-mcp-security.md +++ b/docs/superpowers/plans/2026-10-10-mcp-security.md @@ -82,7 +82,7 @@ Roughly 120 references change (daemon_tests ~48, diagnostics ~30, spawn ~15, eng 1. `state_perms.rs`: the ownership and no-group/other-write check (spec section 3, State directory), used by the daemon and the client. 2. `health.rs`: `HealthListener` wraps `ControlListener`; `DaemonState` stores the endpoint at construction plus a SeqCst `AtomicBool` registered flag (spec); `send_command`, `send_command_with_timeout`, `ping_identified`, `report_hyperd_error_to_daemon` take `&HealthEndpoint`. Keep the absolute I/O deadline and the 64 KiB response cap. Rewrite the module doc's security claims to what is now true. 3. `discovery.rs`: `DaemonInfo.health_endpoint: String`; delete the scan machinery listed in the spec; `discover()` = record + identified `PING`, nothing else. -4. `run.rs`: create and check the state dir, then acquire `DaemonLock` with the `STARTUP_WAIT` retry and the debounced "already running" check (spec section 4, point 3; the pre-flight joins the same budget), bind the control endpoint, start hyperd, write the record. Hold the lock until after `drop(hyper_state)` and socket removal. `DaemonConfig` loses `port`. +4. `run.rs`: create and check the state dir, then acquire `DaemonLock` with the `STARTUP_WAIT` retry and the debounced "already running" check (spec section 4, point 3; the pre-flight joins the same budget), bind the control endpoint, start hyperd, write the record. Hold the lock until after `drop(hyper_state)`. `DaemonConfig` loses `port`. 5. `spawn.rs`: `ensure_daemon()` per spec (no scan; held lock → wait, free lock → stale cleanup then spawn); `spawn_detached()` without `--port`. 6. `main.rs`: drop `--port` from `daemon`, `daemon stop`, `daemon status`; status prints "Health endpoint:"; rewrite the `daemon --help` text. `engine.rs`/`server.rs`: `daemon_health_endpoint`, status JSON key `daemon_health_endpoint`. `diagnostics.rs`: `health_endpoint`, drop `LiveFromScan` and the scan request/candidate types, `StatusHealthPortMismatch` → endpoint mismatch, report the lock state; still side-effect free. 7. Tests: migrate every TCP fake peer (`CliStatusListener`, `run_controlled_health_peer`, `RunningHealthListener`, `HeldHeartbeatPeer`, health.rs `TestPeer`, …) to `ControlListener` on a per-test endpoint; `TestDaemon` (`daemon_tests.rs` ~2008-2103) drops `HYPERDB_DAEMON_PORT` / `port: 0` and its `Drop` waits for the lock to be free; `DoctorSandbox` drops `HYPERDB_DAEMON_PORT` and gets a state dir short enough for `sun_path` (today 93 bytes + `/daemon.sock` = 105 > 103). Delete the port CLI tests (`daemon_tests.rs` ~600-678, ~699-764, the base-port assertion ~887-889) and `doctor_cli_reports_live_from_scan_via_real_health_listener` (`doctor_tests.rs` ~2060-2114); replace `health_listener_second_bind_same_port_fails` with a lock-based single-instance test. New tests: state dir owned by someone else or group-writable → local mode (Unix; group-writable is testable without root); a client that finds the lock held and no record waits and does not spawn. @@ -104,7 +104,7 @@ Commit: `feat(mcp): retire a pre-1.0 TCP daemon on takeover`. **Files:** `daemon/run.rs`, `daemon/health.rs`, `daemon/spawn.rs`, `daemon/mod.rs`, `main.rs`, `engine.rs`, `daemon_tests.rs`, README.md / DEVELOPMENT.md (takeover wording), CHANGELOG.md. -1. `DaemonState` gains `Notify`; `request_shutdown` calls `notify_one()`; `run_daemon`'s `select!` gains the branch. Shutdown order: record removed → listener joined → hyperd dropped → socket removed → lock dropped. +1. `DaemonState` gains `Notify`; `request_shutdown` calls `notify_one()`; `run_daemon`'s `select!` gains the branch. Shutdown order: record removed → listener joined (removes the health socket) → hyperd dropped → lock dropped. 2. `maybe_take_over` and `daemon stop`: send `STOP`, wait until the lock is free (`STOP_WAIT` 15 s), then spawn / report; on timeout local mode / a clear CLI error. 3. `SPAWN_TIMEOUT` 20 s; `STARTUP_WAIT` 10 s (already used since iteration 4; confirm the value). 4. `main` logs a `run_daemon` error with `tracing::error!`; the engine's local-mode fallback logs at `warn`. diff --git a/docs/superpowers/specs/2026-10-10-mcp-security-design.md b/docs/superpowers/specs/2026-10-10-mcp-security-design.md index ab780817..0b4b99e8 100644 --- a/docs/superpowers/specs/2026-10-10-mcp-security-design.md +++ b/docs/superpowers/specs/2026-10-10-mcp-security-design.md @@ -406,8 +406,9 @@ different ports is wrong. `request_shutdown` calls `notify_one()` (which stores a permit, so a STOP that lands before the branch is polled is not lost). `run_daemon`'s `select!` gets a branch awaiting it. Shutdown then runs immediately: - remove the record, join the listener, drop hyperd (`HyperProcess::drop`, - up to about 5.1 s), remove the socket file, and only then drop the lock. + remove the record, join the listener (which removes the health socket), + drop hyperd (`HyperProcess::drop`, up to about 5.1 s), and only then drop + the lock. 2. **Whoever sends STOP waits for the lock.** `maybe_take_over` and `hyperdb-mcp daemon stop` send `STOP` and then wait until the lock is free (the old daemon and its hyperd are gone), bounded by `STOP_WAIT` = 15 s. diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index c5a14744..57486176 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -64,6 +64,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- **Version takeover starts the new daemon reliably.** A newer client used to + spawn the replacement as soon as the old daemon's health channel went quiet, + while the old `hyperd` still held its socket; the new daemon then died + silently and clients ran in local mode with no daemon at all. The old daemon + now shuts down immediately on `STOP` instead of at its next 5 s poll, the + takeover waits until the old daemon has released the daemon lock, and + `SPAWN_TIMEOUT` rises from 10 s to 20 s. Behaviour change: a takeover that + times out (15 s) waiting for the old daemon to exit now falls back to local + mode instead of reusing the dying daemon. +- **`hyperdb-mcp daemon stop` waits for the daemon to exit**, and reports an + error if it is still running 15 s after acknowledging `STOP`, instead of + returning as soon as the daemon replied. +- **A daemon that fails to start now says why.** The reason is logged to + `hyperdb-daemon.log` (a spawned daemon's stderr is discarded), and a client + falling back to local mode logs it at `warn` instead of `debug`. - **`query_data`, `query_file` and `export` now accept only read-only SQL**, with or without `--read-only`. They used to run any statement, so `query_data(sql: "DROP TABLE persistent.public.t")` dropped a persistent diff --git a/hyperdb-mcp/Cargo.toml b/hyperdb-mcp/Cargo.toml index de10e30e..ce25954a 100644 --- a/hyperdb-mcp/Cargo.toml +++ b/hyperdb-mcp/Cargo.toml @@ -24,7 +24,7 @@ path = "src/main.rs" hyperdb-api = { path = "../hyperdb-api", version = "=1.0.0-rc.5" } # x-release-please-end rmcp = { version = "3.4", features = ["server", "transport-io"] } -tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-std", "signal", "time"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-std", "signal", "sync", "time"] } serde = { workspace = true } serde_json = { workspace = true, features = ["preserve_order"] } semver = "1" diff --git a/hyperdb-mcp/DEVELOPMENT.md b/hyperdb-mcp/DEVELOPMENT.md index 732a8826..8b5670dd 100644 --- a/hyperdb-mcp/DEVELOPMENT.md +++ b/hyperdb-mcp/DEVELOPMENT.md @@ -271,11 +271,23 @@ it spawns only when the lock is free, after removing any stale record. **Version takeover.** When discovery finds a running daemon, `maybe_take_over` compares the client's `version::MCP_VERSION` against the daemon's reported version via the pure `client_should_take_over` helper (`semver`). If the client -is *strictly newer* it sends `STOP` (which drops the daemon's `HyperProcess`, -stopping `hyperd`), waits for the health endpoint to stop answering the identity -ping, then respawns a fresh daemon on the same endpoint. Equal/older/unparseable -versions reuse the daemon — never a downgrade-kill. This makes upgrades take -effect immediately instead of waiting for the old daemon to disappear. +is *strictly newer* it sends `STOP`, then waits until the old daemon has +released the daemon lock (`wait_for_lock_release`, bounded by `STOP_WAIT` = 15 s), +and only then spawns a fresh daemon. The lock is the right signal because the +daemon releases it last, after `hyperd` has exited; the health endpoint goes +quiet within milliseconds, long before `hyperd` is gone, and a successor started +then would fail its `hyperd` socket pre-flight. If the old daemon still holds the +lock after `STOP_WAIT`, the takeover gives up and the client runs in local mode. +Equal/older/unparseable versions reuse the daemon — never a downgrade-kill. + +On `STOP` the daemon acts at once: `DaemonState::request_shutdown` stores a +`tokio::sync::Notify` permit that `run_daemon`'s `select!` awaits, so the record +is removed, the listener joined, `hyperd` dropped and the lock released in that +order without waiting for the 5 s `hyperd` monitor poll. A daemon started while +its predecessor is still exiting retries the lock and the `hyperd` socket +pre-flight within `STARTUP_WAIT` (10 s); `SPAWN_TIMEOUT` (20 s) covers that wait +plus `hyperd`'s own start. `hyperdb-mcp daemon stop` waits for the same lock and +reports success only once it is free. **Idle shutdown is opt-in.** `DaemonConfig.idle_timeout` is `Option`, set only when `--idle-timeout` or `HYPERDB_DAEMON_IDLE_TIMEOUT` is provided diff --git a/hyperdb-mcp/README.md b/hyperdb-mcp/README.md index f7e2cdb0..1274cecc 100644 --- a/hyperdb-mcp/README.md +++ b/hyperdb-mcp/README.md @@ -286,7 +286,7 @@ The daemon is normally invisible — it auto-spawns on first use and stays resid ```bash hyperdb-mcp daemon status # Show running daemon (PID, endpoint, started_at, version) -hyperdb-mcp daemon stop # Gracefully shut down the daemon +hyperdb-mcp daemon stop # Gracefully shut down the daemon (returns once it has fully exited) hyperdb-mcp daemon # Run as a daemon explicitly (rarely needed) ``` diff --git a/hyperdb-mcp/src/daemon/health.rs b/hyperdb-mcp/src/daemon/health.rs index b31271ca..2b23b344 100644 --- a/hyperdb-mcp/src/daemon/health.rs +++ b/hyperdb-mcp/src/daemon/health.rs @@ -102,6 +102,11 @@ pub struct DaemonState { /// Whether a [`HealthListener::run`] loop is currently serving /// [`Self::endpoint`] and so can be woken by connecting to it. listener_registered: AtomicBool, + /// Wakes `run_daemon`'s main loop the moment shutdown is requested, so the + /// record is removed and `hyperd` released at once instead of at the next + /// poll of the `hyperd` monitor (every 5 s). `notify_one` stores a permit, + /// so a request that lands before the loop first awaits is not lost. + shutdown_notify: tokio::sync::Notify, } impl DaemonState { @@ -114,6 +119,7 @@ impl DaemonState { restart_requested: AtomicBool::new(false), endpoint, listener_registered: AtomicBool::new(false), + shutdown_notify: tokio::sync::Notify::new(), } } @@ -168,9 +174,16 @@ impl DaemonState { /// `xchg` on two cold paths. pub fn request_shutdown(&self) { self.shutdown.store(true, Ordering::SeqCst); + self.shutdown_notify.notify_one(); self.wake_accept_loop(); } + /// Resolves once [`Self::request_shutdown`] has been called, including a + /// call made before this was first awaited. + pub async fn shutdown_requested(&self) { + self.shutdown_notify.notified().await; + } + /// Nudge the listening endpoint so a waiting accept returns and the loop /// re-reads `should_shutdown`. /// diff --git a/hyperdb-mcp/src/daemon/legacy.rs b/hyperdb-mcp/src/daemon/legacy.rs index ac8461e5..3c066af0 100644 --- a/hyperdb-mcp/src/daemon/legacy.rs +++ b/hyperdb-mcp/src/daemon/legacy.rs @@ -23,7 +23,7 @@ //! record is stale. //! //! Caveats, accepted: a legacy daemon that ignores `STOP` costs up to -//! [`STOP_WAIT`] and another `STOP` on every client start until it exits; the +//! [`STOP_WAIT`](super::spawn::STOP_WAIT) and another `STOP` on every client start until it exits; the //! old daemon removes its own record during its shutdown, which can race this //! module's removal (harmless, both only delete the same legacy file); each //! command uses its own connection; a probe costs about a second at worst; and @@ -44,9 +44,6 @@ use tracing::{info, warn}; use super::discovery; use super::health::PONG_TOKEN; -/// How long to wait, after `STOP`, for the old daemon's port to stop answering. -pub(super) const STOP_WAIT: Duration = Duration::from_secs(15); - /// Budget for one probe (connect, or the whole request/reply exchange). const PROBE_TIMEOUT: Duration = Duration::from_millis(500); diff --git a/hyperdb-mcp/src/daemon/run.rs b/hyperdb-mcp/src/daemon/run.rs index 29893675..43572ce8 100644 --- a/hyperdb-mcp/src/daemon/run.rs +++ b/hyperdb-mcp/src/daemon/run.rs @@ -243,6 +243,11 @@ pub async fn run_daemon(config: DaemonConfig) -> Result<(), Box { info!("received shutdown signal"); } + // A `STOP` from a client: act now rather than at the next 5 s monitor + // poll, so a successor waiting on the lock is not kept waiting. + () = state.shutdown_requested() => { + info!("shutdown requested"); + } } state.request_shutdown(); @@ -321,12 +326,9 @@ fn build_params(state_dir: &Path) -> std::io::Result { // this block compiles out (`domain_socket_directory` is Unix-only). #[cfg(unix)] { - // Landmine for PR B: the basename here must NOT start with `hyper-`. - // `HyperProcess::drop` (`hyperdb-api/src/process.rs`) `remove_dir_all`s - // any caller-supplied socket directory whose basename - // `starts_with("hyper-")`, treating it as a temp dir it owns. `sockets` - // is safe; renaming it to e.g. `hyper-sockets` would make Drop delete - // the daemon's persistent state directory contents. + // `HyperProcess` tracks whether it created the socket directory + // (`owns_socket_directory`) and only removes one it created, so a + // caller-supplied `sockets` directory is never deleted on drop. let socket_dir = state_dir.join("sockets"); super::state_perms::ensure_owner_only_dir(&socket_dir)?; @@ -335,12 +337,15 @@ fn build_params(state_dir: &Path) -> std::io::Result { // this guard hyperd would die with "unable to listen on domain socket: // domain socket is in use" while `HyperProcess::new` masked it as a // 60-second "Timeout waiting for Hyper to connect to callback listener". - // Reachable only when a previous daemon's hyperd has outlived it - // (`build_params_waiting` retries on this). A refused/missing socket - // (stale file, dead owner) is fine: proceed and let hyperd's pid-liveness staleness check reclaim it. This - // keeps the crash-restart path safe: `try_restart_hyperd` reaps the - // SIGKILLed child (`guard.hyper = None`) before calling `build_params`, - // so here the connect is refused and we proceed to rebind. + // Reachable whenever a previous daemon's hyperd has not exited yet: + // a takeover or a manual start right after a `STOP`, or two daemons + // racing for the same state directory (`build_params_waiting` retries + // on this within `STARTUP_WAIT`). A refused/missing socket (stale file, + // dead owner) is fine: proceed and let hyperd's pid-liveness staleness + // check reclaim it. This keeps the crash-restart path safe: + // `try_restart_hyperd` reaps the SIGKILLed child (`guard.hyper = None`) + // before calling `build_params`, so there the connect is refused and + // we proceed to rebind. let socket_path = socket_dir.join("hyper"); if std::os::unix::net::UnixStream::connect(&socket_path).is_ok() { return Err(std::io::Error::new( diff --git a/hyperdb-mcp/src/daemon/spawn.rs b/hyperdb-mcp/src/daemon/spawn.rs index 3d101bca..1024a514 100644 --- a/hyperdb-mcp/src/daemon/spawn.rs +++ b/hyperdb-mcp/src/daemon/spawn.rs @@ -20,8 +20,16 @@ use super::discovery::{self, DaemonInfo}; use super::legacy; use super::lock::DaemonLock; -/// Maximum time to wait for the daemon to write its discovery file after spawning. -const SPAWN_TIMEOUT: Duration = Duration::from_secs(10); +/// Maximum time to wait for the daemon to write its discovery file after +/// spawning. It covers a [`STARTUP_WAIT`](super::run::STARTUP_WAIT) spent +/// waiting out a predecessor's shutdown plus `hyperd`'s own start. +const SPAWN_TIMEOUT: Duration = Duration::from_secs(20); + +/// Maximum time to wait, after sending `STOP`, for the old daemon to let go of +/// the daemon lock. The lock is released last, after `hyperd` has exited +/// (`HyperProcess`'s drop takes up to about 5 s), so a free lock means a +/// successor can start without colliding with it. +pub const STOP_WAIT: Duration = Duration::from_secs(15); /// Polling interval while waiting for the discovery file. const POLL_INTERVAL: Duration = Duration::from_millis(100); @@ -40,7 +48,7 @@ const POLL_INTERVAL: Duration = Duration::from_millis(100); /// period. pub fn ensure_daemon() -> io::Result { let dir = discovery::state_dir()?; - ensure_daemon_in(&dir, spawn_detached, SPAWN_TIMEOUT, legacy::STOP_WAIT) + ensure_daemon_in(&dir, spawn_detached, SPAWN_TIMEOUT, STOP_WAIT) } /// [`ensure_daemon`] for an explicit state directory, spawner and wait budget. @@ -48,7 +56,7 @@ fn ensure_daemon_in( dir: &Path, spawn: impl FnOnce() -> io::Result<()>, timeout: Duration, - legacy_stop_wait: Duration, + stop_wait: Duration, ) -> io::Result { match super::state_perms::verify_state_dir_trusted(dir) { Ok(()) => {} @@ -60,7 +68,7 @@ fn ensure_daemon_in( let start = Instant::now(); loop { if let Some(info) = discovery::discover_in(dir) { - return maybe_take_over(info, dir); + return maybe_take_over(info, dir, spawn, timeout, stop_wait); } // The state directory may not exist yet; the daemon creates it. @@ -78,7 +86,7 @@ fn ensure_daemon_in( // A pre-1.0 daemon holds no lock, so it is retired first; if it // will not stop, return without spawning (the caller runs in // local mode). - legacy::retire_under_lock(dir, legacy_stop_wait)?; + legacy::retire_under_lock(dir, stop_wait)?; discovery::remove_stale_record(dir); drop(lock); info!("no running daemon detected, spawning one"); @@ -184,13 +192,62 @@ pub fn client_should_take_over(client_ver: &str, daemon_ver: &str) -> bool { client > daemon } -/// Decide whether to reuse the running daemon or take it over with a newer version. -/// If the client is newer, we send STOP to the old daemon, wait for it to stop -/// answering, then spawn a fresh daemon. Otherwise we reuse the existing daemon. +/// Wait until nothing holds the daemon lock in `dir`, polling for up to +/// `timeout`. A missing state directory counts as free. +/// +/// The daemon releases the lock last, after its record, listener, socket and +/// `hyperd` are gone, so this is the reliable sign that a daemon has fully +/// exited. +/// +/// # Errors +/// Returns [`io::ErrorKind::TimedOut`] if the lock is still held after +/// `timeout`, or the error from probing the lock. +pub fn wait_for_lock_release(dir: &Path, timeout: Duration) -> io::Result<()> { + let deadline = Instant::now() + timeout; + loop { + match DaemonLock::try_acquire(dir) { + Ok(Some(lock)) => { + drop(lock); + return Ok(()); + } + Ok(None) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "the daemon in {} still holds its lock {} seconds after STOP", + dir.display(), + timeout.as_secs() + ), + )); + } + std::thread::sleep(POLL_INTERVAL); + } +} + +/// Decide whether to reuse the running daemon or take it over with a newer +/// version. If the client is newer, it sends `STOP` to the old daemon, waits +/// until the old daemon has released the daemon lock (so its `hyperd` is gone +/// too), then spawns a fresh daemon. Otherwise the existing daemon is reused. /// /// The health endpoint is the same for the old and the new daemon (it is derived -/// from the state directory), so a fresh daemon is told apart by its pid. -fn maybe_take_over(info: DaemonInfo, dir: &Path) -> io::Result { +/// from the state directory), so a fresh daemon is told apart by its pid and +/// start time, and the lock, not the endpoint, says when the old one is gone. +/// +/// # Errors +/// If the old daemon still holds the lock after `stop_wait` the takeover gives +/// up with [`io::ErrorKind::TimedOut`] (the caller then runs in local mode) +/// rather than reuse a daemon that is shutting down. +fn maybe_take_over( + info: DaemonInfo, + dir: &Path, + spawn: impl FnOnce() -> io::Result<()>, + timeout: Duration, + stop_wait: Duration, +) -> io::Result { let client_ver = crate::version::MCP_VERSION; if !client_should_take_over(client_ver, &info.version) { @@ -226,44 +283,31 @@ fn maybe_take_over(info: DaemonInfo, dir: &Path) -> io::Result { // Send STOP (best-effort; ignore error if daemon is already dying). let _ = super::health::send_command(&endpoint, "STOP"); - // Wait for the old daemon to stop answering. - let deadline = Instant::now() + SPAWN_TIMEOUT; - while Instant::now() < deadline { - if super::health::ping_identified( - &endpoint, - Duration::from_millis(200), - Duration::from_millis(200), - ) - .is_none() - { - // The old daemon is gone, so spawn the new one. - // - // Another client could observe the same and spawn concurrently. - // That is safe: the daemon lock lets exactly one daemon through, the - // loser exits (or waits for the winner), and `wait_for_daemon` - // converges on whichever daemon won. - // - // Narrowing that window: if a concurrent takeover has already - // published a fresh daemon, adopt it instead of spawning. - if let Some(fresh) = discovery::discover_in(dir) - && fresh.pid != info.pid - { - return Ok(fresh); - } - spawn_detached()?; - return wait_for_daemon(dir, SPAWN_TIMEOUT); - } - std::thread::sleep(POLL_INTERVAL); + // Wait for the old daemon to let go of the lock. + if let Err(error) = wait_for_lock_release(dir, stop_wait) { + warn!( + endpoint = %info.health_endpoint, + timeout_secs = stop_wait.as_secs(), + %error, + "old daemon did not exit after STOP, not taking over" + ); + return Err(error); } - // Old daemon didn't die within the deadline — log a warning and reuse it - // rather than fail the client. - warn!( - endpoint = %info.health_endpoint, - timeout_secs = SPAWN_TIMEOUT.as_secs(), - "old daemon did not stop within timeout, reusing it" - ); - Ok(info) + // The old daemon is gone, so spawn the new one. + // + // Another client could observe the same and spawn concurrently. That is + // safe: the daemon lock lets exactly one daemon through, and the loser + // waits for the winner (`run_daemon`) or exits. Narrowing that window: if a + // concurrent takeover has already published a fresh daemon, adopt it + // instead of spawning. + if let Some(fresh) = discovery::discover_in(dir) + && (fresh.pid != info.pid || fresh.started_at != info.started_at) + { + return Ok(fresh); + } + spawn()?; + wait_for_daemon(dir, timeout) } #[cfg(test)] @@ -671,4 +715,150 @@ mod tests { assert!(record.exists()); assert_eq!(stops(&peer.received()), 1); } + + // ─── Takeover waits on the lock ───────────────────────────────────── + + use crate::daemon::health::{DaemonState, HealthListener}; + + /// A stand-in for an old daemon: holds the lock, serves the health + /// channel, and on `STOP` removes its record, closes the listener and + /// (optionally) releases the lock after a pause, as the real one does + /// while `hyperd` winds down. + struct FakeOldDaemon { + state: Arc, + quit: Arc, + handle: Option>, + } + + impl FakeOldDaemon { + fn start(dir: &Path, release_after: Option) -> Self { + let lock = DaemonLock::try_acquire(dir) + .expect("lock probe") + .expect("fresh dir is lockable"); + let endpoint = HealthEndpoint::for_new_daemon(dir).expect("derive endpoint"); + let listener = HealthListener::bind(&endpoint).expect("bind fake health listener"); + let info = DaemonInfo { + pid: 1, + hyperd_endpoint: "127.0.0.1:1".to_string(), + health_endpoint: endpoint.as_str().to_string(), + started_at: "2026-01-01T00:00:00Z".to_string(), + version: "0.0.1".to_string(), + }; + let record = dir.join("daemon.json"); + std::fs::write(&record, serde_json::to_vec(&info).expect("serialize")).expect("write"); + + let state = Arc::new(DaemonState::new(endpoint)); + let quit = Arc::new(AtomicBool::new(false)); + let listener_state = Arc::clone(&state); + let listener_thread = std::thread::spawn(move || { + listener.run(listener_state, Arc::new(Mutex::new(info))); + }); + let watch_state = Arc::clone(&state); + let watch_quit = Arc::clone(&quit); + let handle = std::thread::spawn(move || { + while !watch_state.should_shutdown() && !watch_quit.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + } + let _ = std::fs::remove_file(&record); + let _ = listener_thread.join(); + match release_after { + Some(pause) => std::thread::sleep(pause), + None => { + while !watch_quit.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + } + } + } + drop(lock); + }); + Self { + state, + quit, + handle: Some(handle), + } + } + } + + impl Drop for FakeOldDaemon { + fn drop(&mut self) { + self.quit.store(true, Ordering::SeqCst); + self.state.request_shutdown(); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } + } + + /// The takeover must not spawn while the old daemon still holds the lock + /// (its `hyperd` is still exiting). Red if the lock wait is removed: the + /// spawner would then see the lock held. + #[test] + fn takeover_spawns_only_after_the_old_daemon_released_the_lock() { + let dir = tempfile::tempdir().expect("state dir"); + let _old = FakeOldDaemon::start(dir.path(), Some(Duration::from_millis(500))); + let calls = Cell::new(0); + let lock_was_free = Cell::new(false); + let started = Instant::now(); + + let error = ensure_daemon_in( + dir.path(), + || { + calls.set(calls.get() + 1); + // Probing and dropping the lock here is the check itself. + lock_was_free.set( + DaemonLock::try_acquire(dir.path()) + .expect("lock probe") + .is_some(), + ); + Ok(()) + }, + Duration::from_millis(200), + Duration::from_secs(10), + ) + .expect_err("the fake spawner publishes no daemon"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 1); + assert!( + lock_was_free.get(), + "spawned while the old daemon held the lock" + ); + assert!(started.elapsed() >= Duration::from_millis(500)); + } + + /// An old daemon that never releases the lock: no spawn, a `TimedOut` + /// error (the caller runs in local mode). + #[test] + fn takeover_gives_up_without_spawning_if_the_lock_stays_held() { + let dir = tempfile::tempdir().expect("state dir"); + let _old = FakeOldDaemon::start(dir.path(), None); + let calls = Cell::new(0); + + let error = ensure_daemon_in( + dir.path(), + counting_spawner(&calls), + Duration::from_millis(200), + Duration::from_millis(400), + ) + .expect_err("a daemon that keeps the lock must not be replaced"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(calls.get(), 0, "the spawner must never run"); + } + + #[test] + fn wait_for_lock_release_times_out_while_held_and_accepts_a_missing_dir() { + let dir = tempfile::tempdir().expect("state dir"); + let held = DaemonLock::try_acquire(dir.path()) + .expect("lock probe") + .expect("fresh dir is lockable"); + let error = wait_for_lock_release(dir.path(), Duration::from_millis(200)) + .expect_err("a held lock must time out"); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + + drop(held); + wait_for_lock_release(dir.path(), Duration::from_millis(200)).expect("free lock"); + wait_for_lock_release(&dir.path().join("missing"), Duration::from_millis(200)) + .expect("a missing directory has nothing to wait for"); + } } diff --git a/hyperdb-mcp/src/engine.rs b/hyperdb-mcp/src/engine.rs index 17af9492..4ee0e947 100644 --- a/hyperdb-mcp/src/engine.rs +++ b/hyperdb-mcp/src/engine.rs @@ -610,7 +610,7 @@ impl Engine { let info = match daemon::spawn::ensure_daemon() { Ok(info) => info, Err(e) => { - tracing::debug!(error = %e, "daemon unavailable, falling back to local mode"); + tracing::warn!(error = %e, "daemon unavailable, falling back to local mode"); return Ok(None); } }; diff --git a/hyperdb-mcp/src/main.rs b/hyperdb-mcp/src/main.rs index e3596475..4f898632 100644 --- a/hyperdb-mcp/src/main.rs +++ b/hyperdb-mcp/src/main.rs @@ -219,7 +219,13 @@ async fn run_daemon_mode(idle_timeout: Option) -> Result<(), Box Result<(), Box> { @@ -358,6 +364,24 @@ fn daemon_stop() { std::process::exit(1); } } + // Success means the daemon is gone, not merely that it acknowledged: it + // releases the lock only after `hyperd` has exited. + let dir = match discovery::state_dir() { + Ok(dir) => dir, + Err(e) => { + eprintln!( + "The daemon acknowledged STOP, but its state directory cannot be resolved to confirm it exited: {e}" + ); + std::process::exit(1); + } + }; + match daemon::spawn::wait_for_lock_release(&dir, daemon::spawn::STOP_WAIT) { + Ok(()) => println!("Daemon stopped."), + Err(e) => { + eprintln!("The daemon acknowledged STOP but did not exit: {e}"); + std::process::exit(1); + } + } } fn daemon_status() { diff --git a/hyperdb-mcp/tests/daemon_tests.rs b/hyperdb-mcp/tests/daemon_tests.rs index 05d7c9e6..80816fb8 100644 --- a/hyperdb-mcp/tests/daemon_tests.rs +++ b/hyperdb-mcp/tests/daemon_tests.rs @@ -1329,10 +1329,14 @@ fn daemon_cli_reaches_a_live_daemon_whose_record_is_missing() { let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); let info = test_daemon_info(&endpoint); // No record is written. The held lock stands in for the live daemon. - let _lock = DaemonLock::try_acquire(state.path()) + let lock = DaemonLock::try_acquire(state.path()) .unwrap() .expect("fresh state directory has a free lock"); - let mut peer = FakeHealthPeer::start_at(&endpoint, scripted_daemon(&info)); + let stop_seen = Arc::new(AtomicBool::new(false)); + let mut peer = FakeHealthPeer::start_at( + &endpoint, + scripted_daemon_noting_stop(&info, Arc::clone(&stop_seen)), + ); let status = run_cli_bounded(&["daemon", "status"], state.path(), Duration::from_secs(10)); assert!( @@ -1341,7 +1345,16 @@ fn daemon_cli_reaches_a_live_daemon_whose_record_is_missing() { status.status, String::from_utf8_lossy(&status.stderr) ); + // `daemon stop` succeeds only once the lock is free, so let go of it a + // moment after the peer has actually received STOP (not on a fixed delay + // that a loaded machine could outrun). + let releaser = std::thread::spawn(move || { + wait_for_flag(&stop_seen, Duration::from_secs(30)); + std::thread::sleep(Duration::from_millis(300)); + drop(lock); + }); let stop = run_cli_bounded(&["daemon", "stop"], state.path(), Duration::from_secs(10)); + releaser.join().unwrap(); assert!( stop.status.success(), "`daemon stop` failed: {}\nstderr:\n{}", @@ -1354,6 +1367,52 @@ fn daemon_cli_reaches_a_live_daemon_whose_record_is_missing() { ); } +/// `daemon stop` returns only once the daemon has let go of its lock, not when +/// it merely acknowledges `STOP`. The fake daemon acknowledges at once but +/// releases the lock 1.5 s later; the CLI must not have returned before that. +#[cfg(unix)] +#[test] +fn daemon_stop_cli_returns_only_after_the_lock_is_free() { + let state = TempDir::new().unwrap(); + let endpoint = HealthEndpoint::for_new_daemon(state.path()).unwrap(); + let info = test_daemon_info(&endpoint); + write_record(state.path(), &info); + let lock = DaemonLock::try_acquire(state.path()) + .unwrap() + .expect("fresh state directory has a free lock"); + let stop_seen = Arc::new(AtomicBool::new(false)); + let mut peer = FakeHealthPeer::start_at( + &endpoint, + scripted_daemon_noting_stop(&info, Arc::clone(&stop_seen)), + ); + + let lock_released = Arc::new(AtomicBool::new(false)); + let released_by_thread = Arc::clone(&lock_released); + let freer = std::thread::spawn(move || { + // Hold the lock for 1.5 s after STOP arrives: the CLI must outwait it. + wait_for_flag(&stop_seen, Duration::from_secs(30)); + std::thread::sleep(Duration::from_millis(1500)); + released_by_thread.store(true, Ordering::SeqCst); + drop(lock); + }); + let output = run_cli_bounded(&["daemon", "stop"], state.path(), Duration::from_secs(20)); + let released_when_cli_returned = lock_released.load(Ordering::SeqCst); + freer.join().unwrap(); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "`daemon stop` failed: {stderr}"); + assert!( + released_when_cli_returned, + "`daemon stop` returned while the daemon still held the lock" + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("Daemon stopped."), + "stdout was {:?}", + String::from_utf8_lossy(&output.stdout) + ); + assert!(peer.stop_and_join().iter().any(|command| command == "STOP")); +} + /// A held lock with no record and nothing answering gets a clear message, not /// "No daemon is currently running." #[test] @@ -1893,6 +1952,33 @@ fn scripted_daemon(info: &DaemonInfo) -> impl Fn(&str) -> Vec + Send + 'stat } } +/// [`scripted_daemon`] that also raises `stop_seen` the moment it is sent `STOP`, +/// so a test can release the daemon lock only after the CLI really asked. +fn scripted_daemon_noting_stop( + info: &DaemonInfo, + stop_seen: Arc, +) -> impl Fn(&str) -> Vec + Send + 'static { + let inner = scripted_daemon(info); + move |command| { + if command == "STOP" { + stop_seen.store(true, Ordering::SeqCst); + } + inner(command) + } +} + +/// Block until `flag` is raised, failing the test after `budget`. +fn wait_for_flag(flag: &AtomicBool, budget: Duration) { + let started = Instant::now(); + while !flag.load(Ordering::SeqCst) { + assert!( + started.elapsed() < budget, + "flag not raised within {budget:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + /// Name of the file whose presence proves a daemon lock was taken. #[cfg(unix)] fn control_lock_name() -> &'static str { @@ -2140,6 +2226,50 @@ fn preserve_child_runtime_environment(command: &mut std::process::Command) { } } +/// Run `run_daemon` on its own thread and runtime, for the state directory +/// named by `HYPERDB_STATE_DIR`. Caller MUST hold `ENV_LOCK`. +fn spawn_daemon_thread() -> std::thread::JoinHandle> { + std::thread::spawn(move || -> Result<(), String> { + let rt = tokio::runtime::Runtime::new().map_err(|e| format!("rt: {e}"))?; + rt.block_on(async { + let config = hyperdb_mcp::daemon::run::DaemonConfig { + idle_timeout: Some(Duration::from_secs(300)), + }; + hyperdb_mcp::daemon::run::run_daemon(config) + .await + .map_err(|e| format!("run_daemon: {e}")) + }) + }) +} + +/// Wait up to `budget` for a daemon in `state_dir` whose `started_at` differs +/// from `not_started_at` to become discoverable. Panics with the thread's +/// error if `handle`'s daemon exits first. +fn wait_for_new_daemon( + handle: &std::thread::JoinHandle>, + state_dir: &Path, + not_started_at: Option<&str>, + budget: Duration, +) -> DaemonInfo { + let start = Instant::now(); + loop { + if let Some(info) = discovery::discover_in(state_dir) + && Some(info.started_at.as_str()) != not_started_at + { + return info; + } + assert!( + !handle.is_finished(), + "the daemon thread exited before it became discoverable" + ); + assert!( + start.elapsed() <= budget, + "no new daemon became discoverable within {budget:?}" + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + /// A real daemon running in a background thread for integration tests. /// Sets `HYPERDB_STATE_DIR` to an isolated, short directory. /// Caller MUST hold `ENV_LOCK` before calling `start()`. @@ -2426,3 +2556,109 @@ impl Drop for EnvGuard { } } } + +// ─── Takeover: a successor starts right behind a STOP ───────────────────────── + +/// Two daemons in sequence on ONE state directory. Daemon B is started the +/// instant `STOP` has been sent to daemon A, while A's `hyperd` is still +/// winding down. B must wait out A (lock, then `hyperd` socket) within its +/// `STARTUP_WAIT`, become discoverable with a new `started_at`, serve a query +/// through its engine, and A's thread must exit cleanly. Without the +/// lock wait B failed on the lock or the `hyperd` socket pre-flight and no +/// daemon ran; without the prompt STOP, B is not discoverable within 3 s. +#[test] +#[cfg_attr( + target_os = "macos", + ignore = "flaky on macOS CI — daemon startup exceeds 150s timeout" +)] +fn a_daemon_started_right_after_stop_takes_over_the_state_directory() { + let _lock = acquire_env_lock(); + let tmp = TempDir::new().unwrap(); + let state_dir = tmp.path().to_path_buf(); + let _state_dir_guard = EnvGuard::set("HYPERDB_STATE_DIR", state_dir.to_str().unwrap()); + + let daemon_a = spawn_daemon_thread(); + let info_a = wait_for_new_daemon(&daemon_a, &state_dir, None, Duration::from_secs(150)); + let endpoint = HealthEndpoint::from_record(&info_a.health_endpoint, &state_dir) + .expect("a discovered record names this state directory's endpoint"); + + // B must be discoverable well inside the old daemon's 5 s `hyperd_monitor` + // poll (measured: ~0.3 s with the prompt STOP, ~5.2 s without it), so this + // fails if STOP is only acted on at the next poll. + let bound = Duration::from_secs(3); + let exit_bound = Duration::from_secs(15); + let started = Instant::now(); + let response = health::send_command(&endpoint, "STOP").expect("STOP reaches daemon A"); + assert_eq!(response.trim(), "STOPPING"); + let daemon_b = spawn_daemon_thread(); + + let info_b = wait_for_new_daemon(&daemon_b, &state_dir, Some(&info_a.started_at), bound); + assert_ne!(info_b.started_at, info_a.started_at); + + // B serves a query through an engine that reached it. + let db = tmp.path().join("takeover.hyper"); + let engine = hyperdb_mcp::engine::Engine::new(Some(db.to_str().unwrap().to_string())) + .expect("engine connects to daemon B"); + let engine_endpoint = engine + .hyperd_endpoint() + .expect("engine reports its hyperd endpoint") + .clone(); + assert_eq!( + engine_endpoint, info_b.hyperd_endpoint, + "the engine must be in daemon mode against B, not local mode" + ); + let rows = engine + .execute_query_to_json("SELECT 1 AS one") + .expect("daemon B serves a query"); + assert_eq!(rows.len(), 1); + drop(engine); + + // A exited cleanly, within the bound. + while !daemon_a.is_finished() { + assert!( + started.elapsed() <= exit_bound, + "daemon A did not exit within {exit_bound:?} of STOP" + ); + std::thread::sleep(Duration::from_millis(50)); + } + daemon_a + .join() + .expect("daemon A thread must not panic") + .expect("daemon A must exit cleanly"); + + // Tear B down and wait for its lock, as TestDaemon does. + let endpoint_b = HealthEndpoint::from_record(&info_b.health_endpoint, &state_dir).unwrap(); + let _ = health::send_command(&endpoint_b, "STOP"); + hyperdb_mcp::daemon::spawn::wait_for_lock_release(&state_dir, Duration::from_secs(15)) + .expect("daemon B releases the lock after STOP"); + daemon_b + .join() + .expect("daemon B thread must not panic") + .expect("daemon B must exit cleanly"); +} + +/// `STOP` is acted on at once: the record is gone within a second, not at the +/// next 5 s `hyperd` monitor poll. Measured on the record, because +/// `run_daemon` itself only returns after `hyperd` has been dropped. +#[test] +#[cfg_attr( + target_os = "macos", + ignore = "flaky on macOS CI — daemon startup exceeds 150s timeout" +)] +fn stop_removes_the_daemon_record_promptly() { + let _lock = acquire_env_lock(); + let daemon = TestDaemon::start(); + let record = daemon.state_dir.join("daemon.json"); + assert!(record.exists(), "a running daemon has a record"); + + let sent = Instant::now(); + health::send_command(&daemon.endpoint, "STOP").expect("STOP reaches the daemon"); + while record.exists() { + assert!( + sent.elapsed() < Duration::from_secs(1), + "the record was still there {:?} after STOP", + sent.elapsed() + ); + std::thread::sleep(Duration::from_millis(10)); + } +} From 79174b19821fa86cd4cb12bb69c4965514f4de9f Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 16:49:50 -0700 Subject: [PATCH 37/61] fix(mcp): authenticate the control endpoint server and harden the Windows lock The client now verifies the server behind the control socket or pipe belongs to the current user, the Windows daemon lock uses LockFileEx instead of an exclusive open, and the daemon checks the state directory before creating its log. Doctor and the CLI no longer create the lock file, and daemon stop names a recorded pre-1.0 daemon. --- hyperdb-mcp/CHANGELOG.md | 9 +- hyperdb-mcp/DEVELOPMENT.md | 17 ++- hyperdb-mcp/src/daemon/control.rs | 157 ++++++++++++++++++++++++++-- hyperdb-mcp/src/daemon/discovery.rs | 88 ++++++++++++++-- hyperdb-mcp/src/daemon/legacy.rs | 7 +- hyperdb-mcp/src/daemon/lock.rs | 101 ++++++++++++++---- hyperdb-mcp/src/daemon/run.rs | 19 ++-- hyperdb-mcp/src/diagnostics.rs | 8 +- hyperdb-mcp/src/main.rs | 27 +++-- hyperdb-mcp/tests/daemon_tests.rs | 35 +++++++ 10 files changed, 402 insertions(+), 66 deletions(-) diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 57486176..ab064d3f 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -34,7 +34,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). health port. A single-instance lock (`daemon.lock`) replaces the port bind as the "is a daemon running" decision. An older client talking to a daemon of this release cannot find it (the record no longer has `health_port`) and falls - back to local mode. + back to local mode. A client also checks who serves the endpoint (Unix: + peer uid via `SO_PEERCRED` / `getpeereid`; Windows: the pipe server process's + user) and refuses one that is not the current user. - **The state directory must now be owned by the current user and not writable by group or others.** A daemon refuses to start in one, and a client treats it as untrusted and uses local mode rather than connecting to an endpoint it @@ -50,7 +52,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). (a daemon that ignores `STOP` costs up to 15 s and another `STOP` on every client start until it exits). An old client meeting a new daemon cannot parse its record, so it waits out its spawn timeout and runs in - local mode; nothing is destroyed. + local mode; nothing is destroyed. Running pre-rc.4 clients and new ones + against one state directory at the same time is not supported: an old client + may overwrite `daemon.json` (not verified against the old code), in which + case a new client no longer finds the daemon through the record. ### Removed diff --git a/hyperdb-mcp/DEVELOPMENT.md b/hyperdb-mcp/DEVELOPMENT.md index 8b5670dd..04944231 100644 --- a/hyperdb-mcp/DEVELOPMENT.md +++ b/hyperdb-mcp/DEVELOPMENT.md @@ -251,14 +251,25 @@ Falls back to local mode (per-session `hyperd` via `HyperProcess::new`) when the `ControlListener`, `ControlStream`, `connect`). `daemon.json` carries it as `health_endpoint`; a client accepts the recorded value only if `HealthEndpoint::from_record` finds it equals the endpoint its own state -directory implies, so a tampered record can never redirect it. Liveness is an +directory implies. On Unix that pins the exact socket path, so a tampered +record cannot redirect a client; the client also checks the peer uid of the +socket (`SO_PEERCRED` / `getpeereid`) equals its own. On Windows a pipe name +does not encode the state directory, so the check only pins the +`\\.\pipe\hyperdb-mcp-` prefix and character set; the trust model there is a +random per-daemon pipe name, an owner-only DACL, `FILE_FLAG_FIRST_PIPE_INSTANCE` +at creation, and a client-side check that the pipe's server process runs as the +current user (a squatter's pipe is refused with `PermissionDenied`). Neither +platform verifies the *ancestors* of the state directory: only the directory +itself is checked for ownership and write bits, so a user who can rename an +ancestor can swap the whole directory. Liveness is an identity handshake: `health::ping_identified` sends `PING` and requires the reply's first two tokens to be exactly `PONG` and `hyperdb-mcp` (the third is the daemon version). `discover()` is a pure read (record + identified `PING`); stale-record removal lives in `ensure_daemon`, under the lock. -**Single instance.** `daemon::lock::DaemonLock` is an `flock` (Windows: a locked -file region) on `/daemon.lock`, never unlinked. `run_daemon` creates +**Single instance.** `daemon::lock::DaemonLock` is an `flock` (Windows: an +exclusive `LockFileEx` lock on the first byte, opened without +`FILE_SHARE_DELETE`; `ERROR_LOCK_VIOLATION` means held) on `/daemon.lock`, never unlinked. `run_daemon` creates and checks the state directory (`state_perms::verify_state_dir_trusted`: owned by the effective user, no group/other write), takes the lock (retrying up to `STARTUP_WAIT`; a daemon that answers an identified `PING` continuously for diff --git a/hyperdb-mcp/src/daemon/control.rs b/hyperdb-mcp/src/daemon/control.rs index 18034b43..15cb968a 100644 --- a/hyperdb-mcp/src/daemon/control.rs +++ b/hyperdb-mcp/src/daemon/control.rs @@ -21,6 +21,10 @@ //! same bounded-wait accept ([`ControlListener::accept_timeout`]), so the //! accept loop in the health module keeps its shape. //! +//! Clients also authenticate the server: on Unix the socket peer's uid must be +//! the current user's, on Windows the pipe server's process must run as the +//! current user; otherwise [`connect`] fails with `PermissionDenied`. +//! //! A known limitation on Windows: a daemon started from an elevated prompt //! may create a pipe that clients running without elevation cannot open. //! @@ -53,9 +57,16 @@ impl HealthEndpoint { } /// Parse the endpoint string of a discovery record found in `state_dir`. - /// `None` when it is not exactly an endpoint a daemon of this state - /// directory could have produced, so a tampered or foreign record (say, a - /// path to some other program's socket) is never connected to. + /// `None` when it is not an endpoint a daemon of this state directory + /// could have produced, so a tampered or foreign record (say, a path to + /// some other program's socket) is never connected to. + /// + /// On Unix the record must equal the exact socket path of `state_dir`. On + /// Windows a pipe name does not encode the state directory, so only the + /// `\\.\pipe\hyperdb-mcp-` prefix and character set are checked; the + /// protection there is the random pipe name, the owner-only DACL and the + /// client's server-owner check in [`connect`]. The ancestors of + /// `state_dir` are not examined on either platform. #[must_use] pub fn from_record(record: &str, state_dir: &Path) -> Option { platform::is_valid_endpoint(record, state_dir).then(|| Self(record.to_owned())) @@ -364,7 +375,69 @@ mod platform { } pub(super) fn connect(endpoint: &str, _connect_timeout: Duration) -> io::Result { - UnixStream::connect(endpoint).map(|inner| Stream { inner }) + let inner = UnixStream::connect(endpoint)?; + // The socket path sits in a directory only we may write to, but a + // stale record could still point at a socket some other user's + // process owns; never speak to one. + let peer = peer_uid(&inner)?; + // SAFETY: `geteuid` takes no arguments and cannot fail. + let ours = unsafe { libc::geteuid() }; + if peer != ours { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("the daemon control socket is served by uid {peer}, not by this user"), + )); + } + Ok(Stream { inner }) + } + + /// The effective uid of the process on the other end of `stream`. + #[cfg(any(target_os = "linux", target_os = "android"))] + fn peer_uid(stream: &UnixStream) -> io::Result { + let mut credentials = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let mut length = libc::socklen_t::try_from(std::mem::size_of::()) + .expect("ucred size fits socklen_t"); + // SAFETY: the descriptor is valid for the call; `credentials` is a + // writable `ucred` and `length` holds its size. + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&raw mut credentials).cast(), + &raw mut length, + ) + }; + if status == 0 { + Ok(credentials.uid) + } else { + Err(io::Error::last_os_error()) + } + } + + /// The effective uid of the process on the other end of `stream`. + #[cfg(not(any(target_os = "linux", target_os = "android")))] + fn peer_uid(stream: &UnixStream) -> io::Result { + let mut uid: libc::uid_t = 0; + let mut gid: libc::gid_t = 0; + // SAFETY: the descriptor is valid for the call; both out-pointers are + // writable. + let status = unsafe { libc::getpeereid(stream.as_raw_fd(), &raw mut uid, &raw mut gid) }; + if status == 0 { + Ok(uid) + } else { + Err(io::Error::last_os_error()) + } + } + + #[cfg(test)] + pub(super) fn own_peer_uid_matches(stream: &UnixStream) -> bool { + // SAFETY: `geteuid` takes no arguments and cannot fail. + peer_uid(stream).is_ok_and(|uid| uid == unsafe { libc::geteuid() }) } #[derive(Debug)] @@ -431,11 +504,12 @@ mod platform { }; use windows_sys::Win32::System::IO::{CancelIoEx, GetOverlappedResult, OVERLAPPED}; use windows_sys::Win32::System::Pipes::{ - ConnectNamedPipe, CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, - PIPE_TYPE_BYTE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, + ConnectNamedPipe, CreateNamedPipeW, GetNamedPipeServerProcessId, PIPE_READMODE_BYTE, + PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, }; use windows_sys::Win32::System::Threading::{ - CreateEventW, GetCurrentProcess, OpenProcessToken, WaitForSingleObject, + CreateEventW, GetCurrentProcess, OpenProcess, OpenProcessToken, + PROCESS_QUERY_LIMITED_INFORMATION, WaitForSingleObject, }; /// Prefix every endpoint of ours carries; anything else in a record is @@ -549,10 +623,16 @@ mod platform { /// The current user's SID in string form (`S-1-5-21-...`). fn current_user_sid() -> io::Result { + // SAFETY: the pseudo-handle of the current process needs no closing. + token_user_sid(unsafe { GetCurrentProcess() }) + } + + /// The user SID, in string form, of the process behind `process`. + fn token_user_sid(process: windows_sys::Win32::Foundation::HANDLE) -> io::Result { let mut token = ptr::null_mut(); - // SAFETY: the pseudo-handle of the current process needs no closing; + // SAFETY: `process` is a valid process handle with query access; // `token` is a valid out-pointer. - if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &raw mut token) } == 0 { + if unsafe { OpenProcessToken(process, TOKEN_QUERY, &raw mut token) } == 0 { return Err(io::Error::last_os_error()); } // SAFETY: `token` is a valid handle returned just above, owned here. @@ -775,6 +855,32 @@ mod platform { } } + /// Refuse a pipe whose server process does not run as the current user. + /// The pipe name is random and its DACL owner-only, but a stale record + /// could name a pipe another user's process created; never speak to one. + fn verify_server_is_us(pipe: &OwnedHandle) -> io::Result<()> { + let mut server_pid = 0_u32; + // SAFETY: `pipe` is a valid pipe handle; `server_pid` is writable. + if unsafe { GetNamedPipeServerProcessId(pipe.as_raw_handle(), &raw mut server_pid) } == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: plain call; a null handle on failure is checked below. + let process = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, server_pid) }; + if process.is_null() { + return Err(io::Error::last_os_error()); + } + // SAFETY: `process` is a valid handle returned just above, owned here. + let process = unsafe { OwnedHandle::from_raw_handle(process) }; + if token_user_sid(process.as_raw_handle())? == current_user_sid()? { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "the daemon control pipe is served by a process of another user", + )) + } + } + pub(super) fn connect(endpoint: &str, connect_timeout: Duration) -> io::Result { let deadline = Instant::now() + connect_timeout; loop { @@ -786,7 +892,11 @@ mod platform { .security_qos_flags(SECURITY_IDENTIFICATION) .open(endpoint); match opened { - Ok(file) => return Stream::new(OwnedHandle::from(file)), + Ok(file) => { + let pipe = OwnedHandle::from(file); + verify_server_is_us(&pipe)?; + return Stream::new(pipe); + } Err(error) if error .raw_os_error() @@ -1137,6 +1247,17 @@ mod tests { ); } + #[cfg(windows)] + #[test] + fn windows_records_accept_our_pipe_prefix_and_refuse_others() { + let dir = tempfile::tempdir().expect("tempdir"); + assert!(HealthEndpoint::from_record(r"\\.\pipe\hyperdb-mcp-12-00ab", dir.path()).is_some()); + assert!(HealthEndpoint::from_record(r"\\.\pipe\docker_engine", dir.path()).is_none()); + assert!(HealthEndpoint::from_record(r"\\.\pipe\hyperdb-mcp-", dir.path()).is_none()); + assert!(HealthEndpoint::from_record(r"\\.\pipe\hyperdb-mcp-a\b", dir.path()).is_none()); + assert!(HealthEndpoint::from_record(r"\\server\pipe\hyperdb-mcp-1", dir.path()).is_none()); + } + #[cfg(unix)] mod unix { use std::os::unix::fs::PermissionsExt; @@ -1157,6 +1278,22 @@ mod tests { assert_eq!(mode & 0o777, 0o600); } + /// The peer-uid check passes for a server running as this user (the + /// only kind this test can create without a second account). + #[test] + fn the_peer_uid_of_a_same_user_server_is_accepted() { + let dir = tempfile::tempdir().expect("tempdir"); + let endpoint = endpoint_in(&dir); + let _listener = ControlListener::bind(&endpoint).expect("bind"); + let stream = + std::os::unix::net::UnixStream::connect(endpoint.as_str()).expect("connect"); + assert!(crate::daemon::control::platform::own_peer_uid_matches( + &stream + )); + crate::daemon::control::connect(&endpoint, std::time::Duration::from_secs(5)) + .expect("a same-user server passes the peer check"); + } + #[test] fn a_stale_socket_file_is_reclaimed() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/hyperdb-mcp/src/daemon/discovery.rs b/hyperdb-mcp/src/daemon/discovery.rs index eb37cff6..e4ba8a63 100644 --- a/hyperdb-mcp/src/daemon/discovery.rs +++ b/hyperdb-mcp/src/daemon/discovery.rs @@ -292,17 +292,27 @@ pub fn discovery_file_path() -> io::Result { /// # Errors /// Returns an error if the state directory cannot be created or the file cannot be written. pub fn write_discovery_file(info: &DaemonInfo) -> io::Result<()> { - write_discovery_record(info) + write_discovery_record(&state_dir()?, info) } -pub(super) fn write_enriched_discovery_file(info: &DaemonInfo) -> io::Result<()> { +/// [`write_discovery_file`] for an explicit state directory. +/// +/// # Errors +/// As [`write_discovery_file`]. +pub fn write_discovery_file_in(dir: &Path, info: &DaemonInfo) -> io::Result<()> { + write_discovery_record(dir, info) +} + +/// Write the record with this process's identity, into `dir`. The daemon +/// passes the state directory it already holds rather than re-reading the +/// environment. +pub(super) fn write_enriched_discovery_file_in(dir: &Path, info: &DaemonInfo) -> io::Result<()> { let record = DaemonRecord::with_current_identity(info)?; - write_discovery_record(&record) + write_discovery_record(dir, &record) } -fn write_discovery_record(record: &(impl Serialize + ?Sized)) -> io::Result<()> { - let dir = state_dir()?; - super::state_perms::ensure_owner_only_dir(&dir)?; +fn write_discovery_record(dir: &Path, record: &(impl Serialize + ?Sized)) -> io::Result<()> { + super::state_perms::ensure_owner_only_dir(dir)?; let path = dir.join("daemon.json"); let tmp_path = dir.join("daemon.json.tmp"); @@ -425,6 +435,23 @@ pub(super) fn read_legacy_record(dir: &Path) -> Option Option { + let record = read_legacy_record(dir)?; + Some(format!( + "{} records a pre-1.0 daemon (pid {}, port {}), which this version cannot \ + stop or inspect. Starting a hyperdb-mcp session retires it automatically; \ + or stop it yourself (kill {}) and delete that file.", + dir.join("daemon.json").display(), + record.pid, + record.health_port, + record.pid + )) +} + /// Remove the legacy record `expected`, if `dir` still holds exactly it. /// /// The caller must hold the [`DaemonLock`](super::lock::DaemonLock) for `dir`. @@ -437,11 +464,16 @@ pub(super) fn remove_legacy_record(dir: &Path, expected: super::legacy::LegacyRe /// Remove the discovery file (called during graceful shutdown). pub fn remove_discovery_file() { - if let Ok(path) = discovery_file_path() { - let _ = std::fs::remove_file(&path); + if let Ok(dir) = state_dir() { + remove_discovery_file_in(&dir); } } +/// [`remove_discovery_file`] for an explicit state directory. +pub fn remove_discovery_file_in(dir: &Path) { + let _ = std::fs::remove_file(dir.join("daemon.json")); +} + /// Check if the daemon is alive by sending PING and verifying the identifying /// token. Connecting alone is not enough: any program can hold a socket. fn is_daemon_alive(endpoint: &HealthEndpoint) -> bool { @@ -885,7 +917,9 @@ mod tests { } /// A record pointing at an endpoint this state directory does not imply is - /// never connected to, even if something answers there. + /// never connected to, even if something answers there. Unix pins the + /// exact socket path; the Windows twin is below. + #[cfg(unix)] #[test] fn discover_in_rejects_a_foreign_endpoint() { let live_dir = short_state_dir(); @@ -901,6 +935,32 @@ mod tests { assert_eq!(discover_in(live_dir.path()), Some(live_info)); } + /// Windows: a pipe name does not encode the state directory, so only the + /// prefix and character set are checked. A record naming another + /// program's pipe is never connected to, even if it answers. + #[cfg(windows)] + #[test] + fn discover_in_rejects_a_foreign_endpoint() { + let live_dir = short_state_dir(); + let other_dir = short_state_dir(); + let live_info = info_for(live_dir.path(), 9_200); + let _daemon = LiveDaemon::start(live_dir.path(), &live_info); + + for foreign in [ + r"\\.\pipe\docker_engine", + r"\\.\pipe\hyperdb-mcp-", + "127.0.0.1:7485", + ] { + let mut info = live_info.clone(); + info.health_endpoint = foreign.to_string(); + write_record_for_test(other_dir.path(), &info); + assert!(discover_in(other_dir.path()).is_none(), "{foreign}"); + } + // The daemon is found through a record that names its own pipe. + write_record_for_test(live_dir.path(), &live_info); + assert_eq!(discover_in(live_dir.path()), Some(live_info)); + } + /// A state directory another account can write to is not trusted. #[cfg(unix)] #[test] @@ -1471,6 +1531,16 @@ mod tests { assert!(path.exists(), "a changed record must survive"); } + #[test] + fn legacy_daemon_hint_names_pid_and_port_only_for_a_legacy_record() { + let tmp = TempDir::new().unwrap(); + assert_eq!(legacy_daemon_hint(tmp.path()), None); + write_legacy_file(tmp.path(), 4_242); + let hint = legacy_daemon_hint(tmp.path()).expect("legacy record gives a hint"); + assert!(hint.contains("pre-1.0 daemon"), "{hint}"); + assert!(hint.contains("4242"), "{hint}"); + } + #[test] fn read_legacy_record_ignores_a_current_shape_record() { let tmp = TempDir::new().unwrap(); diff --git a/hyperdb-mcp/src/daemon/legacy.rs b/hyperdb-mcp/src/daemon/legacy.rs index 3c066af0..1425f64b 100644 --- a/hyperdb-mcp/src/daemon/legacy.rs +++ b/hyperdb-mcp/src/daemon/legacy.rs @@ -138,8 +138,11 @@ pub(super) fn retire_under_lock(dir: &Path, stop_wait: Duration) -> io::Result<( io::ErrorKind::TimedOut, format!( "the pre-1.0 daemon record names port {}, which is open but did not \ - identify the recorded daemon (pid {}); nothing was sent to it", - record.health_port, record.pid + identify the recorded daemon (pid {}); nothing was sent to it. If \ + something else holds that port, delete {} and try again", + record.health_port, + record.pid, + dir.join("daemon.json").display() ), )); } diff --git a/hyperdb-mcp/src/daemon/lock.rs b/hyperdb-mcp/src/daemon/lock.rs index 9937df45..17c4dcc5 100644 --- a/hyperdb-mcp/src/daemon/lock.rs +++ b/hyperdb-mcp/src/daemon/lock.rs @@ -15,7 +15,9 @@ //! and locks that one. Releasing the lock only closes the handle. //! //! The lock conflicts within one process too, because each acquire opens the -//! file separately; the in-process test daemons rely on that. +//! file separately (`flock` on Unix, a `LockFileEx` byte-range lock on +//! Windows, both owned by the open handle); the in-process test daemons rely +//! on that. //! //! This module depends only on `std`, `libc` and `windows-sys`, so it can be //! compiled for Windows on its own. @@ -45,9 +47,24 @@ impl DaemonLock { /// any reason other than being held, such as `state_dir` missing, or the /// lock file being a symlink on Unix. pub fn try_acquire(state_dir: &Path) -> io::Result> { - platform::try_lock(&state_dir.join(LOCK_FILE_NAME)) + platform::try_lock(&state_dir.join(LOCK_FILE_NAME), true) .map(|held| held.map(|file| Self { _file: file })) } + + /// Whether another holder has the lock, without ever creating the lock + /// file: a missing file (or state directory) means nobody holds it. The + /// probe takes the lock for an instant and lets go again. + /// + /// # Errors + /// + /// As [`Self::try_acquire`], except that a missing file is not an error. + pub fn is_held(state_dir: &Path) -> io::Result { + match platform::try_lock(&state_dir.join(LOCK_FILE_NAME), false) { + Ok(held) => Ok(held.is_none()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } + } } #[cfg(unix)] @@ -58,11 +75,11 @@ mod platform { use std::os::unix::fs::OpenOptionsExt; use std::path::Path; - pub(super) fn try_lock(path: &Path) -> io::Result> { + pub(super) fn try_lock(path: &Path, create: bool) -> io::Result> { let file = OpenOptions::new() .read(true) .write(true) - .create(true) + .create(create) .truncate(false) .mode(0o600) .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) @@ -86,25 +103,55 @@ mod platform { use std::fs::{File, OpenOptions}; use std::io; use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::AsRawHandle; use std::path::Path; - /// `ERROR_SHARING_VIOLATION`: another handle has the file open. - const ERROR_SHARING_VIOLATION: i32 = 32; - - /// Opening with no sharing is the lock: a second open fails while the - /// first handle lives. - pub(super) fn try_lock(path: &Path) -> io::Result> { - match OpenOptions::new() + use windows_sys::Win32::Storage::FileSystem::{ + FILE_SHARE_READ, FILE_SHARE_WRITE, LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, + LockFileEx, + }; + use windows_sys::Win32::System::IO::OVERLAPPED; + + /// `ERROR_LOCK_VIOLATION`: another handle holds a conflicting byte-range + /// lock. + const ERROR_LOCK_VIOLATION: i32 = 33; + + /// An exclusive `LockFileEx` lock on the first byte is the lock. Byte + /// range locks belong to the handle, so a second open in this process + /// conflicts like one in another process, and the OS drops the lock when + /// the handle closes however the process exits. The file is opened + /// without `FILE_SHARE_DELETE` so it cannot be unlinked while held. + pub(super) fn try_lock(path: &Path, create: bool) -> io::Result> { + let file = OpenOptions::new() .read(true) .write(true) - .create(true) + .create(create) .truncate(false) - .share_mode(0) - .open(path) - { - Ok(file) => Ok(Some(file)), - Err(error) if error.raw_os_error() == Some(ERROR_SHARING_VIOLATION) => Ok(None), - Err(error) => Err(error), + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .open(path)?; + // SAFETY: an all-zero `OVERLAPPED` is the documented initial state + // (offset 0, no event) for a synchronous `LockFileEx`. + let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; + // SAFETY: `file` owns a valid handle for the whole call and + // `overlapped` outlives it. + let locked = unsafe { + LockFileEx( + file.as_raw_handle(), + LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &raw mut overlapped, + ) + }; + if locked != 0 { + return Ok(Some(file)); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(ERROR_LOCK_VIOLATION) { + Ok(None) + } else { + Err(error) } } } @@ -147,6 +194,24 @@ mod tests { ); } + #[test] + fn is_held_never_creates_the_lock_file() { + let dir = tempfile::tempdir().expect("tempdir"); + assert!(!DaemonLock::is_held(dir.path()).expect("probe")); + assert!( + !dir.path().join(LOCK_FILE_NAME).exists(), + "a probe must not create daemon.lock" + ); + assert!(!DaemonLock::is_held(&dir.path().join("missing")).expect("missing dir is free")); + + let lock = DaemonLock::try_acquire(dir.path()) + .expect("acquire") + .expect("free lock"); + assert!(DaemonLock::is_held(dir.path()).expect("probe")); + drop(lock); + assert!(!DaemonLock::is_held(dir.path()).expect("probe")); + } + #[test] fn missing_state_dir_is_an_error() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/hyperdb-mcp/src/daemon/run.rs b/hyperdb-mcp/src/daemon/run.rs index 43572ce8..aaf66955 100644 --- a/hyperdb-mcp/src/daemon/run.rs +++ b/hyperdb-mcp/src/daemon/run.rs @@ -201,8 +201,8 @@ pub async fn run_daemon(config: DaemonConfig) -> Result<(), Box Result<(), Box {} - () = hyperd_monitor(Arc::clone(&state), Arc::clone(&hyper_state), Arc::clone(&info_arc)) => {} + () = hyperd_monitor(Arc::clone(&state), Arc::clone(&hyper_state), Arc::clone(&info_arc), &state_dir) => {} () = shutdown_signal() => { info!("received shutdown signal"); } @@ -259,7 +259,7 @@ pub async fn run_daemon(config: DaemonConfig) -> Result<(), Box, hyper_state: Arc>, info_arc: Arc>, + state_dir: &Path, ) { loop { tokio::time::sleep(HYPERD_POLL_INTERVAL).await; @@ -417,7 +418,7 @@ async fn hyperd_monitor( continue; } - match try_restart_hyperd(&hyper_state, &info_arc) { + match try_restart_hyperd(&hyper_state, &info_arc, state_dir) { Ok(new_endpoint) => { info!(endpoint = %new_endpoint, "hyperd restarted"); // Drain any reports that landed *during* the restart — those @@ -452,6 +453,7 @@ async fn hyperd_monitor( fn try_restart_hyperd( hyper_state: &Mutex, info_arc: &Mutex, + state_dir: &Path, ) -> Result { let mut guard = hyper_state.lock().expect("HyperState mutex poisoned"); @@ -475,8 +477,7 @@ fn try_restart_hyperd( guard.hyper = None; // Spawn the replacement. - let state_dir = discovery::state_dir().map_err(|e| RestartError::SpawnFailed(e.to_string()))?; - let params = build_params(&state_dir).map_err(|e| RestartError::SpawnFailed(e.to_string()))?; + let params = build_params(state_dir).map_err(|e| RestartError::SpawnFailed(e.to_string()))?; let new_hyper = HyperProcess::new(None, Some(¶ms)) .map_err(|e| RestartError::SpawnFailed(e.to_string()))?; // See `run_daemon`: publish the connectable `connection_endpoint_string()`, not @@ -503,7 +504,7 @@ fn try_restart_hyperd( // endpoint the file has not committed yet. The critical section is one // small serialize-and-rename, and the only contender is `status_json`, // whose own critical section is a single clone. - // `write_enriched_discovery_file` never touches `info_arc`, so holding it + // `write_enriched_discovery_file_in` never touches `info_arc`, so holding it // here cannot deadlock. // // Failing the write before mutating `DaemonInfo` also matters: the `?` @@ -513,7 +514,7 @@ fn try_restart_hyperd( let mut info_guard = info_arc.lock().expect("DaemonInfo mutex poisoned"); let mut snapshot = info_guard.clone(); snapshot.hyperd_endpoint.clone_from(&new_endpoint); - discovery::write_enriched_discovery_file(&snapshot) + discovery::write_enriched_discovery_file_in(state_dir, &snapshot) .map_err(|e| RestartError::SpawnFailed(format!("discovery write: {e}")))?; info_guard.hyperd_endpoint.clone_from(&new_endpoint); } diff --git a/hyperdb-mcp/src/diagnostics.rs b/hyperdb-mcp/src/diagnostics.rs index 84ffa26f..cbd41cfa 100644 --- a/hyperdb-mcp/src/diagnostics.rs +++ b/hyperdb-mcp/src/diagnostics.rs @@ -1066,9 +1066,11 @@ fn probe_daemon_lock(state_dir: &Path) -> DoctorLockState { Err(error) if error.kind() == io::ErrorKind::NotFound => return DoctorLockState::Absent, Err(_) => return DoctorLockState::Unknown, } - match crate::daemon::lock::DaemonLock::try_acquire(state_dir) { - Ok(Some(_lock)) => DoctorLockState::Free, - Ok(None) => DoctorLockState::Held, + // `is_held` opens without creating, so a lock file that vanished since + // the check above is not recreated by a read-only probe. + match crate::daemon::lock::DaemonLock::is_held(state_dir) { + Ok(false) => DoctorLockState::Free, + Ok(true) => DoctorLockState::Held, Err(_) => DoctorLockState::Unknown, } } diff --git a/hyperdb-mcp/src/main.rs b/hyperdb-mcp/src/main.rs index 4f898632..122b11dc 100644 --- a/hyperdb-mcp/src/main.rs +++ b/hyperdb-mcp/src/main.rs @@ -203,6 +203,10 @@ async fn run_daemon_mode(idle_timeout: Option) -> Result<(), Box daemon::control::HealthEndpoint { eprintln!("No daemon is currently running."); std::process::exit(1); }; + // The lock-based daemon is not running; a pre-1.0 one holds no lock, so + // say so when its record is there. + let no_daemon = || -> ! { + match discovery::legacy_daemon_hint(&dir) { + Some(hint) => eprintln!("No daemon is currently running. {hint}"), + None => eprintln!("No daemon is currently running."), + } + std::process::exit(1); + }; if let Some(info) = discovery::discover_in(&dir) && let Some(endpoint) = HealthEndpoint::from_record(&info.health_endpoint, &dir) { @@ -311,21 +324,15 @@ fn running_daemon_endpoint() -> daemon::control::HealthEndpoint { match daemon::state_perms::verify_state_dir_trusted(&dir) { Ok(()) => {} // No state directory at all: nothing has ever run here. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("No daemon is currently running."); - std::process::exit(1); - } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => no_daemon(), Err(e) => { eprintln!("The daemon state directory cannot be trusted: {e}"); std::process::exit(1); } } - match daemon::lock::DaemonLock::try_acquire(&dir) { - Ok(Some(_free)) => { - eprintln!("No daemon is currently running."); - std::process::exit(1); - } - Ok(None) => {} + match daemon::lock::DaemonLock::is_held(&dir) { + Ok(false) => no_daemon(), + Ok(true) => {} Err(e) => { eprintln!("Cannot check the daemon lock in {}: {e}", dir.display()); std::process::exit(1); diff --git a/hyperdb-mcp/tests/daemon_tests.rs b/hyperdb-mcp/tests/daemon_tests.rs index 80816fb8..60764b74 100644 --- a/hyperdb-mcp/tests/daemon_tests.rs +++ b/hyperdb-mcp/tests/daemon_tests.rs @@ -1251,6 +1251,41 @@ fn daemon_cli_status_ignores_a_stale_record_and_leaves_it() { ); } +/// A pre-1.0 daemon holds no lock and speaks TCP, so the 1.0 CLI cannot stop +/// it; `daemon stop` and `daemon status` say that it is recorded, with its pid +/// and port, instead of a bare "no daemon". +#[test] +fn daemon_cli_names_a_recorded_pre_1_0_daemon() { + let state = TempDir::new().unwrap(); + std::fs::write( + state.path().join("daemon.json"), + serde_json::json!({ + "pid": 424_242, + "hyperd_endpoint": "127.0.0.1:54321", + "health_port": 7485, + "started_at": "2026-08-13T12:34:56Z", + "version": "0.7.0", + }) + .to_string(), + ) + .unwrap(); + for action in ["stop", "status"] { + let output = run_cli_bounded(&["daemon", action], state.path(), Duration::from_secs(5)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(1), "{action}: {stderr}"); + assert!( + stderr.contains("pre-1.0 daemon") + && stderr.contains("424242") + && stderr.contains("7485"), + "{action}: {stderr}" + ); + } + assert!( + !state.path().join("daemon.lock").exists(), + "the CLI must not create daemon.lock" + ); +} + /// `daemon status` talks to the endpoint recorded in the state directory /// (selected by `HYPERDB_STATE_DIR`) and prints that daemon's own STATUS. #[test] From f22d123b1003c4f6b13c6194c0eeca4bb52658c9 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 17:01:59 -0700 Subject: [PATCH 38/61] chore: drop unpublished npm platform scaffolding and unused workspace deps Remove the linux-arm64-gnu and linux-x64-musl npm package stubs, which no workflow builds or publishes, and their references in the CI version guard, the artifact copier and the loader. Drop the unused root workspace dependencies parking_lot and regex. --- .github/workflows/ci.yml | 2 -- Cargo.toml | 2 -- hyperdb-api-node/index.js | 5 +++-- .../npm/linux-arm64-gnu/CHANGELOG.md | 8 ------- .../npm/linux-arm64-gnu/package.json | 21 ------------------- .../npm/linux-x64-musl/CHANGELOG.md | 8 ------- .../npm/linux-x64-musl/package.json | 21 ------------------- hyperdb-api-node/scripts/ci-copy-artifact.js | 2 -- 8 files changed, 3 insertions(+), 66 deletions(-) delete mode 100644 hyperdb-api-node/npm/linux-arm64-gnu/CHANGELOG.md delete mode 100644 hyperdb-api-node/npm/linux-arm64-gnu/package.json delete mode 100644 hyperdb-api-node/npm/linux-x64-musl/CHANGELOG.md delete mode 100644 hyperdb-api-node/npm/linux-x64-musl/package.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a73db7c2..ed637cf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,9 +423,7 @@ jobs: hyperdb-api-node/package.json hyperdb-api-node/npm/darwin-arm64/package.json hyperdb-api-node/npm/darwin-x64/package.json - hyperdb-api-node/npm/linux-arm64-gnu/package.json hyperdb-api-node/npm/linux-x64-gnu/package.json - hyperdb-api-node/npm/linux-x64-musl/package.json hyperdb-api-node/npm/win32-x64-msvc/package.json ) OFFENDERS=() diff --git a/Cargo.toml b/Cargo.toml index aaeeed28..25b60858 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,12 +66,10 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" url = "2.5" chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } -parking_lot = "0.12" # hyperdb-bootstrap dependencies zip = { version = "8", default-features = false, features = ["deflate"] } toml = "1.1" clap = { version = "4", features = ["derive"] } -regex = "1" anyhow = "1" # Lint configuration, applied to every workspace crate via `[lints] workspace = true`. diff --git a/hyperdb-api-node/index.js b/hyperdb-api-node/index.js index 6fd953dd..c9dce8a8 100644 --- a/hyperdb-api-node/index.js +++ b/hyperdb-api-node/index.js @@ -35,8 +35,9 @@ function getPlatformPackage() { // reliable again — see npm-build-publish.yml matrix. return arch === 'arm64' ? 'hyperdb-api-node-darwin-arm64' : null case 'linux': - if (arch === 'arm64') return isMusl() ? 'hyperdb-api-node-linux-arm64-musl' : 'hyperdb-api-node-linux-arm64-gnu' - return isMusl() ? 'hyperdb-api-node-linux-x64-musl' : 'hyperdb-api-node-linux-x64-gnu' + // Only linux-x64-gnu is published; ARM64 and musl have no prebuilt package. + if (arch === 'arm64' || isMusl()) return null + return 'hyperdb-api-node-linux-x64-gnu' case 'win32': return 'hyperdb-api-node-win32-x64-msvc' default: diff --git a/hyperdb-api-node/npm/linux-arm64-gnu/CHANGELOG.md b/hyperdb-api-node/npm/linux-arm64-gnu/CHANGELOG.md deleted file mode 100644 index 06184599..00000000 --- a/hyperdb-api-node/npm/linux-arm64-gnu/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -# Changelog - -## [0.1.3](https://github.com/tableau/hyper-api-rust/compare/v0.1.2...v0.1.3) (2026-05-18) - - -### Bug Fixes - -* v0.1.2 release — bump versions and add safety net ([#17](https://github.com/tableau/hyper-api-rust/issues/17)) ([bae4536](https://github.com/tableau/hyper-api-rust/commit/bae453600ce94ddc318ccb1cfe89be8fa32eef85)) diff --git a/hyperdb-api-node/npm/linux-arm64-gnu/package.json b/hyperdb-api-node/npm/linux-arm64-gnu/package.json deleted file mode 100644 index dd96922e..00000000 --- a/hyperdb-api-node/npm/linux-arm64-gnu/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "hyperdb-api-node-linux-arm64-gnu", - "os": [ - "linux" - ], - "cpu": [ - "arm64" - ], - "main": "hyperdb-api-node.linux-arm64-gnu.node", - "files": [ - "hyperdb-api-node.linux-arm64-gnu.node" - ], - "description": "hyperdb-api-node native binding for Linux ARM64 (glibc)", - "license": "MIT OR Apache-2.0", - "engines": { - "node": ">= 21" - }, - "libc": [ - "glibc" - ] -} \ No newline at end of file diff --git a/hyperdb-api-node/npm/linux-x64-musl/CHANGELOG.md b/hyperdb-api-node/npm/linux-x64-musl/CHANGELOG.md deleted file mode 100644 index 06184599..00000000 --- a/hyperdb-api-node/npm/linux-x64-musl/CHANGELOG.md +++ /dev/null @@ -1,8 +0,0 @@ -# Changelog - -## [0.1.3](https://github.com/tableau/hyper-api-rust/compare/v0.1.2...v0.1.3) (2026-05-18) - - -### Bug Fixes - -* v0.1.2 release — bump versions and add safety net ([#17](https://github.com/tableau/hyper-api-rust/issues/17)) ([bae4536](https://github.com/tableau/hyper-api-rust/commit/bae453600ce94ddc318ccb1cfe89be8fa32eef85)) diff --git a/hyperdb-api-node/npm/linux-x64-musl/package.json b/hyperdb-api-node/npm/linux-x64-musl/package.json deleted file mode 100644 index 1a2c6c8c..00000000 --- a/hyperdb-api-node/npm/linux-x64-musl/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "hyperdb-api-node-linux-x64-musl", - "os": [ - "linux" - ], - "cpu": [ - "x64" - ], - "main": "hyperdb-api-node.linux-x64-musl.node", - "files": [ - "hyperdb-api-node.linux-x64-musl.node" - ], - "description": "hyperdb-api-node native binding for Linux x64 (musl/Alpine)", - "license": "MIT OR Apache-2.0", - "engines": { - "node": ">= 21" - }, - "libc": [ - "musl" - ] -} \ No newline at end of file diff --git a/hyperdb-api-node/scripts/ci-copy-artifact.js b/hyperdb-api-node/scripts/ci-copy-artifact.js index 0ac27f62..1531ac7c 100644 --- a/hyperdb-api-node/scripts/ci-copy-artifact.js +++ b/hyperdb-api-node/scripts/ci-copy-artifact.js @@ -16,8 +16,6 @@ const TARGET_MAP = { 'aarch64-apple-darwin': { npm: 'darwin-arm64', lib: 'libhyperdb_api_node.dylib' }, 'x86_64-apple-darwin': { npm: 'darwin-x64', lib: 'libhyperdb_api_node.dylib' }, 'x86_64-unknown-linux-gnu': { npm: 'linux-x64-gnu', lib: 'libhyperdb_api_node.so' }, - 'x86_64-unknown-linux-musl': { npm: 'linux-x64-musl', lib: 'libhyperdb_api_node.so' }, - 'aarch64-unknown-linux-gnu': { npm: 'linux-arm64-gnu', lib: 'libhyperdb_api_node.so' }, 'x86_64-pc-windows-msvc': { npm: 'win32-x64-msvc', lib: 'hyperdb_api_node.dll' }, }; From 3ea87a1b876a6643a4f1d3750f48dae68baca20a Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 17:07:54 -0700 Subject: [PATCH 39/61] build: ship third-party license notices in the npm platform packages Generate THIRD-PARTY-LICENSES.txt for the hyperdb-mcp binary and the hyperdb-api-node addon with cargo-about at publish time, include it in each platform package, and add a CI job that fails when a dependency uses a license outside the accepted list. --- .github/workflows/ci.yml | 23 +++++++++++ .github/workflows/npm-build-publish.yml | 7 ++++ about.hbs | 27 +++++++++++++ about.toml | 39 +++++++++++++++++++ deny.toml | 3 +- hyperdb-api-node/CHANGELOG.md | 4 ++ .../npm/darwin-arm64/package.json | 1 + hyperdb-api-node/npm/darwin-x64/package.json | 1 + .../npm/linux-x64-gnu/package.json | 1 + .../npm/win32-x64-msvc/package.json | 1 + hyperdb-mcp/CHANGELOG.md | 7 ++++ hyperdb-mcp/npm/darwin-arm64/package.json | 1 + hyperdb-mcp/npm/darwin-x64/package.json | 1 + hyperdb-mcp/npm/linux-x64-gnu/package.json | 1 + hyperdb-mcp/npm/win32-x64-msvc/package.json | 1 + 15 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 about.hbs create mode 100644 about.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed637cf8..90d0ecba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -385,6 +385,29 @@ jobs: - run: cargo install cargo-audit --locked - run: cargo audit --deny warnings + third-party-licenses: + # Generates the THIRD-PARTY-LICENSES.txt that npm-build-publish.yml ships + # in the npm platform packages, so a dependency under an unaccepted + # license (about.toml) fails here rather than in the middle of a release. + name: third-party licenses + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions-rust-lang/setup-rust-toolchain@v2 + with: + toolchain: stable + cache-key: about + rustflags: "" + - run: cargo install cargo-about --locked --features cli --version 0.9.2 + - name: Generate the notices for both shipped binaries + run: | + set -euo pipefail + for crate in hyperdb-mcp hyperdb-api-node; do + cargo about generate --manifest-path "$crate/Cargo.toml" about.hbs \ + -o "$RUNNER_TEMP/THIRD-PARTY-LICENSES-$crate.txt" + done + version-consistency: # Ensures version.txt (release-please source of truth) stays in sync # with Cargo.toml workspace version. Catches release-please PRs that diff --git a/.github/workflows/npm-build-publish.yml b/.github/workflows/npm-build-publish.yml index 49959204..a998c2ee 100644 --- a/.github/workflows/npm-build-publish.yml +++ b/.github/workflows/npm-build-publish.yml @@ -176,6 +176,9 @@ jobs: MACOSX_DEPLOYMENT_TARGET: "13.0" run: cargo build --release --target "$TARGET" -p hyperdb-api-node + - name: Install cargo-about + run: cargo install cargo-about --locked --features cli --version 0.9.2 + - name: Download and verify hyperd env: SLUG: ${{ matrix.hyperd-slug }} @@ -262,6 +265,8 @@ jobs: cp "$f" "$DEST/" done cp LICENSE-HYPERD "$DEST/" + cargo about generate --manifest-path hyperdb-mcp/Cargo.toml about.hbs \ + -o "$DEST/THIRD-PARTY-LICENSES.txt" chmod +x "$DEST/hyperdb-mcp${EXE}" "$DEST/${HYPERD_BIN}" 2>/dev/null || true echo "Contents of $DEST:" ls -la "$DEST/" @@ -287,6 +292,8 @@ jobs: cp "$f" "$DEST/" done cp LICENSE-HYPERD "$DEST/" + cargo about generate --manifest-path hyperdb-api-node/Cargo.toml about.hbs \ + -o "$DEST/THIRD-PARTY-LICENSES.txt" chmod +x "$DEST/${HYPERD_BIN}" 2>/dev/null || true echo "Contents of $DEST:" ls -la "$DEST/" diff --git a/about.hbs b/about.hbs new file mode 100644 index 00000000..21076e2b --- /dev/null +++ b/about.hbs @@ -0,0 +1,27 @@ +Third-party software in this binary +=================================== + +This binary statically links the open-source Rust crates listed below. Each +crate is used under the license shown. The license texts follow the list. + +Crates by license +----------------- +{{#each overview}} +{{{name}}}: {{count}} +{{/each}} + +License texts +------------- +{{#each licenses}} +================================================================================ +{{{name}}} +================================================================================ + +Used by: +{{#each used_by}} + - {{{crate.name}}} {{{crate.version}}} +{{/each}} + +{{{text}}} + +{{/each}} diff --git a/about.toml b/about.toml new file mode 100644 index 00000000..0c2a3fb3 --- /dev/null +++ b/about.toml @@ -0,0 +1,39 @@ +# cargo-about configuration: generates THIRD-PARTY-LICENSES.txt for the two +# binaries that ship inside the npm platform packages (hyperdb-mcp and the +# hyperdb-api-node addon). Run from the workspace root: +# +# cargo about generate --manifest-path hyperdb-mcp/Cargo.toml about.hbs +# cargo about generate --manifest-path hyperdb-api-node/Cargo.toml about.hbs +# +# `accepted` mirrors the `[licenses] allow` list in deny.toml; keep the two in +# step. A dependency under any other license fails generation, so a new +# license cannot reach a published binary unnoticed. + +accepted = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "BSD-2-Clause", + "BSD-3-Clause", + "BSL-1.0", + "CC0-1.0", + "CDLA-Permissive-2.0", + "ISC", + "MPL-2.0", + "Unicode-3.0", + "Zlib", +] + +# The npm packages are published for these platforms only (see +# .github/workflows/npm-build-publish.yml); dependencies that exist only for +# other targets are not in the binaries and are not listed. +targets = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", +] + +# Dev- and build-only crates are not linked into the shipped binaries. +ignore-dev-dependencies = true +ignore-build-dependencies = true diff --git a/deny.toml b/deny.toml index c2635aa2..3b095401 100644 --- a/deny.toml +++ b/deny.toml @@ -20,7 +20,8 @@ feature-depth = 1 [licenses] # We distribute under a dual MIT OR Apache-2.0 license. The allowlist below # lists every SPDX identifier we accept from dependencies. Copyleft licenses -# (GPL, LGPL, AGPL, SSPL) are deliberately absent. +# (GPL, LGPL, AGPL, SSPL) are deliberately absent. `about.toml`'s `accepted` +# list must stay identical, or the third-party notices job disagrees with this one. allow = [ "Apache-2.0", "Apache-2.0 WITH LLVM-exception", diff --git a/hyperdb-api-node/CHANGELOG.md b/hyperdb-api-node/CHANGELOG.md index eb957b66..dfa5c73b 100644 --- a/hyperdb-api-node/CHANGELOG.md +++ b/hyperdb-api-node/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +* **The npm platform packages now ship `THIRD-PARTY-LICENSES.txt`,** listing + the licenses of the Rust crates linked into the native addon. It is + generated at publish time with `cargo-about` (`about.toml`, `about.hbs`), and + a CI job fails if a dependency uses a license outside the accepted list. * **TLS for TCP connections:** `ConnectionBuilder.tls({ mode, rootCert, clientCert, clientKey, serverName })`, with libpq `sslmode` semantics (`"disable"`, `"prefer"`, `"require"`, `"verify-ca"`, `"verify-full"`), and diff --git a/hyperdb-api-node/npm/darwin-arm64/package.json b/hyperdb-api-node/npm/darwin-arm64/package.json index 7f390e40..48176356 100644 --- a/hyperdb-api-node/npm/darwin-arm64/package.json +++ b/hyperdb-api-node/npm/darwin-arm64/package.json @@ -11,6 +11,7 @@ "hyperdb-api-node.darwin-arm64.node", "hyperd", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-api-node/npm/darwin-x64/package.json b/hyperdb-api-node/npm/darwin-x64/package.json index 014b5e87..b8ac7a36 100644 --- a/hyperdb-api-node/npm/darwin-x64/package.json +++ b/hyperdb-api-node/npm/darwin-x64/package.json @@ -11,6 +11,7 @@ "hyperdb-api-node.darwin-x64.node", "hyperd", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-api-node/npm/linux-x64-gnu/package.json b/hyperdb-api-node/npm/linux-x64-gnu/package.json index 0c8b0eaf..bc7a3c8c 100644 --- a/hyperdb-api-node/npm/linux-x64-gnu/package.json +++ b/hyperdb-api-node/npm/linux-x64-gnu/package.json @@ -11,6 +11,7 @@ "hyperdb-api-node.linux-x64-gnu.node", "hyperd", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-api-node/npm/win32-x64-msvc/package.json b/hyperdb-api-node/npm/win32-x64-msvc/package.json index 355ec6c1..48a9c83a 100644 --- a/hyperdb-api-node/npm/win32-x64-msvc/package.json +++ b/hyperdb-api-node/npm/win32-x64-msvc/package.json @@ -11,6 +11,7 @@ "hyperdb-api-node.win32-x64-msvc.node", "hyperd.exe", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index ab064d3f..39b141eb 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- **The npm platform packages now ship `THIRD-PARTY-LICENSES.txt`,** listing + the licenses of the Rust crates linked into the `hyperdb-mcp` binary. It is + generated at publish time with `cargo-about` (`about.toml`, `about.hbs`), and + a CI job fails if a dependency uses a license outside the accepted list. + ### Changed - **Upgraded the `rmcp` SDK dependency from 1.x to 3.4.** Resolves the diff --git a/hyperdb-mcp/npm/darwin-arm64/package.json b/hyperdb-mcp/npm/darwin-arm64/package.json index 55e43516..d110e1d7 100644 --- a/hyperdb-mcp/npm/darwin-arm64/package.json +++ b/hyperdb-mcp/npm/darwin-arm64/package.json @@ -11,6 +11,7 @@ "hyperdb-mcp", "hyperd", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-mcp/npm/darwin-x64/package.json b/hyperdb-mcp/npm/darwin-x64/package.json index 92779353..7b1e3256 100644 --- a/hyperdb-mcp/npm/darwin-x64/package.json +++ b/hyperdb-mcp/npm/darwin-x64/package.json @@ -7,6 +7,7 @@ "hyperdb-mcp", "hyperd", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-mcp/npm/linux-x64-gnu/package.json b/hyperdb-mcp/npm/linux-x64-gnu/package.json index 2dbf5f2c..32d813c2 100644 --- a/hyperdb-mcp/npm/linux-x64-gnu/package.json +++ b/hyperdb-mcp/npm/linux-x64-gnu/package.json @@ -14,6 +14,7 @@ "hyperdb-mcp", "hyperd", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" diff --git a/hyperdb-mcp/npm/win32-x64-msvc/package.json b/hyperdb-mcp/npm/win32-x64-msvc/package.json index a83eef12..a29717a8 100644 --- a/hyperdb-mcp/npm/win32-x64-msvc/package.json +++ b/hyperdb-mcp/npm/win32-x64-msvc/package.json @@ -11,6 +11,7 @@ "hyperdb-mcp.exe", "hyperd.exe", "LICENSE-HYPERD", + "THIRD-PARTY-LICENSES.txt", "LICENSE-MIT.txt", "LICENSE-APACHE.txt", "NOTICE" From 3072dd4bd2eb63801b7d9ccfc7845586851d4dd2 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 17:48:16 -0700 Subject: [PATCH 40/61] docs(mcp): rewrite get_readme in compact schema notation Tool signatures now read `tool(required, optional?=default) -> result` with a legend up front, and prose becomes terse fragments. The document shrinks from 23.9 KB to 21.5 KB while adding facts verified against the tool parameter structs: chart size clamps and format inference, export format_options and sql/table precedence, attach on_missing, copy_query temp_attach, save_query duplicate rejection, the watch_directory .ready sentinel protocol, and the kv_get / kv_pop / kv_set_many response shapes. --- hyperdb-mcp/CHANGELOG.md | 7 + hyperdb-mcp/src/readme.rs | 691 ++++++++++++++------------------------ 2 files changed, 263 insertions(+), 435 deletions(-) diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 39b141eb..ebc272a4 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -16,6 +16,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **`get_readme` is now written in a compact schema notation.** Tool + signatures read `tool(required, optional?=default) → result`, with a legend + at the top. The document is about 10% smaller yet carries more: per-tool + parameter defaults and limits, `chart` size clamps, export `format_options`, + `attach_database` `on_missing`, `copy_query` `temp_attach`, the + `watch_directory` `.ready` sentinel protocol, and `kv_get` / `kv_pop` + response shapes. - **Upgraded the `rmcp` SDK dependency from 1.x to 3.4.** Resolves the outstanding `rmcp` security advisories. Purely an internal dependency bump — the MCP wire protocol and tool surface exposed by this server are unchanged. diff --git a/hyperdb-mcp/src/readme.rs b/hyperdb-mcp/src/readme.rs index d8e56c42..5f3e56cb 100644 --- a/hyperdb-mcp/src/readme.rs +++ b/hyperdb-mcp/src/readme.rs @@ -3,408 +3,270 @@ //! LLM-facing README returned by the `get_readme` tool. //! -//! Structured as: purpose → tool index → parameter rules → SQL quirks → -//! examples. Optimized for token efficiency: every sentence earns its -//! place. When tools or features change, update this string and the -//! `readme_tests.rs` coverage will fail loudly if a tool name was missed. +//! Written in a compact pseudo-code / schema notation (legend at the top of +//! the string): tool signatures as `tool(required, optional?=default) → +//! result`, `a|b` alternatives, `*` defaults, and terse fragments instead of +//! prose. The notation buys room for more facts within the byte budget that +//! `readme_tests.rs` enforces. Order: purpose → database model → chart → +//! tool index → parameter rules → SQL quirks → examples → tips. When tools +//! or features change, update this string; `readme_tests.rs` fails loudly if +//! a tool name or a pinned contract phrase goes missing. -pub const README: &str = "\ -# HyperDB MCP +pub const README: &str = r##"# HyperDB MCP -## What this is +In-process SQL analytics on the Tableau Hyper engine. In: CSV|JSON|JSONL|Parquet|Arrow IPC|Apache Iceberg. +SQL: PostgreSQL-compatible, Salesforce Data Cloud dialect. Out: Parquet|Iceberg|Arrow IPC|CSV|.hyper. +Use for any tabular analysis, SQL, file transform, or chart; prefer over ad-hoc Python/shell (faster +parsing, native SQL, state stays re-queryable without re-loading). -HyperDB MCP is an in-process SQL analytics service powered by the Tableau -Hyper database engine. Load data from CSV / JSON / JSONL / Parquet / -Arrow IPC / Apache Iceberg, query with PostgreSQL-compatible SQL -(Salesforce Data Cloud SQL dialect), and export results to Parquet, -Iceberg, Arrow IPC, CSV, or .hyper. - -## When to use this MCP - -Whenever the user asks to analyze tabular data, run SQL, transform a -file, or build a chart from a query. Prefer this MCP over ad-hoc Python -or shell pipelines: it parses files faster, runs SQL natively, and keeps -intermediate state in a local database the LLM can re-query without -re-loading. +Notation: `tool`(required, optional?) → result · `a|b` one of · `*` or `?=v` default · `≡` same as · ✓ supported · +✗ unsupported · ⚠ gotcha. File paths absolute; relative rejected. ## Database model — queryable memory -Every session has TWO databases, plus optional user-attached ones: - -- **Local database** (ephemeral and the default destination). Created - fresh per session, deleted on exit, and addressed as `\"local\"`. - Unqualified SQL routes here; use it for scratch data. -- **Persistent database** (alias `\"persistent\"`). Survives across - sessions as queryable long-term structured memory. Disabled with - `--ephemeral-only`. -- **Attached databases** via `attach_database`. Each lives in its own - `.hyper` file under a user-chosen canonical lowercase alias. Attachments - are read-only by default; pass `writable: true` when writes are needed. - -### Persistent as memory - -Use persistent for reference data, preferences, accumulated results, or -anything the user asks to remember or query in a future session. - ``` -load_data({ table: \"project_decisions\", data: \"[...]\", persist: true }) -query({ sql: \"SELECT * FROM project_decisions\", database: \"persistent\" }) +local*: ephemeral (fresh per session, deleted on exit); unqualified SQL lands here; scratch data +persistent: survives sessions = long-term structured memory (reference data, preferences, accumulated + results, anything to remember or query later); off under --ephemeral-only +: attach_database → user-chosen name for its own .hyper file, canonical lowercase alias; read-only unless writable: true +remember: few scraps (variables, flags, summaries, JSON, queue entries) → kv_*, no schema/DDL · rows to + filter/JOIN/aggregate → table (load_data | execute CREATE TABLE) · both local (lost on restart) + unless database: "persistent" +database: "local"* | "persistent" | , case-insensitive; writes need a writable attachment. On: + query execute load_data load_file load_files watch_directory describe sample chart export + set_table_metadata kv_* +persist: true ≡ database: "persistent" (database wins if both set). On: load_data load_file load_files watch_directory kv_* +execute: database only · load_iceberg: always local · copy_query: target_database +fully-qualified SQL (power users; prefer database where a tool takes it): + INSERT INTO "persistent"."public"."customers" SELECT ... +e.g. load_data({table: "project_decisions", data: "[...]", persist: true}) + query({sql: "SELECT * FROM project_decisions", database: "persistent"}) ``` -### KV store vs. a custom table — which to remember with - -When you need to remember something, pick the lighter tool: - -- **A few scraps** — variables, flags, summaries, JSON, queue entries — - use the key-value store (`kv_set` / `kv_get`); no schema or DDL. -- **Structured rows** you'll filter, JOIN, or aggregate — use a real - table (`load_data` / `execute CREATE TABLE`), so SQL can reason over - typed columns. - -Both persist the same way: default is the local database (lost on -restart); pass `database: \"persistent\"` to keep either one across -sessions. The KV tools, `load_data`, `load_file`, `load_files`, and -`watch_directory` also accept the `persist: true` shorthand; `execute` -takes `database` only, and `load_iceberg` always loads into local. - -### Routing data to a destination - -- **`database` parameter** (preferred for tools that build their own - SQL): `query`, `execute`, `load_data`, `load_file`, `load_files`, - `watch_directory`, `describe`, `sample`, `chart`, `export`, and - `set_table_metadata` accept `database: \"persistent\"`, - `database: \"local\"`, or any user-attached alias (write tools require - a writable attachment). Case-insensitive. Defaults to local. -- **`persist: true` shorthand** on `load_data`, `load_file`, - `load_files`, `watch_directory` — equivalent to - `database: \"persistent\"`. -- **Fully-qualified SQL** for power users: - `INSERT INTO \"persistent\".\"public\".\"customers\" SELECT ...` - ### Chart delivery and presentation -Long and Unicode labels are not truncated or auto-sized; if they clip, -increase `width` or `height`. -Log bars start at the positive lower bound, not zero. - -`chart` is a bounded quick diagnostic, not a dashboard system. With no -`output_path`, its PNG (default) or SVG is returned `inline` and no file -is written. Supplying `output_path` writes the file and still returns it -inline; set `inline=false` for disk-only output (an omitted path then gets -an auto-generated temp file). An existing file needs `overwrite=true`. -The path extension and explicit `format`, when both supplied, -must agree. Use `database` to route the SQL to local, persistent, or an -attached database. - -Bars are vertical by default; set `bar_orientation` to `horizontal` for -rankings. `label_values=true` writes each original y scalar beside its -bar. `show_legend` defaults to true; set it false to suppress the legend. -With a `series` column, `color_map` maps series names to hex colors; -`label_points=true` labels line/scatter points and suppresses their legend. -`y_scale` defaults to `linear`; `log` -applies to the data-role y measure, including the physical x axis of -horizontal bars. Log values and ranges must be finite and strictly -positive (> 0): zero and negative values are invalid, logarithmic -histograms are unsupported, and an explicit range must contain every -plotted value. Explicit `x_range` and `y_range` bounds must also be finite, -strictly increasing, and representable. - -`x_range` means something different per chart type. On line/scatter it sets -the axis bounds only: out-of-range points are still counted in -`rows_plotted` but render invisibly (clipped), never moved. On a histogram -it is the binning extent, and values outside it are **excluded from the -bins** rather than folded into the edge bins, so `rows_plotted` counts only -the values actually binned and `excluded_out_of_range` appears in the stats -block reporting how many were dropped. A histogram `x_range` containing no -data renders empty at the requested extent instead of erroring. Bar charts -ignore `x_range` entirely (their x positions are categorical). `bins` is -clamped to 1..=500. - -Line/scatter DATE, TIMESTAMP, and TIMESTAMPTZ x values use a proportional -time axis automatically. TEXT is categorical; set `x_as_category=true` -to deliberately give temporal observations even spacing. Bars always -treat x as categorical. - -Every successful database-routed tool response carries canonical -`resolved_database`: `\"local\"`, `\"persistent\"`, or the lowercase attached -alias. `copy_query` also retains `target_database`. +`chart`(sql, chart_type: bar|line|scatter|histogram, x?, y?, series?, title?, format?: png|svg (else +from output_path extension, else png), +width?=800, height?=480, bins?=20, ...) — a bounded quick diagnostic, not a dashboard system. +Long-format data: numeric y, optional series. x, y required for bar|line|scatter; histogram bins x. +``` +delivery: no output_path → PNG (default) or SVG returned inline, no file · output_path → file written + (parents created) and returned inline · inline=false → disk only (no path ⇒ auto-generated temp + file) · existing file → PERMISSION_DENIED unless overwrite=true; an open database is never replaced · + explicit format and path extension, if both given, must agree (.png|.svg only) +database: routes the SQL to local|persistent|attached +size: width 200-4096, height 150-4096 (clamped). Long and Unicode labels are not truncated or + auto-sized; if they clip, increase width or height +bar_orientation: vertical* | horizontal (rankings); bars only +label_values=true: each original y scalar written beside its bar +show_legend: true by default; false to suppress the legend +color_map: {series name → "#rrggbb"}, needs series; unlisted series use the default palette +label_points=true: labels line/scatter points, suppresses their legend +y_scale: linear (default) | log, on the data-role y measure (horizontal bars: physical x axis). Log + values and ranges: finite, strictly positive (> 0); zero and negative invalid; logarithmic + histograms unsupported; an explicit range must contain every plotted value. Log bars start at the + positive lower bound, not zero +x_range, y_range: [min, max]; finite, strictly increasing, representable; omit to auto-scale +x_range: line|scatter → axis bounds only; out-of-range points still counted in rows_plotted, rendered + invisibly (clipped), never moved · histogram → binning extent; outside values excluded from the bins + (not folded into edge bins), rows_plotted counts binned values only, stats.excluded_out_of_range = + dropped count, an x_range containing no data renders empty at that extent (no error) · bar → ignored (categorical x) +bins: clamped 1..=500 +x axis: line|scatter DATE|TIMESTAMP|TIMESTAMPTZ → proportional time axis (automatic) · numeric → + numeric · TEXT → categorical · x_as_category=true → even spacing for temporal observations · bars + always categorical +``` + +Every successful database-routed tool response carries canonical `resolved_database`: "local", +"persistent", or the lowercase attached alias. `copy_query` also retains `target_database`. ## Tool index ### Query -- `query` — run a read-only SELECT / WITH / EXPLAIN / SHOW / VALUES. - Max 10,000 rows; check `truncated` / `total_rows`. -- `execute` — run one or more DDL/DML statements as an atomic batch. - `sql` is an array; multi-element batches run inside a transaction - (all commit or all roll back). Disabled in read-only mode. -- `query_data` — ingest inline JSON or CSV and run one SQL query in a - single call (table is temporary). -- `query_file` — same as `query_data` but reads from a file path. The - fastest path when the user asks \"what's in this file?\". +- `query`(sql, database?) → rows; read-only SELECT|WITH|EXPLAIN|SHOW|VALUES; max 10,000 rows (check + `truncated`, `total_rows`) +- `execute`(sql: [stmt, ...], database?) — DDL/DML; several elements = one transaction (all commit or + all roll back), one = auto-commit. Disabled in read-only mode +- `query_data`(data, sql, format?: json|csv, table_name?="data", schema?) — inline JSON array or CSV + + one query, temporary table; format auto-detected (`[`/`{` → JSON, else CSV) +- `query_file`(path, sql, table_name?="data", schema?, json_extract_path?) — same for a CSV|JSON|JSONL| + Parquet|Arrow IPC file; fastest answer to "what's in this file?" +- `schema`: partial override `{"col": "BIGINT"}`, rest inferred · `json_extract_path`: dot path to a + nested JSON array, numeric segments index arrays (`content.0`), stringified JSON parsed en route ### Load - -**Format preference (ingest and export):** Parquet (fastest, server-side, -preserves every type incl. NUMERIC precision / DATE / TIMESTAMP — best for -large data) > Arrow IPC (very fast, no compression; schema overrides -rejected since its schema is authoritative) > CSV (portable but types are -inferred on load / lost on export) > JSON / JSONL (small or irregular data -only — parsed row-by-row). Iceberg is a directory of Parquet for data-lake -interop; `.hyper` export snapshots every table for Tableau. - -- `load_file` — load one CSV / JSON / JSONL / Parquet / Arrow IPC file - into a named database table. `mode`: `replace` (default) / - `append` / `merge`. Use `merge` to upsert by `merge_key` (column - name or list); new columns in the incoming file are auto-added via - `ALTER TABLE`. -- `load_files` — load many files in parallel; each entry - `{path, table, mode?}` loads into its own table. `merge` mode is - not supported here — call `load_file` per-file if you need merge. -- `load_data` — load inline JSON / CSV into a named database table. -- `load_iceberg` — load an Apache Iceberg table by absolute path to its - root directory; supports snapshot pinning via `metadata_filename` or - `version_as_of`. +Format preference (ingest and export): Parquet (fastest, server-side, keeps every type incl. NUMERIC +precision/DATE/TIMESTAMP; large data) > Arrow IPC (very fast, uncompressed; authoritative schema, so +overrides rejected) > CSV (portable; types inferred on load, lost on export) > JSON|JSONL (small or +irregular only; row-by-row). Iceberg: directory of Parquet, data-lake interop. `.hyper` export: +snapshot of every table, for Tableau. +- `load_file`(path, table, mode?: replace*|append|merge, merge_key?, schema?, json_extract_path?, + database?, persist?) — one CSV|JSON|JSONL|Parquet|Arrow IPC file → table. merge: upsert by + `merge_key` (column or list); new incoming columns auto-added via `ALTER TABLE` (Parameter rules) +- `load_files`(files: [{path, table, mode?: replace*|append, schema?, json_extract_path?}], + concurrency?, database?, persist?) — parallel, one table each. ⚠ No merge: `load_file` per file +- `load_data`(table, data, format?: json|csv, mode?: replace*|append, schema?, database?, persist?) — + inline JSON/CSV → table; replace drops and recreates +- `load_iceberg`(path, table, mode?: replace*|append, metadata_filename?, version_as_of?) — Apache + Iceberg table root directory (`metadata/`, `data/`); pin a snapshot by `metadata_filename` (e.g. + "v2.metadata.json"; latest*) or `version_as_of` ### Inspect -- `describe` — list local tables (no args) or describe one table - (`table` arg) with columns, types, row count, and prose metadata. - **Defaults to the local database** — pass `database: \"persistent\"` - (or an attached alias) to list/inspect durable tables. `status` reports - table *counts* only, never names, so check the right database here - before assuming a persistent table is missing. -- `sample` — return schema + first N rows of a table. Use this before - writing a non-trivial query. -- `inspect_file` — dry-run schema inference on a CSV / Parquet / Arrow - IPC / JSON file without loading it. -- `status` — plugin and native/API identity; daemon/Hyper connection facts; - local/persistent paths; table count; disk usage; watchers; attachments; - read-only flag. - `engine.connection` gives the `hyperd` endpoint in the forms another - Hyper client needs: `transport` (`tcp` / `unix_domain_socket` / - `named_pipe`), `host` + `port` (TCP only, else null), `socket_path` - (IPC only, else null), and `connection_descriptor` — the scheme- - qualified string `hyperd` emits and the Hyper API accepts, e.g. - `tab.tcp://host:port`. The shared daemon uses IPC; a private - `hyperd` (`--no-daemon` or daemon fallback) uses TCP. - Both full and degraded responses report `default_database: \"local\"`. - When - `engine_busy: true`, the response is partial and non-definitive: - `hyperd_running: false` is inconclusive; retry `status` for full - statistics after the in-progress operation completes. +- `describe`(table?, database?) — no table → list tables; table → columns, types, row count, prose + metadata. ⚠ Local by default: pass `database: "persistent"` (or an alias) for durable tables; + `status` gives table counts, never names, so check here before assuming a table is missing +- `sample`(table, n?, database?) → schema + first N rows; use before a non-trivial query +- `inspect_file`(path, sample_rows?, json_extract_path?) — dry-run schema inference on a CSV|Parquet| + Arrow IPC|JSON file, no load +- `status`() → plugin and native/API identity; daemon/Hyper connection facts; local/persistent paths; + table count; disk usage; watchers; attachments; read-only flag; `default_database: "local"` (full or + degraded). `engine.connection`, the `hyperd` endpoint for another Hyper client: `transport` + tcp|unix_domain_socket|named_pipe · `host`, `port` (TCP only, else null) · `socket_path` (IPC only, + else null) · `connection_descriptor`, the scheme-qualified string `hyperd` emits and the Hyper API + accepts (`tab.tcp://host:port`). Shared daemon → IPC; private `hyperd` (`--no-daemon` or daemon + fallback) → TCP. ⚠ `engine_busy: true` → partial, non-definitive response; `hyperd_running: false` + is inconclusive; retry `status` for full statistics after the in-progress operation completes ### Export -- `export` — write a table or query result to a file (Parquet, Iceberg, - Arrow IPC, CSV, .hyper). Replaces a file only with `overwrite: true`, - never an open database. A `.hyper` export leaves the source unchanged - and writes every user table to the destination — a faithful backup: - NOT NULL, DEFAULT, COLLATE, ASSUMED PRIMARY KEY, and ASSUMED UNIQUE all - survive (Hyper never accepts enforced PRIMARY KEY / UNIQUE / FOREIGN - KEY / CHECK at CREATE TABLE, so no source table carries those). The response carries a `schema_fidelity` - object (`fully_preserved` plus per-class counts and an `unpreserved` - list naming each `table` + `column`) — check it before trusting an - export as a backup. -- `chart` — render a bar / line / scatter / histogram PNG or SVG from a - SQL query as a quick diagnostic. Use long-format data (numeric y; - optional `series` grouping). See `Chart delivery and presentation`. -- `copy_query` — run a SELECT across local + attached databases and - insert the result into a target table (`mode`: `create`, `append`, - `replace`). Cross-database analytics in one tool call. +- `export`(path, format: parquet|iceberg|arrow_ipc|csv|hyper, sql? | table?, overwrite?, format_options?, + database?) — table or query result → file (iceberg: directory); `sql` or `table` required except + hyper (exports everything, ignores both); both given → `sql` wins. Existing destination → + PERMISSION_DENIED unless `overwrite: true`; iceberg replaces only an Iceberg table directory, hyper + only a `.hyper` file, never an open database. `database` = source; sql-mode unqualified names resolve + there. `.hyper` export leaves the source unchanged and writes every user table to the destination, a + faithful backup: NOT NULL, DEFAULT, COLLATE, ASSUMED PRIMARY KEY, ASSUMED UNIQUE survive (Hyper never + accepts enforced PRIMARY KEY/UNIQUE/FOREIGN KEY/CHECK at CREATE TABLE, so no source table has them). + Check `schema_fidelity` {`fully_preserved`, per-class counts, `unpreserved`: [{`table`, `column`}]} + before trusting an export as a backup +- `format_options` → hyperd `COPY ... WITH (...)`; exact hyperd names, scalar values; ignored for hyper. + parquet: `codec` snappy*|zstd|gzip|uncompressed|…, `rows_per_row_group` · iceberg: parquet's + + `table_scheme` metastore*|filesystem, `max_file_size` (bytes) · csv: `header` (true*), `delimiter` + (","*), `null` (""*), `quote` +- `chart` — see Chart delivery and presentation +- `copy_query`(sql, target_table, mode: create|append|replace, target_database?, temp_attach?: + [attach_database args]) — SELECT|WITH|VALUES across local + attached (qualified: `src.public.t`) → + `public` schema of `target_database` (local*; else writable). `temp_attach`: this call only, + auto-detached even on failure; aliases must be unused ### Saved queries & metadata -- `save_query` — save a named read-only SQL query for later reuse. -- `delete_query` — delete a named saved query. -- `set_table_metadata` — update prose metadata (source_url, purpose, - notes, license, source_description) on an existing table catalog entry. - Local and persistent tables share one name-keyed persistent catalog; - writable user-attached databases have per-database catalogs. +- `save_query`(name, sql, description?) — named read-only query → resources + `hyper://queries/{name}/definition`, `.../result` (re-runs on read); survives restarts unless + `--ephemeral-only`; duplicate name rejected (`delete_query` first) +- `delete_query`(name) — missing name: no-op → `{"deleted": false}` +- `set_table_metadata`(table, source_url?, purpose?, notes?, license?, source_description?, data_url?, + database?) — prose metadata on an existing table catalog entry; omitted unchanged, "" clears. Local + and persistent tables share one name-keyed persistent catalog; writable user-attached databases have + per-database catalogs ### Multi-database -- `attach_database` — attach an additional .hyper database under an - alias. Pass `writable: true` to allow writes through it. -- `detach_database` — detach a previously attached database. -- `list_attached_databases` — list current attachments. +- `attach_database`(alias, path, writable?=false, on_missing?: error*|create, kind?: local_file*) — + alias: SQL identifier, not `local`. `writable: true` allows writes through it. `on_missing: "create"` + makes an empty file (needs `writable: true`; parent directory must exist) +- `detach_database`(alias) · `list_attached_databases`() → current attachments ### Directory watching -- `watch_directory` — watch a directory and auto-ingest matching files - as they appear. -- `unwatch_directory` — stop watching a previously registered - directory. +- `watch_directory`(path, table, max_concurrent?=4 (≤32), database?, persist?) — producer writes + `x.csv` atomically (tmp + rename), then empty `x.csv.ready`; watcher appends x.csv to `table`, + deletes both on success; on failure moves both to `failed/` + `x.csv.error` JSON (no retry) +- `unwatch_directory`(path) — stop ### Key-value store (scratchpad) -- `kv_set` — save a variable / state / summary / JSON string under a - store + key. Returns `{stored, created, value_bytes}`. Pass - `overwrite: false` to skip writes that would clobber an existing key - (response: `{stored: false, existed: true}`). Pass - `value_path: ` to store a file's contents server-side - instead of inlining `value` (exactly one of `value` / `value_path`; - reads any path the server process can read — no sandbox; files over - 64 MiB are rejected before reading). -- `kv_set_many` — atomic batch write. Pass an `entries` array of - `{key, value}` objects. All keys validated up front; an invalid key - aborts the whole batch without writing anything. `overwrite: false` - skips existing keys. Returns counts plus `total_bytes`, which counts - submitted values and may exceed bytes persisted when entries skip. -- `kv_get` — read a value by store + key. -- `kv_delete` — delete a key. -- `kv_list` — list keys in a store. Pass `values: true` to return - `{entries: [{key, value}, ...]}` instead of `{keys: [...]}` — reads - the whole store in one call (eliminates N×`kv_get`). -- `kv_list_stores` — list store namespaces that hold data in a database. -- `kv_size` — count keys and total value bytes in a store. Returns - `{size, bytes}`. -- `kv_pop` — destructively read-and-remove the lowest-keyed entry - (lexicographic key order, not insertion order). -- `kv_clear` — delete all keys in a store. - -Every kv_* tool takes the same optional `database` parameter as the data -tools. Omit it and the store lives in the local database (lost on -restart); pass `\"persistent\"` (or `persist: true`) to persist across -restarts, or any attached alias to target that database. Every user-attached -target must be writable, even for readers, because the backing table may -need initialization. The global `--read-only` guard blocks the five KV -mutators but not the four readers. Each database has its own isolated set -of stores. Enrich analytical tables with KV -metadata via LEFT JOIN — always filter `kv.store_name = ''` -to avoid row multiplication, and keep the KV table in the same database -as the joined table. See the `hyper://schema/kv` resource for the join -template, and `KV store vs. a custom table` above for when to reach for -this instead of a real table. - -**Querying JSON in a KV value:** Values are TEXT. To query JSON -structure, cast first: `SELECT value::json ->> 'field' FROM -_hyperdb_kv_store WHERE store_name = '...'`. The `->` / `->>` / -`JSON_EACH(...)` operators work AFTER the `::json` cast. Applying `->` -or `->>` to raw TEXT fails with SQLSTATE 42601 (\"requires a structured -data type\"). `JSON_VALUE(...)` is not implemented in this engine — use -the `::json` cast instead. - -**::numeric truncation gotcha:** A bare `::numeric` cast defaults to -scale 0 and truncates decimal places. Example: `41.54178215::numeric` -→ `42`. Always specify precision and scale: `::numeric(20,10)`. +- `kv_set`(store, key, value | value_path, overwrite?=true) → `{stored, created, value_bytes}`; + `overwrite: false` + existing key → `{stored: false, existed: true}`. `value_path`: absolute path, + contents stored server-side instead of `value` (exactly one of the two; any path the server process + can read, no sandbox; over 64 MiB rejected before reading) +- `kv_set_many`(store, entries: [{key, value}], overwrite?=true) — atomic batch; keys validated up + front, one invalid key aborts it, nothing written; `overwrite: false` skips existing keys → + `{stored, created, overwritten|skipped, total_bytes}` (submitted values; may exceed bytes persisted + when entries skip) +- `kv_get`(store, key) → `{found, value}` · `kv_delete`(store, key) · `kv_clear`(store) — delete all keys +- `kv_list`(store, values?) → `{keys: [...]}`; `values: true` → `{entries: [{key, value}, ...]}`, whole + store in one call (no N×`kv_get`) +- `kv_list_stores`() → store namespaces holding data in a database · `kv_size`(store) → `{size, bytes}` +- `kv_pop`(store) → `{found, key, value}` — destructive read-and-remove of the lowest-keyed entry (lexicographic key order, not + insertion order) + +Stores appear on first write. Every kv_* tool also takes `database?`/`persist?`: local* (lost on +restart); "persistent" or `persist: true` survives; an attached alias targets that database. Every +user-attached target must be writable, even for readers (the backing table may need initialization). +`--read-only` blocks the five KV mutators, not the four readers. Each database has its own isolated +stores. Enrich tables via LEFT JOIN: always filter `kv.store_name = ''` (else rows +multiply); keep the KV table in the joined table's database; template: `hyper://schema/kv` resource. + +**Querying JSON in a KV value:** values are TEXT; cast first: `SELECT value::json ->> 'field' FROM +_hyperdb_kv_store WHERE store_name = '...'`. `->`, `->>`, `JSON_EACH(...)` work after `::json`; on raw +TEXT → SQLSTATE 42601 ("requires a structured data type"). `JSON_VALUE(...)` not implemented; cast. + +**`::numeric` gotcha:** bare `::numeric` is scale 0, drops decimals: `41.54178215::numeric` → `42`. +Give precision and scale: `::numeric(20,10)`. ### Introspection -- `get_readme` — this document. Call once at the start of a session. +- `get_readme`() — this document; call once at session start ## Parameter rules -- **File paths must be absolute.** Relative paths are rejected. -- **Identifiers fold to lowercase** unless double-quoted. `SELECT * FROM - Sales` reads `sales`. Use `\"Sales\"` to preserve case. -- **Query SQL is read-only** in every tool that takes it (`query`, - `query_data`, `query_file`, `export`, `chart`, ...): one SELECT / WITH / - EXPLAIN / SHOW / VALUES, even without `--read-only`. A write behind - `WITH` or `EXPLAIN (ANALYZE)` is refused. For DDL / DML use `execute`. -- **Read-only mode** (`--read-only` flag on the server) guards exactly: - `execute`, `load_data`, `load_file`, `load_files`, `load_iceberg`, - `watch_directory`, `save_query`, `delete_query`, `set_table_metadata`, - `copy_query`, `kv_set`, `kv_set_many`, `kv_delete`, `kv_pop`, and - `kv_clear`. A writable `attach_database` or `on_missing: \"create\"` is - also guarded, while a read-only attachment remains available. - Queries and inspection, `chart`, and detach/list operations remain - available. unwatch_directory remains allowed; export formats, including - Hyper, remain allowed. Hyper export does not mutate its source database, - but it does create or replace its materialized destination file. -- **Persistent-file contention:** only a reserved persistent attachment - lock is reported as `RESOURCE_BUSY`. Run `hyperdb-mcp doctor`, compare - client/daemon identities, close the possible owner (Hyper, Tableau, or - another process), or copy/select another `.hyper` file, then retry. -- **Table names** in `load_*` and `query_data` / `query_file` accept - unquoted identifiers; the server lowercases them. -- **`copy_query` modes:** `create` requires the target not exist; - `append` requires it does; `replace` drops and recreates atomically. -- **`load_file` merge mode:** `mode = \"merge\"` requires `merge_key` - (column name or list of column names). Rows whose key matches are - deleted and replaced by the incoming row (target columns absent from - the file become NULL; duplicate incoming keys all insert); - non-matching rows INSERT. Columns present in - the incoming file but not the target are auto-added via - `ALTER TABLE ADD COLUMN` (nullable; existing rows fill with NULL). - **Type changes on existing columns are rejected** — use `replace` - or apply a `schema` override. The DELETE+INSERT pair is not - transactional (Hyper auto-commits DDL); a mid-run failure leaves - partial state, same as `replace`. +- Identifiers fold to lowercase unless double-quoted: `SELECT * FROM Sales` reads `sales`; `"Sales"` + keeps case. Table names in `load_*`, `query_data`, `query_file`: unquoted identifiers, lowercased. +- Query SQL is read-only in every tool taking it (`query`, `query_data`, `query_file`, `export`, + `chart`, `save_query`, ...): one SELECT|WITH|EXPLAIN|SHOW|VALUES, even without `--read-only`; a write + behind `WITH` or `EXPLAIN (ANALYZE)` is refused. DDL/DML → `execute`. +- `--read-only` server flag guards exactly: execute, load_data, load_file, load_files, load_iceberg, + watch_directory, save_query, delete_query, set_table_metadata, copy_query, kv_set, kv_set_many, + kv_delete, kv_pop, kv_clear, plus a writable `attach_database` or `on_missing: "create"`. Read-only + attachments, queries, inspection, `chart`, detach/list stay available. + unwatch_directory remains allowed; export formats, including Hyper, remain allowed. Hyper export does not mutate its source + database, but it does create or replace its materialized destination file. +- Persistent-file contention: only a reserved persistent attachment lock is reported as + `RESOURCE_BUSY`. Run `hyperdb-mcp doctor`, compare client/daemon identities, close the possible owner + (Hyper, Tableau, or another process) or copy/select another `.hyper` file, then retry. +- `copy_query` mode: create (target must not exist; CREATE TABLE AS) | append (must exist; INSERT + INTO ... SELECT) | replace (drop and recreate, atomically). +- `load_file` merge (needs `merge_key`): key match → row deleted, incoming row inserted (target columns + absent from the file → NULL; duplicate incoming keys all insert) · no match → INSERT · new file + column → `ALTER TABLE ADD COLUMN` (nullable, existing rows NULL) · type change on an existing column + → rejected (use replace or a `schema` override). ⚠ DELETE+INSERT is not transactional (Hyper + auto-commits DDL): a mid-run failure leaves partial state, same as replace. ## SQL dialect quick-reference -PostgreSQL-compatible with Salesforce Data Cloud SQL extensions. Key -differences from standard PostgreSQL: - -- **No `information_schema` / `pg_catalog`.** Use `describe` / `sample` - instead. -- **No JSON / JSONB / UUID / SERIAL / BIGSERIAL / geometry types.** - Atomic types only: SMALLINT, INTEGER, BIGINT, REAL, DOUBLE PRECISION, - NUMERIC(p,s), BOOLEAN, TEXT, CHAR(n), VARCHAR(n), BYTES, DATE, TIME, - TIMESTAMP, TIMESTAMPTZ, INTERVAL, plus arrays of any atomic type. -- **`external(path, format => '...')`** — read Parquet / CSV / Iceberg - directly from disk inside a query without first loading it as a - table. Usable in the FROM clause. -- **Physical `NullType` Parquet columns are unreadable** — what a writer - emits for an all-null optional column in one partition. Hyper rejects - the *whole file* with `42804` (\"a data type that cannot be read by - Hyper ... hinting at a corrupted file\"); the file is not corrupt, and - selecting only the other columns does not help. A `schema` override - cannot fix it either: the override becomes a cast in the projection, - which the engine never evaluates. Re-type the column at the writer - (e.g. cast it to DOUBLE) and regenerate. `inspect_file` reports such a - column as type `NULL`; `load_file`, `load_files`, and `query_file` - fail fast and name it. -- **`to_char` is date/time only.** `to_char(TIMESTAMP '2020-01-02 - 03:04:05', 'YYYY-MM-DD')` → `2020-01-02` and `to_char(DATE - '2020-01-02', 'YYYY')` → `2020` both work. There is **no numeric - overload** — integer and NUMERIC arguments alike fail with `42601 - unsupported data types in call to 'to_char'`, e.g. - `to_char(123.456, 'FM990.00')`. That is an argument-type error, not a - missing function: a function Hyper genuinely lacks returns `42883`, as - `format()` does. -- **`expr::TEXT` is the numeric-formatting idiom** and preserves scale: - `CAST(9.5 AS NUMERIC(8,2))::TEXT` → `9.50`. Reach for this instead of - `to_char` on a number, and whenever you need exact decimal output. -- **A NUMERIC too large for an `f64` comes back as a JSON string**, not a - number, so no precision is lost in the result: `99999999999999999.99` - arrives as `\"99999999999999999.99\"`. Values that fit an `f64` — which - includes anything with 15 or fewer significant digits — stay JSON - numbers, so `9.50` arrives as `9.5`. Parse such a field as a decimal - string rather than assuming a number, and note it may already be exact - text if you applied `::TEXT` above. -- **`APPROX_COUNT_DISTINCT(expr)`** — approximate cardinality, 5-100x - faster than `COUNT(DISTINCT ...)` at high cardinality, on TEXT as well - as numeric keys. It accelerates the distinct step only, so a per-row - `a || '-' || b` inside `expr` is paid either way and can swamp the win: - prefer a numeric composite key (`a * 1000 + b`). -- **No `QUALIFY`** — absent from the grammar, so it fails the statement - with SQLSTATE `42601` (syntax error; a missing *function* would be - `42883`). Wrap the window in a subquery or CTE and filter outside. The - rewrite is exactly equivalent and costs nothing measurable: - ``` - SELECT * FROM (SELECT k, ROW_NUMBER() OVER (PARTITION BY p - ORDER BY c DESC) AS rnk FROM t) s WHERE rnk <= 5 - ``` -- **Window functions:** all standard ones plus `modified_rank()` (like - `rank()` but assigns the LOWEST rank on ties). `IGNORE NULLS` / - `RESPECT NULLS` only on `last_value`. -- **`DISTINCT ON (expr, ...)`, `GROUPING SETS`, `ROLLUP`, `CUBE`, - `FILTER (WHERE ...)`, ordered-set aggregates (`MODE()`, - `PERCENTILE_CONT()`, `PERCENTILE_DISC()` with `WITHIN GROUP`)** all - supported. -- **CTEs:** `WITH` and `WITH RECURSIVE`. CTEs evaluate once per query. -- **`TOP N`** is accepted alongside `LIMIT`. -- **No AI scalar functions** (`AI_CLASSIFY`, `AI_SENTIMENT`, etc. — those - are Data Cloud federation features, not Hyper). -- **No `ON CONFLICT` / `INSERT ... ON DUPLICATE KEY`.** Pass an array of - statements to `execute` and they run atomically inside a transaction: - ``` - execute({ \"sql\": [ - \"UPDATE settings SET value = 'dark' WHERE key = 'theme'\", - \"INSERT INTO settings (key, value) SELECT 'theme', 'dark' - WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'theme')\" - ]}) - ``` - A single-element array runs one auto-commit statement. Mixing DDL - with DML in one batch is rejected — Hyper aborts - such transactions with SQLSTATE 0A000. Issue DDL in its own `execute` - call. Do NOT include `BEGIN` / `COMMIT` / `ROLLBACK` / `SAVEPOINT` in - batch elements — the tool manages the transaction for you and these - are rejected up front. +PostgreSQL-compatible plus Salesforce Data Cloud SQL extensions. Differences: + +- ✗ `information_schema`, `pg_catalog` → `describe`/`sample`. ✗ JSON, JSONB, UUID, SERIAL, BIGSERIAL, + geometry. Atomic types only: SMALLINT, INTEGER, BIGINT, REAL, DOUBLE PRECISION, NUMERIC(p,s), + BOOLEAN, TEXT, CHAR(n), VARCHAR(n), BYTES, DATE, TIME, TIMESTAMP, TIMESTAMPTZ, INTERVAL, + arrays of + any atomic type. +- ✓ `external(path, format => '...')` in FROM: read Parquet|CSV|Iceberg from disk, no load. +- ⚠ Physical `NullType` Parquet column (a writer's all-null optional column in one partition) → Hyper + rejects the whole file, `42804` ("a data type that cannot be read by Hyper ... hinting at a corrupted + file"). Not corrupt; selecting other columns doesn't help; a `schema` override can't fix it (it + becomes a projection cast the engine never evaluates). Re-type it at the writer (e.g. cast to + DOUBLE), regenerate. `inspect_file` shows type `NULL`; `load_file`, `load_files`, `query_file` fail + fast naming it. +- ⚠ `to_char` is date/time only: `to_char(TIMESTAMP '2020-01-02 03:04:05', 'YYYY-MM-DD')` → + `2020-01-02`, `to_char(DATE '2020-01-02', 'YYYY')` → `2020`. No numeric overload: integer and + NUMERIC alike → `42601 unsupported data types in call to 'to_char'` (e.g. `to_char(123.456, + 'FM990.00')`), an argument-type error; a truly missing function is `42883`, like `format()`. +- ✓ `expr::TEXT` formats numbers, keeps scale: `CAST(9.5 AS NUMERIC(8,2))::TEXT` → `9.50`. Use it + instead of `to_char` on numbers and for exact decimal output. +- ⚠ NUMERIC too large for `f64` → JSON string, no precision lost: `99999999999999999.99` → + `"99999999999999999.99"`. Fits `f64` (any ≤15 significant digits) → JSON number: `9.50` → `9.5`. + Parse such fields as decimal strings; after `::TEXT` they are exact text. +- ✓ `APPROX_COUNT_DISTINCT(expr)`: approximate cardinality, 5-100x faster than `COUNT(DISTINCT ...)` + at high cardinality, TEXT or numeric. Speeds only the distinct step: per-row `a || '-' || b` in + `expr` is paid either way and can swamp the win; prefer a numeric key (`a * 1000 + b`). +- ✗ `QUALIFY`: not in the grammar → `42601` (syntax; a missing function would be `42883`). Filter a + subquery/CTE instead (exactly equivalent, no measurable cost): `SELECT * FROM (SELECT k, + ROW_NUMBER() OVER (PARTITION BY p ORDER BY c DESC) AS rnk FROM t) s WHERE rnk <= 5` +- ✓ All standard window functions + `modified_rank()` (`rank()` but LOWEST rank on ties); `IGNORE + NULLS`/`RESPECT NULLS` only on `last_value`. +- ✓ `DISTINCT ON (expr, ...)`, `GROUPING SETS`, `ROLLUP`, `CUBE`, `FILTER (WHERE ...)`, ordered-set + aggregates (`MODE()`, `PERCENTILE_CONT()`, `PERCENTILE_DISC()` + `WITHIN GROUP`), `WITH`, `WITH + RECURSIVE` (each CTE evaluated once per query), `TOP N` alongside `LIMIT`. +- ✗ AI scalar functions (`AI_CLASSIFY`, `AI_SENTIMENT`, ...): Data Cloud federation, not Hyper. +- ✗ `ON CONFLICT`, `INSERT ... ON DUPLICATE KEY` → statement array to `execute`, run atomically (see + the upsert example). ⚠ DDL + DML in one batch rejected (Hyper aborts it, SQLSTATE 0A000): DDL in its + own `execute`. ⚠ No `BEGIN`/`COMMIT`/`ROLLBACK`/`SAVEPOINT` elements: the tool manages the + transaction and rejects them up front. Full reference: https://developer.salesforce.com/docs/data/data-cloud-query-guide/references/dc-sql-reference @@ -412,82 +274,41 @@ Full reference: https://developer.salesforce.com/docs/data/data-cloud-query-guid ``` // Quickest path: query a file without loading it as a table -query_file({ - \"path\": \"/tmp/sales.csv\", - \"sql\": \"SELECT region, SUM(amount) FROM data GROUP BY region\" -}) - +query_file({ "path": "/tmp/sales.csv", "sql": "SELECT region, SUM(amount) FROM data GROUP BY region" }) // Inspect a file before committing to a load -inspect_file({ \"path\": \"/tmp/sales.csv\" }) - -// Loaded-table workflow (best when you'll run multiple queries) -load_file({ \"path\": \"/tmp/sales.csv\", \"table\": \"sales\" }) -sample({ \"table\": \"sales\" }) -query({ \"sql\": \"SELECT region, SUM(amount) FROM sales GROUP BY region\" }) - +inspect_file({ "path": "/tmp/sales.csv" }) +// Loaded-table workflow (best for multiple queries) +load_file({ "path": "/tmp/sales.csv", "table": "sales" }); sample({ "table": "sales" }) +query({ "sql": "SELECT region, SUM(amount) FROM sales GROUP BY region" }) // Cross-database join via attachment -attach_database({ \"alias\": \"lookup\", \"path\": \"/data/dim.hyper\" }) -query({ - \"sql\": \"SELECT s.region, d.country_name, SUM(s.amount) \ - FROM sales s JOIN lookup.public.dim_region d ON s.region = d.code \ - GROUP BY s.region, d.country_name\" -}) - -// Read Parquet directly inside a query — no load step -query({ - \"sql\": \"SELECT COUNT(*) FROM external('/tmp/events.parquet', format => 'parquet')\" -}) - +attach_database({ "alias": "lookup", "path": "/data/dim.hyper" }) +query({ "sql": "SELECT s.region, d.country_name, SUM(s.amount) FROM sales s + JOIN lookup.public.dim_region d ON s.region = d.code GROUP BY s.region, d.country_name" }) +// Read Parquet directly inside a query, no load step +query({ "sql": "SELECT COUNT(*) FROM external('/tmp/events.parquet', format => 'parquet')" }) // Export a query result -export({ - \"sql\": \"SELECT * FROM sales WHERE amount > 1000\", - \"path\": \"/tmp/big_sales.parquet\", - \"format\": \"parquet\" -}) - -// Refresh existing rows + auto-add new columns (upsert by job_id). -// Use this when you re-parsed source data with extra fields and want -// to refresh the table in place without dropping it. -load_file({ - \"path\": \"/tmp/extract_failures-with-host.json\", - \"table\": \"extract_timing_failures\", - \"mode\": \"merge\", - \"merge_key\": \"job_id\" -}) - +export({ "sql": "SELECT * FROM sales WHERE amount > 1000", "path": "/tmp/big.parquet", "format": "parquet" }) +// Re-parsed source gained fields: upsert by job_id, auto-add columns, refresh in place, no drop +load_file({ "path": "/tmp/failures.json", "table": "failures", "mode": "merge", "merge_key": "job_id" }) // Single-statement execute (auto-commit) -execute({ - \"sql\": [\"DELETE FROM events WHERE created_at < CURRENT_DATE - INTERVAL '90' DAY\"] -}) - -// Atomic upsert — both statements commit together or both roll back -execute({ - \"sql\": [ - \"UPDATE settings SET value = 'dark' WHERE key = 'theme'\", - \"INSERT INTO settings (key, value) SELECT 'theme', 'dark' \ - WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'theme')\" - ], - \"database\": \"persistent\" -}) - +execute({ "sql": ["DELETE FROM events WHERE created_at < CURRENT_DATE - INTERVAL '90' DAY"] }) +// Atomic upsert: both statements commit together or both roll back +execute({ "database": "persistent", "sql": [ + "UPDATE settings SET value = 'dark' WHERE key = 'theme'", + "INSERT INTO settings (key, value) SELECT 'theme', 'dark' + WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'theme')" ] }) // Chart -chart({ - \"sql\": \"SELECT region, SUM(amount) AS total FROM sales GROUP BY region\", - \"chart_type\": \"bar\", - \"x\": \"region\", - \"y\": \"total\" -}) +chart({ "sql": "SELECT region, SUM(amount) AS total FROM sales GROUP BY region", + "chart_type": "bar", "x": "region", "y": "total" }) ``` ## Tips for picking the right tool -- One-shot \"what's in this file?\" → `query_file` or `inspect_file`. -- Repeated analysis on the same data → `load_file` once, then `query`. -- File too large to fit in memory comfortably → `external()` inside - `query` (streams from disk). -- Joining datasets across .hyper files → `attach_database` + `query`. -- Materializing a query result as a new table → `copy_query`. -- Need a picture for the user → `chart`. -- Re-parsed source data with new columns and want to update existing - table in place → `load_file` with `mode: \"merge\"`. -"; +- one-shot "what's in this file?" → `query_file` or `inspect_file` +- repeated analysis of the same data → `load_file` once, then `query` +- file too large for memory → `external()` inside `query` (streams from disk) +- join across .hyper files → `attach_database` + `query` +- materialize a query result as a table → `copy_query` +- a picture for the user → `chart` +- re-parsed source with new columns, update in place → `load_file` with `mode: "merge"` +"##; From 3f9ae0d62f5c723155eefd3dcbfe35218f3f1799 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 17:48:16 -0700 Subject: [PATCH 41/61] test(mcp): gate unix-only test helpers so the Windows legs compile clean Windows clippy and test failed on items used only under #[cfg(unix)]: an AtomicUsize import, run_discovery_compatibility_child, scripted_daemon_noting_stop, wait_for_flag, and two `let mut targets` that only the unix symlink block mutates. Gate the helpers and rebind the target lists under cfg(unix) instead of mutating them. --- hyperdb-mcp/src/daemon/discovery.rs | 1 + hyperdb-mcp/src/diagnostics.rs | 4 +++- hyperdb-mcp/tests/daemon_tests.rs | 2 ++ hyperdb-mcp/tests/end_to_end_mcp_tests.rs | 8 ++++---- hyperdb-mcp/tests/export_tests.rs | 8 ++++---- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/hyperdb-mcp/src/daemon/discovery.rs b/hyperdb-mcp/src/daemon/discovery.rs index e4ba8a63..04fcc73a 100644 --- a/hyperdb-mcp/src/daemon/discovery.rs +++ b/hyperdb-mcp/src/daemon/discovery.rs @@ -551,6 +551,7 @@ mod tests { entries } + #[cfg(unix)] fn run_discovery_compatibility_child(test_name: &str, child_sentinel_env: &str) { use std::process::{Command, Stdio}; use std::time::Instant; diff --git a/hyperdb-mcp/src/diagnostics.rs b/hyperdb-mcp/src/diagnostics.rs index cbd41cfa..2ab26dfb 100644 --- a/hyperdb-mcp/src/diagnostics.rs +++ b/hyperdb-mcp/src/diagnostics.rs @@ -1714,7 +1714,9 @@ mod tests { use std::io::{self, Read as _, Write as _}; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::Path; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + #[cfg(unix)] + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; diff --git a/hyperdb-mcp/tests/daemon_tests.rs b/hyperdb-mcp/tests/daemon_tests.rs index 60764b74..404f4239 100644 --- a/hyperdb-mcp/tests/daemon_tests.rs +++ b/hyperdb-mcp/tests/daemon_tests.rs @@ -1989,6 +1989,7 @@ fn scripted_daemon(info: &DaemonInfo) -> impl Fn(&str) -> Vec + Send + 'stat /// [`scripted_daemon`] that also raises `stop_seen` the moment it is sent `STOP`, /// so a test can release the daemon lock only after the CLI really asked. +#[cfg(unix)] fn scripted_daemon_noting_stop( info: &DaemonInfo, stop_seen: Arc, @@ -2003,6 +2004,7 @@ fn scripted_daemon_noting_stop( } /// Block until `flag` is raised, failing the test after `budget`. +#[cfg(unix)] fn wait_for_flag(flag: &AtomicBool, budget: Duration) { let started = Instant::now(); while !flag.load(Ordering::SeqCst) { diff --git a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs index 012d38d1..7eabd814 100644 --- a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs +++ b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs @@ -4155,13 +4155,13 @@ async fn export_and_chart_never_replace_session_databases() -> TestResult { assert!(!is_error(&r), "seed attached failed: {:?}", first_text(&r)); } - let mut targets = vec![workspace.clone(), attached.clone()]; + let targets = vec![workspace.clone(), attached.clone()]; #[cfg(unix)] - { + let targets = { let link = dir.path().join("link.hyper"); std::os::unix::fs::symlink(&workspace, &link)?; - targets.push(link); - } + targets.into_iter().chain([link]).collect::>() + }; let mut failures = Vec::new(); for target in &targets { diff --git a/hyperdb-mcp/tests/export_tests.rs b/hyperdb-mcp/tests/export_tests.rs index ba7dfe26..ea7d69f8 100644 --- a/hyperdb-mcp/tests/export_tests.rs +++ b/hyperdb-mcp/tests/export_tests.rs @@ -1388,16 +1388,16 @@ fn export_refuses_a_protected_file_by_identity() { let dir = tempfile::tempdir().unwrap(); let protected = dir.path().join("session.hyper"); std::fs::write(&protected, b"Hyper\x08\0\0 session").unwrap(); - let mut targets = vec![ + let targets = vec![ protected.clone(), dir.path().join(".").join("session.hyper"), ]; #[cfg(unix)] - { + let targets = { let link = dir.path().join("link.hyper"); std::os::unix::fs::symlink(&protected, &link).unwrap(); - targets.push(link); - } + targets.into_iter().chain([link]).collect::>() + }; for target in &targets { for format in ["csv", "parquet", "arrow_ipc", "iceberg", "hyper"] { From 46e0755efa6ce0ad55760e827b29a34d067b4074 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 18:33:18 -0700 Subject: [PATCH 42/61] test(core): close the TLS test client gracefully so Windows does not reset the connection A TLS 1.3 server sends session tickets the test client never reads. On Windows, dropping the socket with that data unread sends a reset (os error 10053), which discards the payload the server has not read yet. Send close_notify, shut down the write side and drain until the server hangs up. --- hyperdb-api-core/src/client/tls.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/hyperdb-api-core/src/client/tls.rs b/hyperdb-api-core/src/client/tls.rs index bb0aa3ce..6b4d847f 100644 --- a/hyperdb-api-core/src/client/tls.rs +++ b/hyperdb-api-core/src/client/tls.rs @@ -1065,6 +1065,16 @@ mod tests { Negotiated::Tls(mut stream) => { stream.write_all(PAYLOAD).map_err(Error::from_io)?; stream.flush().map_err(Error::from_io)?; + // Close gracefully and drain until the server hangs up. A TLS + // 1.3 server sends session tickets the client never reads; + // dropping the socket with that data unread makes Windows send + // a reset (os error 10053), which discards the payload the + // server has not read yet. + stream.conn.send_close_notify(); + stream.flush().ok(); + stream.sock.shutdown(std::net::Shutdown::Write).ok(); + stream.sock.set_read_timeout(Some(IO_TIMEOUT)).ok(); + std::io::copy(&mut stream.sock, &mut std::io::sink()).ok(); Ok(true) } } @@ -1537,6 +1547,14 @@ mod tests { Negotiated::Tls(mut stream) => { stream.write_all(PAYLOAD).await.map_err(Error::from_io)?; stream.flush().await.map_err(Error::from_io)?; + // Graceful close plus drain; see `negotiate_and_send`. + stream.shutdown().await.ok(); + tokio::time::timeout( + IO_TIMEOUT, + tokio::io::copy(&mut stream, &mut tokio::io::sink()), + ) + .await + .ok(); Ok(true) } } From 0e4cc06e6ebe895f5770d5ec9ae11e1bb1171f2c Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 18:33:18 -0700 Subject: [PATCH 43/61] fix(sea-query): escape string literals Postgres-style and stop dropping or panicking on Postgres constructs Hyper's builder inherited sea-query's MySQL-style backslash escaping for every path that did not go through the delegated prepare_value, so a Constant string like x\' OR 1=1 -- matched every row. Escaping, string and byte literals now follow Postgres rules (E'..', '\x..', NUL as \000). Also emit DISTINCT ON and NULLS FIRST/LAST, render PgOperator, PgFunc, TABLESAMPLE and AsEnum instead of panicking or dropping them, use COALESCE, and convert Value::Enum to a plain string. Fixes a panic on non-ASCII Char. Adds integration tests against the pinned hyperd. Documents the clauses that still render through the Postgres builder. --- Cargo.lock | 1 + sea-query-hyperdb/CHANGELOG.md | 64 ++ sea-query-hyperdb/Cargo.toml | 7 +- sea-query-hyperdb/DEVELOPMENT.md | 75 +- sea-query-hyperdb/README.md | 41 +- sea-query-hyperdb/src/lib.rs | 916 +++++++++++++++++-- sea-query-hyperdb/tests/hyper_integration.rs | 406 ++++++++ 7 files changed, 1413 insertions(+), 97 deletions(-) create mode 100644 sea-query-hyperdb/tests/hyper_integration.rs diff --git a/Cargo.lock b/Cargo.lock index 51c87ac8..fc2dcc70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3622,6 +3622,7 @@ dependencies = [ name = "sea-query-hyperdb" version = "1.0.0-rc.5" dependencies = [ + "hyperdb-api", "sea-query", ] diff --git a/sea-query-hyperdb/CHANGELOG.md b/sea-query-hyperdb/CHANGELOG.md index cd526758..ebddf637 100644 --- a/sea-query-hyperdb/CHANGELOG.md +++ b/sea-query-hyperdb/CHANGELOG.md @@ -7,6 +7,70 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Security + +- **Fixed SQL injection in inlined string literals.** `HyperQueryBuilder` used + sea-query's default MySQL backslash escaping, but Hyper (like PostgreSQL with + `standard_conforming_strings`) treats a backslash in a plain `'...'` literal + literally. A value such as `x\' OR 1=1 --` rendered through + `Expr::Constant(Value::String(..))`, `Order::Field`, a `LIKE ... ESCAPE` + constant, `inject_parameters`, or `HyperQueryBuilder::value_to_string` escaped + its own quote and matched every row. Escaping now follows PostgreSQL rules + (`E'...'` whenever a backslash, quote or control character is present). + `Expr::val(..)` / `.eq(..)` / `INSERT` values were already safe by accident of + delegation. Anyone who inlined untrusted strings through the affected paths + should upgrade. + +### Fixed + +- `escape_string` / `unescape_string` now use PostgreSQL rules instead of MySQL + rules, which corrupted data containing `\` or `"`. +- `Value::Bytes` renders as `'\x...'` instead of `x'...'` (Hyper: `0A000 bit + strings not implemented`). +- Non-ASCII `Value::Char` no longer panics (it was truncated to one byte). +- Every `PgBinOper` (`ILIKE`, `NOT ILIKE`, `~`, `~*`, `||`, `->`, `->>`, ...), the + `PgExpr` helpers, every `PgFunc` (`STARTS_WITH`, `DATE_TRUNC`, `ANY`, ...) and + `SqliteBinOper` no longer hit `unimplemented!()`. Operators Hyper lacks (`@>`, + `<@`, `&&`, `@@`, `%`, `<->`) are rendered verbatim and rejected by the server + instead of panicking in the builder. +- `Select::distinct_on(..)` rendered `SELECT "a"` with `DISTINCT ON` dropped; + `order_by_with_nulls(.., NullOrdering::..)` lost `NULLS FIRST/LAST` (also + inside `FROM` subqueries). Both are now emitted. +- `Expr::as_enum(..)` keeps its `CAST(.. AS "type")`; `TABLESAMPLE` (with + `REPEATABLE`) is no longer silently dropped; `Func::if_null` emits `COALESCE`. +- `PARTITION BY` / `FOR VALUES` clauses are written out instead of being + silently dropped or panicking (Hyper has no declarative partitioning and + rejects them). +- A NUL character is escaped as `\000` rather than `\0`, so literal digits that + follow it can no longer extend the octal escape and close the string literal. +- `Value::Enum` bound through `.build(..)` is sent as a plain string parameter, + and `ColumnDef` `DEFAULT` expressions are rendered by Hyper (non-ASCII + `Value::Char`, bare enum strings) instead of by the Postgres builder; + auto-increment on a non-integer column no longer panics. `Expr::as_enum` with + a multi-dimensional array type (`"t[][]"`) keeps every `[]` outside the quotes. +- SQLite-only operators unknown to this crate are written as a placeholder the + server rejects, never as Rust `Debug` text. + +### Changed + +- **Breaking:** the SQL text for inlined strings containing `\`, `'`, `"` or + control characters, for `Value::Bytes`, and for `Value::Char` changes as + described above. Code that compared generated SQL byte-for-byte must update. +- **Breaking:** `Value::Enum` is rendered as a bare string literal (no + `::"type"` hint), because Hyper has no enum types, and `Expr::as_enum(..)` + renders `CAST( AS "type")`, so enum-typed casts still need a real type + on the server. +- Minimum `sea-query` is now 1.0.2 (the version the builder was written and + tested against). +- Known limitation: `ORDER BY` expressions, `EXPLAIN`, `CREATE INDEX` and + `ALTER TABLE` are rendered by `PostgresQueryBuilder` because sea-query keeps + their fields crate-private; Hyper-specific rendering of non-ASCII + `Value::Char`, `Value::Enum` and SQLite operators does not apply inside them. +- `HyperQueryBuilder` is no longer documented as identical to + `PostgresQueryBuilder`; it owns statement dispatch, value rendering and + `AsEnum`. `!~`, `!~*` and `SIMILAR TO` need `BinOper::Custom` / `Expr::cust`. +- New dev-dependency on `hyperdb-api` (path-only) for `tests/hyper_integration.rs`. + ## [1.0.0-rc.3] - 2026-09-07 These changes first shipped in 1.0.0-rc.1. diff --git a/sea-query-hyperdb/Cargo.toml b/sea-query-hyperdb/Cargo.toml index 12759023..14ab004e 100644 --- a/sea-query-hyperdb/Cargo.toml +++ b/sea-query-hyperdb/Cargo.toml @@ -15,4 +15,9 @@ categories = ["database"] workspace = true [dependencies] -sea-query = "1.0" +sea-query = "1.0.2" + +[dev-dependencies] +# path-only: no version pin so cargo doesn't try to resolve hyperdb-api against +# crates.io during `cargo publish` (same pattern as hyperdb-api-derive). +hyperdb-api = { path = "../hyperdb-api" } diff --git a/sea-query-hyperdb/DEVELOPMENT.md b/sea-query-hyperdb/DEVELOPMENT.md index 32af782d..7ed595a3 100644 --- a/sea-query-hyperdb/DEVELOPMENT.md +++ b/sea-query-hyperdb/DEVELOPMENT.md @@ -5,42 +5,69 @@ Contributor guide for the `sea-query-hyperdb` crate. ## Dialect Implementation `HyperQueryBuilder` implements all sea-query backend traits (`QueryBuilder`, -`SchemaBuilder`, `GenericBuilder`, and their sub-traits) by delegating every -method to `PostgresQueryBuilder`. This is a deliberate design choice: HyperDB's -SQL dialect is currently PostgreSQL-compatible for identifier quoting, placeholder -syntax (`$1, $2, ...`), operator precedence, and DDL/DML syntax. - -The delegation is explicit per-method rather than using a blanket `Deref` or -macro, so that individual methods can be overridden independently as the dialect -diverges. See the doc comment on `HyperQueryBuilder` in `src/lib.rs` for the -full rationale. +`SchemaBuilder`, `GenericBuilder`, and their sub-traits). HyperDB's SQL dialect +is PostgreSQL-compatible for identifier quoting, placeholder syntax (`$1, $2, +...`), operator precedence, string escaping and most DDL/DML syntax, so most +hooks are explicit one-line delegations to `PostgresQueryBuilder`. + +It is **not** a blanket pass-through, and that distinction matters. sea-query's +`QueryBuilder` calls back into `self` for every nested construct. If a hook +that `PostgresQueryBuilder` overrides is left at sea-query's generic default +here, the output changes silently: `EscapeBuilder` used MySQL backslash rules +(an injection hole against Hyper), `ILIKE` and `PgFunc` panicked, +`DISTINCT ON` and `NULLS FIRST/LAST` were dropped, `TABLESAMPLE` vanished and +`IFNULL` replaced `COALESCE`. When adding a hook, diff against the +Postgres builder's overrides first (`sea-query/src/backend/postgres/`). + +Pieces that are Hyper's own (not delegated): + +- `prepare_query_statement` dispatches on `SubQueryStatement` itself so nested + statements are rendered with Hyper's overrides +- `prepare_value` (`push_param(.., self)`), `write_value` (`Char` as a full + scalar, `Enum` without a type hint), `prepare_expr` (`AsEnum` as `CAST`) +- `prepare_bin_oper` (SQLite operators, never panics) and the partition hooks + (no panic on `KEY` / `LESS THAN`) + +- `write_escaped` / `write_string_quoted` (NUL as `\000`) and + `prepare_column_def` (via `ColumnDef`'s public accessors, so `DEFAULT` / + `CHECK` expressions use Hyper's `prepare_expr`) + +`prepare_explain_statement`, `prepare_order_expr`, `prepare_index_create_statement` +and `prepare_table_alter_statement` stay delegated: `ExplainStatement`, +`OrderExpr`, `IndexCreateStatement` and `TableAlterStatement` have crate-private +fields and no accessors, so nothing nested inside them (including the `ORDER BY` +expression and `Order::Field`) can be re-rendered with Hyper's overrides. If +sea-query ever exposes them, replace the delegations with local copies. +Upstream `unimplemented!`/`panic!` sites remain for `ColumnType::Year`, +`ALTER TABLE` with no options, `JoinOn::Columns`, a `WITH` query without a +statement and `prepare_table_ref_table_stmt` forms; they are not masked. ## When Hyper Diverges from PostgreSQL -As HyperDB adds dialect-specific features, override individual trait methods -to emit Hyper-specific SQL while leaving the rest delegated. Likely candidates: - -- **Type names** -- Hyper may introduce types not in PostgreSQL -- **Function syntax** -- Hyper-specific built-in functions -- **DDL extensions** -- Hyper-specific table or column options +Override individual trait methods to emit Hyper-specific SQL while leaving the +rest delegated. When adding an override: 1. Override the relevant method on the appropriate trait impl -2. Add a unit test that asserts the Hyper output differs from `PostgresQueryBuilder` +2. Add a unit test asserting the exact SQL, and an integration test in + `tests/hyper_integration.rs` if the claim is about what Hyper accepts 3. Document the divergence in the `HyperQueryBuilder` struct doc comment ## Testing -All tests live in the `#[cfg(test)] mod tests` block in `src/lib.rs`. The test -strategy verifies that `HyperQueryBuilder` produces identical SQL to -`PostgresQueryBuilder` for all standard operations (SELECT, INSERT, UPDATE, -DELETE, CREATE TABLE). +Unit tests live in the `#[cfg(test)] mod tests` block in `src/lib.rs` and assert +exact SQL: escaping and injection cases, every `PgBinOper`, `PgFunc` samples, +`DISTINCT ON`, `NULLS FIRST/LAST`, `TABLESAMPLE`, DDL parity with Postgres. -Run tests: +`tests/hyper_integration.rs` runs generated SQL on a real `hyperd` (round trips +of hostile strings through every inlining path, bytes, operators, `DISTINCT ON`, +and a `@>` query that must fail server-side, not panic). It needs the pinned +engine; set an absolute `HYPERD_PATH` (see the workspace `AGENTS.md`): ```bash -cargo test -p sea-query-hyperdb +make download-hyperd +HYPERD_PATH="$PWD/.hyperd/current" cargo test -p sea-query-hyperdb ``` Run the example: @@ -48,7 +75,3 @@ Run the example: ```bash cargo run -p sea-query-hyperdb --example basic_usage ``` - -When Hyper-specific overrides are added, tests should assert both that the -override produces the expected Hyper-specific SQL and that non-overridden -operations still match `PostgresQueryBuilder`. diff --git a/sea-query-hyperdb/README.md b/sea-query-hyperdb/README.md index ef128b01..573fe14a 100644 --- a/sea-query-hyperdb/README.md +++ b/sea-query-hyperdb/README.md @@ -3,10 +3,12 @@ HyperDB SQL dialect backend for [sea-query](https://crates.io/crates/sea-query). This crate provides `HyperQueryBuilder`, a sea-query backend that generates SQL -compatible with HyperDB's SQL dialect. HyperDB is largely PostgreSQL-compatible, -so this builder currently delegates to `PostgresQueryBuilder` for all operations. -As Hyper's SQL dialect evolves, this builder will be updated to emit Hyper-specific -syntax where needed. +compatible with HyperDB's SQL dialect. HyperDB speaks a large subset of +PostgreSQL, so the builder reuses `PostgresQueryBuilder`'s rendering for most +constructs (quoting, `$n` placeholders, `E'...'` string escaping, `ILIKE`, +`DISTINCT ON`, `NULLS FIRST/LAST`, `TABLESAMPLE`, ...). It is not a pass-through: +statement dispatch, value rendering and some expression forms are Hyper's own, +and the output is tested against a real `hyperd`. --- @@ -80,9 +82,34 @@ let result = conn.execute_query(&sql)?; ## HyperDB-Specific Dialect -Use `HyperQueryBuilder` instead of `PostgresQueryBuilder` so your code -automatically picks up Hyper-specific syntax as the dialect evolves. Both produce -identical SQL today. +Use `HyperQueryBuilder` instead of `PostgresQueryBuilder`. For ordinary +queries the SQL is the same, but the two differ where Hyper does: + +- `Value::Enum` is a bare string (Hyper has no enum types, so no `::"type"` hint) +- `Value::Char` supports any Unicode character +- SQLite-only operators and the MySQL-only `PARTITION BY KEY` are written out + rather than panicking, and Hyper then rejects them +- Syntax Hyper lacks (`@>`, `<@`, `&&`, `@@`, `%`, `<->`, partitioning) is + rendered as written and fails server-side with an error + +Known limitations (sea-query keeps these structs' fields crate-private, so they +are rendered by `PostgresQueryBuilder` and Hyper's overrides for non-ASCII +`Value::Char`, `Value::Enum` and SQLite operators do not apply inside them): +`ORDER BY` expressions (including `Order::Field`), `EXPLAIN`, `CREATE INDEX` +(partial-index `WHERE`) and `ALTER TABLE`. sea-query itself still panics on +`ColumnType::Year`, `ALTER TABLE` with no options, `JoinOn::Columns`, a +`WITH` query without a statement and some table-reference forms; those +panics are upstream and not masked here. + +`!~`, `!~*` and `SIMILAR TO` have no typed sea-query operator; use +`BinOper::Custom("!~")` or `Expr::cust(..)`. `JoinOn::Columns` is not implemented +by sea-query for any backend. + +Always build values through `Expr::val(..)` / `.eq(..)` where you can. Inlined +literals (`Expr::Constant`, `to_string`, `inject_parameters`) are escaped with +PostgreSQL rules (`E'...'` when a backslash or control character is present), +which is what Hyper expects; earlier releases used MySQL rules here (see the +changelog). `HyperQueryBuilder` implements all sea-query backend traits: diff --git a/sea-query-hyperdb/src/lib.rs b/sea-query-hyperdb/src/lib.rs index 1f6e63b9..7d52681f 100644 --- a/sea-query-hyperdb/src/lib.rs +++ b/sea-query-hyperdb/src/lib.rs @@ -4,12 +4,13 @@ //! `HyperDB` SQL dialect backend for [sea-query](https://crates.io/crates/sea-query). //! //! This crate provides [`HyperQueryBuilder`], a sea-query backend that generates SQL -//! compatible with `HyperDB`'s SQL dialect. `HyperDB` is largely PostgreSQL-compatible, so -//! this builder currently delegates to [`PostgresQueryBuilder`] for all operations. -//! -//! As Hyper's SQL dialect evolves and diverges from `PostgreSQL`, this builder will be -//! updated to emit Hyper-specific syntax where needed (e.g., type names, function calls, -//! or DDL extensions). +//! compatible with `HyperDB`'s SQL dialect. `HyperDB` speaks a large subset of the +//! `PostgreSQL` dialect, so this builder reuses [`PostgresQueryBuilder`]'s rendering for +//! most constructs, but it is **not** a pass-through: the builder owns statement +//! dispatch, value rendering and a few expression forms (see "Where Hyper differs" on +//! [`HyperQueryBuilder`]), and its output is checked against a real `hyperd` in the +//! integration tests. Hyper rejects some syntax Postgres accepts; those constructs are +//! rendered as written and the server reports the error. //! //! # Usage //! @@ -73,11 +74,13 @@ use sea_query::backend::{ EscapeBuilder, ForeignKeyBuilder, GenericBuilder, IndexBuilder, OperLeftAssocDecider, PrecedenceDecider, QueryBuilder, QuotedBuilder, SchemaBuilder, TableBuilder, TableRefBuilder, }; +use sea_query::extension::sqlite::SqliteBinOper; use sea_query::{ - BinOper, ColumnDef, ColumnType, ExplainStatement, Expr, ForeignKeyCreateStatement, - ForeignKeyDropStatement, IndexCreateStatement, IndexDropStatement, Oper, PostgresQueryBuilder, - Quote, SelectInto, SubQueryStatement, TableAlterStatement, TableRef, TableRenameStatement, - Value, + Alias, BinOper, ColumnDef, ColumnType, DynIden, ExplainStatement, Expr, + ForeignKeyCreateStatement, ForeignKeyDropStatement, Func, IndexCreateStatement, + IndexDropStatement, IntoIden, Oper, OptionEnum, OrderExpr, PartitionBy, PartitionValues, + PostgresQueryBuilder, Quote, SelectDistinct, SelectInto, SelectStatement, SqlWriter, + SubQueryStatement, TableAlterStatement, TableRef, TableRenameStatement, Value, }; /// HyperDB-specific SQL dialect backend for sea-query. @@ -88,13 +91,37 @@ use sea_query::{ /// # Delegation Strategy /// /// `HyperDB`'s SQL dialect is largely PostgreSQL-compatible — identifier quoting, -/// placeholder syntax (`$1, $2, ...`), operator precedence, and DDL/DML syntax -/// all match `PostgreSQL`. Rather than duplicating that logic, this builder -/// delegates to [`PostgresQueryBuilder`] for all operations. +/// placeholder syntax (`$1, $2, ...`), operator precedence, string escaping +/// (`E'...'`), and most DDL/DML syntax match `PostgreSQL`. Rather than +/// duplicating that logic, this builder delegates the individual rendering hooks +/// (escaping, operators, functions, `DISTINCT ON`, `NULLS FIRST/LAST`, +/// `TABLESAMPLE`, index and DDL hooks) to [`PostgresQueryBuilder`]. +/// +/// It is deliberately **not** a blanket pass-through. sea-query's `QueryBuilder` +/// calls back into `self` for every nested construct, so a hook left at the +/// generic default silently changes what is emitted (dropping `DISTINCT ON`, +/// panicking on `ILIKE`, MySQL-style backslash escaping). Every hook Postgres +/// overrides is therefore either delegated or replaced here. +/// +/// # Where Hyper differs from `PostgresQueryBuilder` +/// +/// - Statement dispatch is Hyper's own, so nested statements use Hyper's rules. +/// - `Value::Enum` renders as a bare string (Hyper has no enum types, so no +/// `::"type"` hint); `Expr::AsEnum` keeps its `CAST(.. AS "type")`. +/// - `Value::Char` is rendered as a full Unicode scalar (no byte truncation). +/// - SQLite-only operators are written out instead of panicking; Hyper then +/// rejects them (except `->` / `->>`). +/// - `PARTITION BY KEY` / `VALUES LESS THAN` are written out instead of +/// panicking. Hyper has no declarative partitioning and rejects all of it. +/// - Operators Hyper lacks (`@>`, `<@`, `&&`, `@@`, `%`, `<->`, ...) are +/// rendered verbatim and fail server-side with a syntax error. /// -/// As Hyper's dialect evolves (e.g., Hyper-specific type names, functions, or -/// DDL extensions), individual trait methods will be overridden to emit -/// Hyper-specific syntax while the rest continues to delegate. +/// Known limitations: `!~`, `!~*` and `SIMILAR TO` have no typed sea-query +/// operator; use `BinOper::Custom("!~")` or `Expr::cust`. The statement nested +/// inside `EXPLAIN`, `ORDER BY` expressions, `CREATE INDEX` and `ALTER TABLE` is +/// rendered by [`PostgresQueryBuilder`] (their fields are crate-private in +/// sea-query), so Hyper's overrides do not apply there. `JoinOn::Columns` +/// is not implemented by sea-query itself and panics for every backend. /// /// Notable `HyperDB` SQL features: /// - Optional `FROM` clause for simple expressions @@ -132,7 +159,34 @@ impl QuotedBuilder for HyperQueryBuilder { } } -impl EscapeBuilder for HyperQueryBuilder {} +/// String escaping follows `PostgreSQL` rules, which is what Hyper implements: +/// `standard_conforming_strings` is on, so a plain `'...'` literal treats the +/// backslash literally and only `E'...'` literals interpret backslash escapes. +/// sea-query's default (MySQL) rules would turn `\` and `'` into sequences that +/// a plain Hyper literal does not understand, which both corrupts data and +/// opens an injection hole. +impl EscapeBuilder for HyperQueryBuilder { + fn needs_escape(&self, s: &str) -> bool { + PostgresQueryBuilder.needs_escape(s) + } + + fn write_escaped(&self, buffer: &mut impl std::fmt::Write, string: &str) { + // Same as Postgres except NUL, which is written as a full three-digit + // octal escape: a bare `\0` followed by literal digits would be read + // back as a longer octal escape (`\0` + `47` -> `\047`, a quote). + let mut rest = string; + while let Some(pos) = rest.find('\0') { + PostgresQueryBuilder.write_escaped(buffer, &rest[..pos]); + let _ = buffer.write_str("\\000"); + rest = &rest[pos + 1..]; + } + PostgresQueryBuilder.write_escaped(buffer, rest); + } + + fn unescape_string(&self, string: &str) -> String { + PostgresQueryBuilder.unescape_string(string) + } +} impl TableRefBuilder for HyperQueryBuilder {} @@ -149,44 +203,169 @@ impl PrecedenceDecider for HyperQueryBuilder { } impl QueryBuilder for HyperQueryBuilder { - fn prepare_query_statement( - &self, - query: &SubQueryStatement, - sql: &mut impl sea_query::SqlWriter, - ) { - PostgresQueryBuilder.prepare_query_statement(query, sql); + // Statement dispatch is Hyper's own. Delegating this to + // `PostgresQueryBuilder` would render the whole statement with *its* + // `self`, so none of the overrides below would ever be consulted for + // anything nested inside a statement. + fn prepare_query_statement(&self, query: &SubQueryStatement, sql: &mut impl SqlWriter) { + match query { + SubQueryStatement::SelectStatement(stmt) => self.prepare_select_statement(stmt, sql), + SubQueryStatement::InsertStatement(stmt) => self.prepare_insert_statement(stmt, sql), + SubQueryStatement::UpdateStatement(stmt) => self.prepare_update_statement(stmt, sql), + SubQueryStatement::DeleteStatement(stmt) => self.prepare_delete_statement(stmt, sql), + SubQueryStatement::WithStatement(stmt) => self.prepare_with_query(stmt, sql), + } } - fn prepare_select_into(&self, into_table: &SelectInto, sql: &mut impl sea_query::SqlWriter) { + fn prepare_select_into(&self, into_table: &SelectInto, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_select_into(into_table, sql); } - fn prepare_explain_statement( - &self, - explain: &ExplainStatement, - sql: &mut impl sea_query::SqlWriter, - ) { + // KNOWN LIMITATION: the statement nested in an `EXPLAIN`, and the options, + // are rendered by `PostgresQueryBuilder` (`ExplainStatement`'s fields are + // crate-private in sea-query, so they cannot be re-rendered here). Every + // Hyper override is bypassed inside it: `Value::Char` outside ASCII panics, + // `Value::Enum` keeps its `::"type"` hint, and SQLite operators panic. + fn prepare_explain_statement(&self, explain: &ExplainStatement, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_explain_statement(explain, sql); } - fn prepare_value(&self, value: Value, sql: &mut impl sea_query::SqlWriter) { - PostgresQueryBuilder.prepare_value(value, sql); + fn prepare_value(&self, value: Value, sql: &mut impl SqlWriter) { + // Hyper has no enum types: bind the variant as a plain string, with + // no `::"type"` hint (matches `write_value`'s inlined form). + let value = match value { + Value::Enum(OptionEnum::Some(variant)) => { + Value::String(Some(variant.value.into_owned())) + } + Value::Enum(OptionEnum::None(_)) => Value::String(None), + other => other, + }; + sql.push_param(value, self); } fn placeholder(&self) -> (&'static str, bool) { PostgresQueryBuilder.placeholder() } + + fn prepare_expr(&self, simple_expr: &Expr, sql: &mut impl SqlWriter) { + match simple_expr { + Expr::AsEnum(type_name, inner) => { + // `CAST( AS "")`, with embedded quotes doubled. + // A trailing `[]` stays outside the quotes (array type). + let name = type_name.inner(); + let mut base: &str = name.as_ref(); + let mut dims = 0; + while let Some(stripped) = base.strip_suffix("[]") { + base = stripped; + dims += 1; + } + let array_suffix = "[]".repeat(dims); + write_or_ignore(sql, "CAST("); + self.prepare_expr(inner, sql); + write_or_ignore(sql, " AS "); + write_or_ignore( + sql, + &format!("\"{}\"{array_suffix}", base.replace('"', "\"\"")), + ); + write_or_ignore(sql, ")"); + } + _ => self.prepare_expr_common(simple_expr, sql), + } + } + + fn write_value(&self, buf: &mut impl std::fmt::Write, value: &Value) -> std::fmt::Result { + match value { + // `Value::Char` holds a full Unicode scalar; sea-query's default + // truncates it to one byte and panics on non-ASCII input. + Value::Char(Some(c)) => { + self.write_string_quoted(c.encode_utf8(&mut [0u8; 4]), buf); + Ok(()) + } + // Hyper has no enum types: emit the bare string, no `::"type"` hint. + Value::Enum(OptionEnum::Some(variant)) => { + self.write_string_quoted(variant.value.as_ref(), buf); + Ok(()) + } + _ => self.write_value_common(buf, value), + } + } + + fn write_string_quoted(&self, string: &str, buffer: &mut impl std::fmt::Write) { + // Postgres' version escapes through *its* `write_escaped`, which would + // bypass the NUL fix above, so spell it out against `self`. + let prefix = if self.needs_escape(string) { "E'" } else { "'" }; + let _ = buffer.write_str(prefix); + self.write_escaped(buffer, string); + let _ = buffer.write_str("'"); + } + + fn write_bytes(&self, bytes: &[u8], buffer: &mut impl std::fmt::Write) { + PostgresQueryBuilder.write_bytes(bytes, buffer); + } + + fn if_null_function(&self) -> &str { + PostgresQueryBuilder.if_null_function() + } + + fn prepare_bin_oper(&self, bin_oper: &BinOper, sql: &mut impl SqlWriter) { + match bin_oper { + // Hyper rejects the operators it lacks (`@>`, `<@`, `&&`, `@@`, + // `%`, `<->`, ...) with a clean `42601`, so emit them verbatim. + BinOper::PgOperator(_) => PostgresQueryBuilder.prepare_bin_oper(bin_oper, sql), + // Hyper has no SQLite operators; write them out so the server + // answers with a syntax error instead of the builder panicking. + BinOper::SqliteOperator(oper) => match oper { + SqliteBinOper::GetJsonField => write_or_ignore(sql, "->"), + SqliteBinOper::CastJsonField => write_or_ignore(sql, "->>"), + SqliteBinOper::Glob => write_or_ignore(sql, "GLOB"), + SqliteBinOper::Match => write_or_ignore(sql, "MATCH"), + // `SqliteBinOper` is `#[non_exhaustive]`: a future variant gets + // a placeholder the server rejects, never a panic and never + // Rust `Debug` text in the SQL. + _ => write_or_ignore(sql, "/* unsupported SQLite operator */"), + }, + _ => self.prepare_bin_oper_common(bin_oper, sql), + } + } + + fn prepare_function_name(&self, function: &Func, sql: &mut impl SqlWriter) { + PostgresQueryBuilder.prepare_function_name(function, sql); + } + + fn prepare_table_sample(&self, select: &SelectStatement, sql: &mut impl SqlWriter) { + PostgresQueryBuilder.prepare_table_sample(select, sql); + } + + fn prepare_select_distinct(&self, select_distinct: &SelectDistinct, sql: &mut impl SqlWriter) { + PostgresQueryBuilder.prepare_select_distinct(select_distinct, sql); + } + + // KNOWN LIMITATION: `OrderExpr` has no public accessors, so the ORDER BY + // expression cannot be re-rendered here; it (and `Order::Field`) is + // rendered by `PostgresQueryBuilder`, bypassing Hyper overrides for + // `Value::Char` outside ASCII, `Value::Enum` and SQLite operators. Plain + // columns, strings (properly escaped), numbers and functions are fine. + fn prepare_order_expr(&self, order_expr: &OrderExpr, sql: &mut impl SqlWriter) { + PostgresQueryBuilder.prepare_order_expr(order_expr, sql); + } +} + +/// Write a fragment to a [`SqlWriter`]. Writing into sea-query's in-memory +/// `String` buffers cannot fail, so the `fmt::Result` is intentionally dropped +/// rather than turned into a panic. +fn write_or_ignore(sql: &mut impl SqlWriter, fragment: &str) { + let _ = sql.write_str(fragment); } impl ForeignKeyBuilder for HyperQueryBuilder { - fn prepare_table_ref_fk_stmt(&self, table_ref: &TableRef, sql: &mut impl sea_query::SqlWriter) { + fn prepare_table_ref_fk_stmt(&self, table_ref: &TableRef, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_table_ref_fk_stmt(table_ref, sql); } fn prepare_foreign_key_create_statement_internal( &self, create: &ForeignKeyCreateStatement, - sql: &mut impl sea_query::SqlWriter, + sql: &mut impl SqlWriter, mode: sea_query::backend::Mode, ) { PostgresQueryBuilder.prepare_foreign_key_create_statement_internal(create, sql, mode); @@ -195,7 +374,7 @@ impl ForeignKeyBuilder for HyperQueryBuilder { fn prepare_foreign_key_drop_statement_internal( &self, drop: &ForeignKeyDropStatement, - sql: &mut impl sea_query::SqlWriter, + sql: &mut impl SqlWriter, mode: sea_query::backend::Mode, ) { PostgresQueryBuilder.prepare_foreign_key_drop_statement_internal(drop, sql, mode); @@ -203,45 +382,70 @@ impl ForeignKeyBuilder for HyperQueryBuilder { } impl IndexBuilder for HyperQueryBuilder { + // KNOWN LIMITATION: `IndexCreateStatement` (and `TableAlterStatement` + // below) expose no public accessors, so they are rendered whole by + // `PostgresQueryBuilder`; a partial-index `WHERE` / `ALTER ... DEFAULT` + // expression bypasses Hyper overrides (non-ASCII `Value::Char`, `Value::Enum`, + // SQLite operators). fn prepare_index_create_statement( &self, create: &IndexCreateStatement, - sql: &mut impl sea_query::SqlWriter, + sql: &mut impl SqlWriter, ) { PostgresQueryBuilder.prepare_index_create_statement(create, sql); } - fn prepare_table_ref_index_stmt( - &self, - table_ref: &TableRef, - sql: &mut impl sea_query::SqlWriter, - ) { + fn prepare_table_ref_index_stmt(&self, table_ref: &TableRef, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_table_ref_index_stmt(table_ref, sql); } - fn prepare_index_drop_statement( - &self, - drop: &IndexDropStatement, - sql: &mut impl sea_query::SqlWriter, - ) { + fn prepare_index_drop_statement(&self, drop: &IndexDropStatement, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_index_drop_statement(drop, sql); } - fn prepare_index_prefix( + fn prepare_index_prefix(&self, create: &IndexCreateStatement, sql: &mut impl SqlWriter) { + PostgresQueryBuilder.prepare_index_prefix(create, sql); + } + + // Consulted by the default `CREATE TABLE ... INDEX / PRIMARY KEY` + // constraint path; Postgres adds `CONSTRAINT`, `NULLS NOT DISTINCT` and + // `INCLUDE`. Delegated whole, so the pieces it calls (index type, columns, + // filter) are Postgres' own and need no separate override. + fn prepare_table_index_expression( &self, create: &IndexCreateStatement, - sql: &mut impl sea_query::SqlWriter, + sql: &mut impl SqlWriter, ) { - PostgresQueryBuilder.prepare_index_prefix(create, sql); + PostgresQueryBuilder.prepare_table_index_expression(create, sql); } } impl TableBuilder for HyperQueryBuilder { - fn prepare_column_def(&self, column_def: &ColumnDef, sql: &mut impl sea_query::SqlWriter) { - PostgresQueryBuilder.prepare_column_def(column_def, sql); + // Hyper's own copy of Postgres' body (via the public accessors) so the + // `DEFAULT` / `CHECK` expressions are rendered by `self`. + fn prepare_column_def(&self, column_def: &ColumnDef, sql: &mut impl SqlWriter) { + let name = Alias::new(column_def.get_column_name()).into_iden(); + self.prepare_iden(&name, sql); + let spec = column_def.get_column_spec(); + if let Some(column_type) = column_def.get_column_type() { + write_or_ignore(sql, " "); + if spec.auto_increment { + match column_type { + ColumnType::SmallInteger => write_or_ignore(sql, "smallint"), + ColumnType::Integer => write_or_ignore(sql, "integer"), + ColumnType::BigInteger => write_or_ignore(sql, "bigint"), + // Postgres panics here; keep the type and let Hyper reject it. + other => self.prepare_column_type(other, sql), + } + write_or_ignore(sql, " GENERATED BY DEFAULT AS IDENTITY"); + } else { + self.prepare_column_type(column_type, sql); + } + } + self.prepare_column_spec(spec, sql); } - fn prepare_column_type(&self, column_type: &ColumnType, sql: &mut impl sea_query::SqlWriter) { + fn prepare_column_type(&self, column_type: &ColumnType, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_column_type(column_type, sql); } @@ -249,21 +453,86 @@ impl TableBuilder for HyperQueryBuilder { PostgresQueryBuilder.column_spec_auto_increment_keyword() } - fn prepare_table_alter_statement( - &self, - alter: &TableAlterStatement, - sql: &mut impl sea_query::SqlWriter, - ) { + fn prepare_table_alter_statement(&self, alter: &TableAlterStatement, sql: &mut impl SqlWriter) { PostgresQueryBuilder.prepare_table_alter_statement(alter, sql); } fn prepare_table_rename_statement( &self, rename: &TableRenameStatement, - sql: &mut impl sea_query::SqlWriter, + sql: &mut impl SqlWriter, ) { PostgresQueryBuilder.prepare_table_rename_statement(rename, sql); } + + // Hyper has no declarative partitioning. sea-query's default hooks would + // silently drop `PARTITION BY` / `FOR VALUES`, leaving an unpartitioned + // table the caller thinks is partitioned; write the Postgres syntax so + // the server rejects it. Postgres' own hooks panic on the MySQL-only + // `KEY` / `LESS THAN` forms, so those are spelled out here instead. + fn prepare_partition_by(&self, partition_by: &PartitionBy, sql: &mut impl SqlWriter) { + let (keyword, cols) = match partition_by { + PartitionBy::Range(cols) => ("RANGE", cols), + PartitionBy::List(cols) => ("LIST", cols), + PartitionBy::Hash(cols) => ("HASH", cols), + PartitionBy::Key(cols) => ("KEY", cols), + }; + write_or_ignore(sql, keyword); + write_or_ignore(sql, " ("); + write_partition_cols(*self, cols, sql); + write_or_ignore(sql, ")"); + } + + fn prepare_partition_values( + &self, + partition_values: &PartitionValues, + sql: &mut impl SqlWriter, + ) { + write_or_ignore(sql, "FOR VALUES "); + match partition_values { + PartitionValues::In(values) => { + write_or_ignore(sql, "IN ("); + write_partition_exprs(*self, values, sql); + write_or_ignore(sql, ")"); + } + PartitionValues::FromTo(from, to) => { + write_or_ignore(sql, "FROM ("); + write_partition_exprs(*self, from, sql); + write_or_ignore(sql, ") TO ("); + write_partition_exprs(*self, to, sql); + write_or_ignore(sql, ")"); + } + PartitionValues::LessThan(values) => { + write_or_ignore(sql, "LESS THAN ("); + write_partition_exprs(*self, values, sql); + write_or_ignore(sql, ")"); + } + PartitionValues::With(modulus, remainder) => { + write_or_ignore( + sql, + &format!("WITH (MODULUS {modulus}, REMAINDER {remainder})"), + ); + } + } + } +} + +fn write_partition_cols(builder: HyperQueryBuilder, cols: &[DynIden], sql: &mut impl SqlWriter) { + for (index, col) in cols.iter().enumerate() { + if index > 0 { + write_or_ignore(sql, ", "); + } + builder.prepare_iden(col, sql); + } +} + +fn write_partition_exprs(builder: HyperQueryBuilder, exprs: &[Expr], sql: &mut impl SqlWriter) { + for (index, expr) in exprs.iter().enumerate() { + if index > 0 { + write_or_ignore(sql, ", "); + } + builder.prepare_expr(expr, sql); + } } impl SchemaBuilder for HyperQueryBuilder {} @@ -273,7 +542,7 @@ impl GenericBuilder for HyperQueryBuilder {} #[cfg(test)] mod tests { use super::*; - use sea_query::{ExprTrait, Iden, Query}; + use sea_query::{ExprTrait, Iden, Query, Table}; #[derive(Iden)] enum Users { @@ -295,7 +564,7 @@ mod tests { assert_eq!( sql, query.to_string(PostgresQueryBuilder), - "HyperQueryBuilder should produce identical SQL to PostgresQueryBuilder" + "for plain SELECTs HyperQueryBuilder must match PostgresQueryBuilder's wording" ); } @@ -364,10 +633,531 @@ mod tests { assert_eq!(sql, stmt.to_string(PostgresQueryBuilder)); } + // ---- Q1: string / bytes / char escaping (security) ---------------------- + + fn constant_eq(text: &str) -> String { + Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where( + Expr::col(Users::Name).eq(Expr::Constant(Value::String(Some(text.to_owned())))), + ) + .to_string(HyperQueryBuilder) + } + + #[test] + fn constant_string_injection_is_escaped() { + assert_eq!( + constant_eq("x\\' OR 1=1 --"), + r#"SELECT "id" FROM "users" WHERE "name" = E'x\\\' OR 1=1 --'"# + ); + } + + #[test] + fn constant_plain_string_is_plain_literal() { + assert_eq!( + constant_eq("abc"), + r#"SELECT "id" FROM "users" WHERE "name" = 'abc'"# + ); + } + + #[test] + fn constant_string_with_nul_never_emits_raw_nul() { + let sql = constant_eq("a\0b"); + assert!(!sql.contains('\0'), "raw NUL in SQL text: {sql:?}"); + assert!(sql.ends_with(r"= E'a\000b'"), "{sql}"); + } + + #[test] + fn nul_followed_by_digits_cannot_extend_the_octal_escape() { + // `\0` + `47` would read back as `\047`, a quote, and break out of the literal. + let sql = constant_eq("\u{0}47' OR 1=1 --"); + assert!(sql.contains(r"E'\00047\' OR 1=1 --'"), "{sql}"); + } + + #[test] + fn column_default_char_non_ascii_is_rendered_by_hyper() { + let sql = Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Name).char().default('\u{e9}')) + .to_string(HyperQueryBuilder); + assert!(sql.contains("DEFAULT '\u{e9}'"), "{sql}"); + } + + #[test] + fn column_default_enum_is_a_bare_string() { + let v = Value::Enum(OptionEnum::Some(Box::new(sea_query::Enum { + type_name: "mood".into(), + value: "happy".into(), + }))); + let sql = Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Name).string().default(Expr::val(v))) + .to_string(HyperQueryBuilder); + assert!(sql.contains("DEFAULT 'happy'"), "{sql}"); + assert!(!sql.contains("mood"), "{sql}"); + } + + #[test] + fn column_auto_increment_matches_postgres_identity() { + let build = |b: &dyn Fn(&mut ColumnDef) -> &mut ColumnDef| { + let mut c = ColumnDef::new(Users::Id); + b(&mut c); + Table::create().table(Users::Table).col(c).to_owned() + }; + let t = build(&|c| c.integer().auto_increment().primary_key()); + assert_eq!( + t.to_string(HyperQueryBuilder), + t.to_string(PostgresQueryBuilder) + ); + } + + #[test] + fn column_auto_increment_on_non_integer_does_not_panic() { + let t = Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Name).string().auto_increment()) + .to_owned(); + let sql = t.to_string(HyperQueryBuilder); + assert!(sql.contains("varchar"), "{sql}"); + } + + #[test] + fn build_turns_enum_values_into_plain_strings() { + let v = Value::Enum(OptionEnum::Some(Box::new(sea_query::Enum { + type_name: "mood".into(), + value: "happy".into(), + }))); + let (sql, values) = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where(Expr::col(Users::Name).eq(Expr::val(v))) + .build(HyperQueryBuilder); + assert!(!sql.contains("::"), "{sql}"); + assert_eq!(values.0, vec![Value::String(Some("happy".into()))]); + } + + #[test] + fn as_enum_strips_every_array_suffix() { + let sql = Query::select() + .expr(Expr::col(Users::Name).as_enum("my_type[][]")) + .from(Users::Table) + .to_string(HyperQueryBuilder); + assert_eq!(sql, r#"SELECT CAST("name" AS "my_type"[][]) FROM "users""#); + } + + #[test] + fn like_escape_character_uses_standard_escaping() { + use sea_query::LikeExpr; + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where(Expr::col(Users::Name).like(LikeExpr::new(r"a\_%").escape('\\'))) + .to_string(HyperQueryBuilder); + assert!(sql.ends_with(r"ESCAPE E'\\'"), "{sql}"); + } + + #[test] + fn order_field_values_are_escaped() { + use sea_query::{Order, Values}; + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .order_by_expr( + Expr::col(Users::Name), + Order::Field(Values(vec!["a'b".into(), r"c\d".into()])), + ) + .to_string(HyperQueryBuilder); + assert!( + sql.ends_with( + r#"ORDER BY CASE WHEN "name"=E'a\'b' THEN 0 WHEN "name"=E'c\\d' THEN 1 ELSE 2 END"# + ), + "{sql}" + ); + } + + #[test] + fn inject_parameters_matches_to_string() { + let query = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where(Expr::col(Users::Name).eq("x\\' OR 1=1 --")) + .and_where(Expr::col(Users::Age).eq("tab\there")) + .to_owned(); + let (sql, values) = query.build(HyperQueryBuilder); + let injected = sea_query::prepare::inject_parameters(&sql, &values.0, &HyperQueryBuilder); + assert_eq!(injected, query.to_string(HyperQueryBuilder)); + assert!(injected.contains(r"E'x\\\' OR 1=1 --'"), "{injected}"); + } + + #[test] + fn value_to_string_uses_postgres_escaping() { + let v = Value::String(Some(r"a\b'c".to_owned())); + assert_eq!(HyperQueryBuilder.value_to_string(&v), r"E'a\\b\'c'"); + } + + #[test] + fn escape_string_round_trips_with_postgres_rules() { + let raw = "a\"b\\c'"; + let escaped = HyperQueryBuilder.escape_string(raw); + assert_eq!(escaped, r#"a"b\\c\'"#); + assert_eq!(HyperQueryBuilder.unescape_string(&escaped), raw); + } + + #[test] + fn non_ascii_char_does_not_panic() { + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where(Expr::col(Users::Name).eq(Expr::Constant(Value::Char(Some('é'))))) + .to_string(HyperQueryBuilder); + assert!(sql.ends_with("= 'é'"), "{sql}"); + let emoji = HyperQueryBuilder.value_to_string(&Value::Char(Some('😀'))); + assert_eq!(emoji, "'😀'"); + } + + #[test] + fn bytes_use_hex_escape_format() { + let v = Value::Bytes(Some(vec![0x00, 0x01, 0xff])); + let text = HyperQueryBuilder.value_to_string(&v); + assert_eq!(text, PostgresQueryBuilder.value_to_string(&v)); + assert!(text.starts_with(r"'\x"), "{text}"); + } + + // ---- Q3: operators and functions must not panic --------------------------- + + fn pg_ops() -> Vec<(sea_query::extension::postgres::PgBinOper, &'static str)> { + use sea_query::extension::postgres::PgBinOper as P; + vec![ + (P::ILike, "ILIKE"), + (P::NotILike, "NOT ILIKE"), + (P::Matches, "@@"), + (P::Contains, "@>"), + (P::Contained, "<@"), + (P::Concatenate, "||"), + (P::Overlap, "&&"), + (P::Similarity, "%"), + (P::WordSimilarity, "<%"), + (P::StrictWordSimilarity, "<<%"), + (P::SimilarityDistance, "<->"), + (P::WordSimilarityDistance, "<<->"), + (P::StrictWordSimilarityDistance, "<<<->"), + (P::GetJsonField, "->"), + (P::CastJsonField, "->>"), + (P::Regex, "~"), + (P::RegexCaseInsensitive, "~*"), + ] + } + + #[test] + fn every_pg_bin_oper_renders() { + for (op, text) in pg_ops() { + let query = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where(Expr::col(Users::Name).binary(BinOper::PgOperator(op), Expr::val("x"))) + .to_owned(); + let expected = format!(r#"SELECT "id" FROM "users" WHERE "name" {text} 'x'"#); + assert_eq!(query.to_string(HyperQueryBuilder), expected); + assert_eq!( + query.to_string(HyperQueryBuilder), + query.to_string(PostgresQueryBuilder) + ); + } + } + + #[test] + fn pg_expr_helpers_render() { + use sea_query::extension::postgres::PgExpr; + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where(Expr::col(Users::Name).ilike("a%")) + .and_where(Expr::col(Users::Name).not_ilike("b%")) + .and_where(Expr::col(Users::Name).concat("z").eq("yz")) + .to_string(HyperQueryBuilder); + // `PgExpr` is in scope here and blankets `contains` onto anything + // `Into`, so name the `str` method explicitly. + assert!(str::contains(&sql, r#""name" ILIKE 'a%'"#), "{sql}"); + assert!(str::contains(&sql, r#""name" NOT ILIKE 'b%'"#), "{sql}"); + assert!(str::contains(&sql, r#""name" || 'z'"#), "{sql}"); + } + + #[test] + fn sqlite_operators_never_panic() { + use sea_query::extension::sqlite::SqliteBinOper as S; + for (op, text) in [ + (S::GetJsonField, "->"), + (S::CastJsonField, "->>"), + (S::Glob, "GLOB"), + (S::Match, "MATCH"), + ] { + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .and_where( + Expr::col(Users::Name).binary(BinOper::SqliteOperator(op), Expr::val("x")), + ) + .to_string(HyperQueryBuilder); + assert!(sql.contains(&format!(r#""name" {text} 'x'"#)), "{sql}"); + } + } + + #[test] + fn pg_functions_render() { + use sea_query::PgDateTruncUnit; + use sea_query::extension::postgres::PgFunc; + let sql = Query::select() + .expr(PgFunc::starts_with(Expr::col(Users::Name), "ab")) + .expr(PgFunc::date_trunc( + PgDateTruncUnit::Day, + Expr::col(Users::Age), + )) + .expr(PgFunc::gen_random_uuid()) + .from(Users::Table) + .to_string(HyperQueryBuilder); + assert_eq!( + sql, + r#"SELECT STARTS_WITH("name", 'ab'), DATE_TRUNC('day', "age"), GEN_RANDOM_UUID() FROM "users""# + ); + } + + #[test] + fn as_enum_keeps_its_cast() { + let sql = Query::select() + .expr(Expr::col(Users::Name).as_enum("my_type")) + .from(Users::Table) + .to_string(HyperQueryBuilder); + assert_eq!(sql, r#"SELECT CAST("name" AS "my_type") FROM "users""#); + } + + #[test] + fn table_sample_is_rendered() { + use sea_query::extension::postgres::{PostgresSelectStatementExt, SampleMethod}; + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .table_sample(SampleMethod::BERNOULLI, 50.0, Some(7.0)) + .to_string(HyperQueryBuilder); + assert_eq!( + sql, + r#"SELECT "id" FROM "users" TABLESAMPLE BERNOULLI (50) REPEATABLE (7)"# + ); + } + + #[test] + fn if_null_is_coalesce() { + use sea_query::Func; + let sql = Query::select() + .expr(Func::if_null(Expr::col(Users::Name), "n/a")) + .from(Users::Table) + .to_string(HyperQueryBuilder); + assert_eq!(sql, r#"SELECT COALESCE("name", 'n/a') FROM "users""#); + } + + // ---- Q2: DISTINCT ON and NULLS ordering ------------------------------------ + + #[test] + fn distinct_on_is_rendered() { + let sql = Query::select() + .distinct_on([Users::Id]) + .columns([Users::Id, Users::Name]) + .from(Users::Table) + .to_string(HyperQueryBuilder); + assert_eq!( + sql, + r#"SELECT DISTINCT ON ("id") "id", "name" FROM "users""# + ); + } + + #[test] + fn distinct_on_multi_column_and_subquery() { + let inner = Query::select() + .distinct_on([Users::Id, Users::Name]) + .columns([Users::Id, Users::Name]) + .from(Users::Table) + .to_owned(); + let sql = Query::select() + .column(Users::Id) + .from_subquery(inner, "s") + .to_string(HyperQueryBuilder); + assert_eq!( + sql, + r#"SELECT "id" FROM (SELECT DISTINCT ON ("id", "name") "id", "name" FROM "users") AS "s""# + ); + } + + #[test] + fn nulls_ordering_matrix() { + use sea_query::{NullOrdering, Order}; + for (order, o_txt) in [(Order::Asc, "ASC"), (Order::Desc, "DESC")] { + for (nulls, n_txt) in [(NullOrdering::First, "FIRST"), (NullOrdering::Last, "LAST")] { + let sql = Query::select() + .column(Users::Id) + .from(Users::Table) + .order_by_with_nulls(Users::Age, order.clone(), nulls) + .to_string(HyperQueryBuilder); + assert_eq!( + sql, + format!(r#"SELECT "id" FROM "users" ORDER BY "age" {o_txt} NULLS {n_txt}"#) + ); + } + } + } + + #[test] + fn nulls_ordering_inside_subquery() { + use sea_query::{NullOrdering, Order}; + let inner = Query::select() + .column(Users::Id) + .from(Users::Table) + .order_by_with_nulls(Users::Age, Order::Desc, NullOrdering::Last) + .to_owned(); + let sql = Query::select() + .column(Users::Id) + .from_subquery(inner, "s") + .to_string(HyperQueryBuilder); + assert!(sql.contains(r#"ORDER BY "age" DESC NULLS LAST"#), "{sql}"); + } + + // ---- DDL hooks and Hyper-vs-Postgres parity -------------------------------- + + #[test] + fn inline_table_index_matches_postgres() { + use sea_query::{ColumnDef, Index, IndexOrder, IndexType, Table}; + let stmt = Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Id).integer().not_null()) + .col(ColumnDef::new(Users::Name).string()) + .index( + Index::create() + .name("users_name_uq") + .col((Users::Name, IndexOrder::Desc)) + .unique() + .nulls_not_distinct() + .index_type(IndexType::Hash), + ) + .to_owned(); + let hyper = stmt.to_string(HyperQueryBuilder); + assert_eq!(hyper, stmt.to_string(PostgresQueryBuilder)); + assert!(hyper.contains("NULLS NOT DISTINCT"), "{hyper}"); + } + + #[test] + fn partition_by_is_never_silently_dropped_and_never_panics() { + use sea_query::{ColumnDef, Table}; + for (stmt, keyword) in [ + ( + Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Id).integer()) + .partition_by_range([Users::Id]) + .to_owned(), + r#"PARTITION BY RANGE ("id")"#, + ), + ( + Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Id).integer()) + .partition_by_list([Users::Id]) + .to_owned(), + r#"PARTITION BY LIST ("id")"#, + ), + ( + Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Id).integer()) + .partition_by_hash([Users::Id]) + .to_owned(), + r#"PARTITION BY HASH ("id")"#, + ), + ( + // MySQL-only form: Postgres' builder panics, ours must not. + Table::create() + .table(Users::Table) + .col(ColumnDef::new(Users::Id).integer()) + .partition_by_key([Users::Id]) + .to_owned(), + r#"PARTITION BY KEY ("id")"#, + ), + ] { + let sql = stmt.to_string(HyperQueryBuilder); + assert!(sql.contains(keyword), "{sql}"); + } + } + + #[test] + fn dml_and_cte_match_postgres() { + use sea_query::{ + CommonTableExpression, JoinType, OnConflict, Order, Query, UnionType, WithClause, + }; + let insert = Query::insert() + .into_table(Users::Table) + .columns([Users::Name, Users::Age]) + .values_panic(["it's".into(), 3.into()]) + .on_conflict( + OnConflict::column(Users::Id) + .update_column(Users::Name) + .to_owned(), + ) + .returning_col(Users::Id) + .to_owned(); + assert_eq!( + insert.to_string(HyperQueryBuilder), + insert.to_string(PostgresQueryBuilder) + ); + + let union = Query::select() + .column(Users::Id) + .from(Users::Table) + .union( + UnionType::All, + Query::select() + .column(Users::Id) + .from(Users::Table) + .to_owned(), + ) + .join( + JoinType::LeftJoin, + Users::Table, + Expr::col((Users::Table, Users::Id)).equals((Users::Table, Users::Age)), + ) + .order_by(Users::Id, Order::Desc) + .limit(5) + .offset(2) + .to_owned(); + assert_eq!( + union.to_string(HyperQueryBuilder), + union.to_string(PostgresQueryBuilder) + ); + + let cte = CommonTableExpression::new() + .query( + Query::select() + .column(Users::Id) + .from(Users::Table) + .to_owned(), + ) + .table_name("c") + .to_owned(); + let with = Query::select() + .column(Users::Id) + .from("c") + .to_owned() + .with(WithClause::new().cte(cte).to_owned()); + assert_eq!( + with.to_string(HyperQueryBuilder), + with.to_string(PostgresQueryBuilder) + ); + } + #[test] - fn default_trait() { - let builder = HyperQueryBuilder; - // Touch `builder` so the test body demonstrates the unit-struct is constructible. - let _ = builder; + fn enum_values_are_bare_strings() { + // Hyper has no enum types, so unlike Postgres no `::"type"` hint. + let v = Value::Enum(OptionEnum::Some(Box::new(sea_query::Enum { + type_name: "mood".into(), + value: "ha'ppy".into(), + }))); + assert_eq!(HyperQueryBuilder.value_to_string(&v), r"E'ha\'ppy'"); } } diff --git a/sea-query-hyperdb/tests/hyper_integration.rs b/sea-query-hyperdb/tests/hyper_integration.rs new file mode 100644 index 00000000..3585630b --- /dev/null +++ b/sea-query-hyperdb/tests/hyper_integration.rs @@ -0,0 +1,406 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! End-to-end tests: SQL produced by [`HyperQueryBuilder`] is executed on a +//! real `hyperd`, so the dialect claims are checked against the engine rather +//! than against sea-query's idea of Postgres. +//! +//! Needs `hyperd` (see `AGENTS.md`: `make download-hyperd`, then an absolute +//! `HYPERD_PATH`). + +use hyperdb_api::{Connection, CreateMode, HyperProcess, Parameters}; +use sea_query::extension::postgres::{ + PgBinOper, PgExpr, PgFunc, PostgresSelectStatementExt, SampleMethod, +}; +use sea_query::{ + Alias, BinOper, Expr, ExprTrait, Iden, LikeExpr, NullOrdering, Order, PgDateTruncUnit, Query, + Value, Values, +}; +use sea_query_hyperdb::HyperQueryBuilder; + +#[derive(Iden)] +enum T { + Table, + Id, + S, +} + +/// A running `hyperd` plus a fresh database; both are cleaned up on drop. +struct Fixture { + // Field order matters: the connection must drop before the process. + conn: Connection, + _hyper: HyperProcess, + dir: std::path::PathBuf, +} + +impl Fixture { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!("sq_hyperdb_{name}_{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let mut params = Parameters::new(); + params.set("log_dir", dir.to_string_lossy().to_string()); + let hyper = HyperProcess::new(None, Some(¶ms)).expect("start hyperd"); + let conn = Connection::new(&hyper, dir.join("t.hyper"), CreateMode::CreateAndReplace) + .expect("connect"); + Self { + conn, + _hyper: hyper, + dir, + } + } + + fn exec(&self, sql: &str) { + self.conn + .execute_command(sql) + .unwrap_or_else(|e| panic!("`{sql}` failed: {e}")); + } + + fn strings(&self, sql: &str) -> Vec> { + self.conn + .fetch_all(sql) + .unwrap_or_else(|e| panic!("`{sql}` failed: {e}")) + .iter() + .map(|row| row.get::(0)) + .collect() + } + + fn one_string(&self, sql: &str) -> String { + self.strings(sql) + .pop() + .flatten() + .unwrap_or_else(|| panic!("`{sql}` returned no value")) + } + + fn count(&self, sql: &str) -> usize { + self.conn + .fetch_all(sql) + .unwrap_or_else(|e| panic!("`{sql}` failed: {e}")) + .len() + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + // Best effort: the temp dir only holds test artifacts. + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn hostile_strings() -> Vec { + // Every ASCII scalar except NUL (Hyper text cannot hold NUL), one at a time. + let mut all: Vec = (1u8..=127).map(|b| char::from(b).to_string()).collect(); + all.extend( + [ + "é", + "😀", + r"\", + "'", + "\"", + r"\'", + r"\\'", + "x\\' OR 1=1 --", + "'; DROP TABLE t; --", + "a\nb\tc\rd\u{8}e\u{c}f", + r"\x41A\101\n", + "%_\\%", + ] + .map(String::from), + ); + all +} + +#[test] +fn hostile_strings_round_trip_through_every_path() { + let fx = Fixture::new("roundtrip"); + for text in hostile_strings() { + // Expr::val -> placeholder + Values, then re-inlined with inject_parameters. + let select = Query::select().expr(Expr::val(text.as_str())).to_owned(); + let to_string = fx.one_string(&select.to_string(HyperQueryBuilder)); + assert_eq!(to_string, text, "Expr::val via to_string"); + + let (sql, values) = select.build(HyperQueryBuilder); + let injected = sea_query::prepare::inject_parameters(&sql, &values.0, &HyperQueryBuilder); + assert_eq!(fx.one_string(&injected), text, "inject_parameters"); + + // Expr::Constant goes through Hyper's own `write_value` / `write_string_quoted`. + let constant = Query::select() + .expr(Expr::Constant(Value::String(Some(text.clone())))) + .to_string(HyperQueryBuilder); + assert_eq!(fx.one_string(&constant), text, "Expr::Constant"); + } +} + +#[test] +fn order_field_round_trips_hostile_strings() { + let fx = Fixture::new("orderfield"); + fx.exec(r#"CREATE TABLE "t" ("id" INT, "s" TEXT)"#); + let hostile = hostile_strings(); + for (id, text) in hostile.iter().enumerate() { + let insert = Query::insert() + .into_table(T::Table) + .columns([T::Id, T::S]) + .values_panic([ + i32::try_from(id).expect("small index").into(), + text.as_str().into(), + ]) + .to_string(HyperQueryBuilder); + fx.exec(&insert); + } + for text in &hostile { + // The row equal to `text` must sort first iff `text` survived escaping. + let sql = Query::select() + .column(T::S) + .from(T::Table) + .order_by_expr( + Expr::col(T::S), + Order::Field(Values(vec![text.as_str().into()])), + ) + .limit(1) + .to_string(HyperQueryBuilder); + assert_eq!(fx.one_string(&sql), *text, "Order::Field"); + } +} + +#[test] +fn constant_injection_attempt_matches_nothing() { + let fx = Fixture::new("injection"); + fx.exec(r#"CREATE TABLE "t" ("id" INT, "s" TEXT)"#); + fx.exec(r#"INSERT INTO "t" VALUES (1, 'a'), (2, 'b'), (3, 'c')"#); + let sql = Query::select() + .column(T::Id) + .from(T::Table) + .and_where(Expr::col(T::S).eq(Expr::Constant(Value::String(Some( + "x\\' OR 1=1 --".to_owned(), + ))))) + .to_string(HyperQueryBuilder); + assert_eq!(fx.count(&sql), 0, "{sql}"); +} + +#[test] +fn like_escape_clause_is_accepted() { + let fx = Fixture::new("like"); + fx.exec(r#"CREATE TABLE "t" ("id" INT, "s" TEXT)"#); + fx.exec(r#"INSERT INTO "t" VALUES (1, 'a_b'), (2, 'axb')"#); + let sql = Query::select() + .column(T::Id) + .from(T::Table) + .and_where(Expr::col(T::S).like(LikeExpr::new(r"a\_b").escape('\\'))) + .to_string(HyperQueryBuilder); + assert_eq!(fx.count(&sql), 1, "{sql}"); +} + +#[test] +fn bytes_round_trip() { + let fx = Fixture::new("bytes"); + let payload: Vec = vec![0x00, 0x01, 0x27, 0x5c, 0x7f, 0x80, 0xff]; + // The literal is `'\x...'` (untyped, like Postgres), so it is read back + // through a BYTEA column rather than a bare SELECT. + fx.exec(r#"CREATE TABLE "b" ("v" BYTEA)"#); + let insert = Query::insert() + .into_table(Alias::new("b")) + .columns([Alias::new("v")]) + .values_panic([Expr::Constant(Value::Bytes(Some(payload.clone())))]) + .to_string(HyperQueryBuilder); + fx.exec(&insert); + let sql = Query::select() + .column(Alias::new("v")) + .from(Alias::new("b")) + .to_string(HyperQueryBuilder); + let row = fx + .conn + .fetch_one(&sql) + .unwrap_or_else(|e| panic!("`{sql}` failed: {e}")); + assert_eq!(row.get::>(0), Some(payload)); +} + +#[test] +fn nul_round_trips_exactly_and_cannot_end_the_literal() { + let fx = Fixture::new("nul"); + for text in ["a\0b", "\u{0}47' OR 1=1 --", "\u{0}\u{0}7"] { + let sql = Query::select() + .expr(Expr::Constant(Value::String(Some(text.to_owned())))) + .to_string(HyperQueryBuilder); + assert!(!str::contains(&sql, '\0'), "raw NUL in SQL text"); + assert_eq!(fx.one_string(&sql), text, "sql: {sql}"); + } + assert_eq!(fx.one_string("SELECT 'still alive'"), "still alive"); +} + +#[test] +fn non_ascii_char_constant_executes() { + let fx = Fixture::new("char"); + let sql = Query::select() + .expr(Expr::Constant(Value::Char(Some('é')))) + .to_string(HyperQueryBuilder); + assert_eq!(fx.one_string(&sql), "é"); +} + +#[test] +fn postgres_operators_and_functions_execute() { + let fx = Fixture::new("operators"); + fx.exec(r#"CREATE TABLE "t" ("id" INT, "s" TEXT)"#); + fx.exec(r#"INSERT INTO "t" VALUES (1, 'Alpha'), (2, 'beta'), (3, 'Gamma')"#); + + let count_where = |cond: Expr| { + let sql = Query::select() + .column(T::Id) + .from(T::Table) + .and_where(cond) + .to_string(HyperQueryBuilder); + fx.count(&sql) + }; + + assert_eq!(count_where(Expr::col(T::S).ilike("alp%")), 1); + assert_eq!(count_where(Expr::col(T::S).not_ilike("alp%")), 2); + assert_eq!( + count_where(Expr::col(T::S).binary(PgBinOper::Regex, Expr::val("^[A-Z]"))), + 2 + ); + assert_eq!( + count_where(Expr::col(T::S).binary(PgBinOper::RegexCaseInsensitive, Expr::val("^b"))), + 1 + ); + assert_eq!( + count_where(Expr::col(T::S).binary(BinOper::Custom("!~"), Expr::val("^[A-Z]"))), + 1 + ); + assert_eq!(count_where(Expr::col(T::S).concat("!").eq("beta!")), 1); + assert_eq!( + count_where(Expr::expr(PgFunc::starts_with(Expr::col(T::S), "Ga"))), + 1 + ); + assert_eq!( + count_where( + Expr::col(T::Id) + .eq(Expr::expr(PgFunc::any(Expr::cust("ARRAY[2,3]")))) + .not() + ), + 1 + ); +} + +#[test] +fn date_trunc_and_json_operators_execute() { + let fx = Fixture::new("json"); + let sql = Query::select() + .expr( + Expr::expr(PgFunc::date_trunc( + PgDateTruncUnit::Day, + Expr::cust("TIMESTAMP '2024-05-06 07:08:09'"), + )) + .cast_as(Alias::new("text")), + ) + .to_string(HyperQueryBuilder); + assert!(fx.one_string(&sql).starts_with("2024-05-06"), "{sql}"); + + let sql = Query::select() + .expr(Expr::cust(r#"'{"a": "v"}'::json"#).cast_json_field("a")) + .to_string(HyperQueryBuilder); + assert_eq!(fx.one_string(&sql), "v", "{sql}"); +} + +#[test] +fn unsupported_operator_is_a_server_error_not_a_panic() { + let fx = Fixture::new("unsupported"); + let sql = Query::select() + .expr(Expr::cust("ARRAY[1,2]").contains(Expr::cust("ARRAY[1]"))) + .to_string(HyperQueryBuilder); + assert!(str::contains(&sql, "@>"), "{sql}"); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + fx.conn.fetch_all(&sql).map(|rows| rows.len()) + })); + let result = outcome.expect("builder or client panicked"); + let err = result.expect_err("Hyper has no @> operator"); + assert_eq!(err.sqlstate(), Some("42601"), "unexpected error: {err:?}"); +} + +#[test] +fn cast_tablesample_and_coalesce_execute() { + let fx = Fixture::new("misc"); + fx.exec(r#"CREATE TABLE "t" ("id" INT, "s" TEXT)"#); + fx.exec(r#"INSERT INTO "t" VALUES (1, NULL), (2, 'b'), (3, 'c')"#); + + let cast = Query::select() + .expr(Expr::val("12").as_enum(Alias::new("text"))) + .to_string(HyperQueryBuilder); + assert!(str::contains(&cast, r#"CAST('12' AS "text")"#), "{cast}"); + assert_eq!(fx.one_string(&cast), "12", "{cast}"); + + let sample = Query::select() + .column(T::Id) + .from(T::Table) + .table_sample(SampleMethod::BERNOULLI, 100.0, Some(1.0)) + .to_string(HyperQueryBuilder); + assert!(str::contains(&sample, "REPEATABLE"), "{sample}"); + assert_eq!(fx.count(&sample), 3, "{sample}"); + + let coalesce = Query::select() + .expr(sea_query::Func::if_null(Expr::col(T::S), "none")) + .from(T::Table) + .order_by(T::Id, Order::Asc) + .to_string(HyperQueryBuilder); + assert!(str::contains(&coalesce, "COALESCE"), "{coalesce}"); + assert_eq!( + fx.strings(&coalesce), + vec![ + Some("none".to_owned()), + Some("b".to_owned()), + Some("c".to_owned()) + ] + ); +} + +#[test] +fn distinct_on_and_nulls_ordering_execute() { + let fx = Fixture::new("distinct"); + fx.exec(r#"CREATE TABLE "t" ("id" INT, "s" TEXT)"#); + fx.exec(r#"INSERT INTO "t" VALUES (1, 'a'), (1, 'a'), (2, NULL), (2, 'z'), (3, NULL)"#); + + let sql = Query::select() + .distinct_on([T::Id]) + .columns([T::Id, T::S]) + .from(T::Table) + .order_by(T::Id, Order::Asc) + .to_string(HyperQueryBuilder); + assert_eq!(fx.count(&sql), 3, "{sql}"); + + let first = |nulls: NullOrdering, order: Order| { + let sql = Query::select() + .column(T::S) + .from(T::Table) + .order_by_with_nulls(T::S, order, nulls) + .to_string(HyperQueryBuilder); + fx.strings(&sql) + }; + let nulls_first = first(NullOrdering::First, Order::Asc); + assert_eq!(nulls_first.first(), Some(&None)); + let nulls_last = first(NullOrdering::Last, Order::Desc); + assert_eq!(nulls_last.last(), Some(&None)); + assert_eq!(nulls_last.first(), Some(&Some("z".to_owned()))); +} + +#[test] +fn column_defaults_with_char_and_enum_execute() { + let fx = Fixture::new("defaults"); + let enum_value = Value::Enum(sea_query::OptionEnum::Some(Box::new(sea_query::Enum { + type_name: "mood".into(), + value: "happy".into(), + }))); + let ddl = sea_query::Table::create() + .table(Alias::new("d")) + .col( + sea_query::ColumnDef::new(Alias::new("c")) + .text() + .default('é'), + ) + .col( + sea_query::ColumnDef::new(Alias::new("m")) + .text() + .default(Expr::val(enum_value)), + ) + .to_string(HyperQueryBuilder); + fx.exec(&ddl); + fx.exec(r#"INSERT INTO "d" ("c") VALUES (DEFAULT)"#); + assert_eq!(fx.one_string(r#"SELECT "c" FROM "d""#), "é"); + assert_eq!(fx.one_string(r#"SELECT "m" FROM "d""#), "happy"); +} From 25301684f12cd67b1a521bbb04cfed92995e5a79 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 20:31:40 -0700 Subject: [PATCH 44/61] fix(core): drain or cancel abandoned result streams, wait out cancel connections, and avoid inserter drop hangs A late cancel request could land on the next query and fail it with 57014; the cancel now waits for the server to close the cancel connection. Dropping a MappedInserter while a result stream was live hung the thread. Abandoned streams drain up to a bounded ceiling before cancelling. --- hyperdb-api-core/CHANGELOG.md | 5 + hyperdb-api-core/src/client/async_client.rs | 27 ++ .../src/client/async_prepared_stream.rs | 54 ++- .../src/client/async_stream_query.rs | 54 ++- hyperdb-api-core/src/client/cancel.rs | 28 +- hyperdb-api-core/src/client/client.rs | 191 +++++++++-- hyperdb-api-core/src/client/connection.rs | 182 +++++++++-- .../src/client/prepared_stream.rs | 59 +++- hyperdb-api-core/src/client/tls.rs | 47 ++- hyperdb-api-core/tests/copy_tests.rs | 139 ++++++++ hyperdb-api/CHANGELOG.md | 8 + hyperdb-api/src/async_connection.rs | 6 + hyperdb-api/src/async_inserter.rs | 28 +- hyperdb-api/src/async_prepared.rs | 43 ++- hyperdb-api/src/async_result.rs | 35 +- hyperdb-api/src/connection.rs | 23 +- hyperdb-api/src/copy.rs | 20 +- hyperdb-api/src/inserter.rs | 246 ++++++++++---- hyperdb-api/src/prepared.rs | 10 + hyperdb-api/src/query_as.rs | 38 ++- hyperdb-api/src/result.rs | 19 +- hyperdb-api/src/transport.rs | 11 + hyperdb-api/tests/async_connection_tests.rs | 44 +++ hyperdb-api/tests/inserter_tests.rs | 308 ++++++++++++++++++ hyperdb-api/tests/wire_desync_tests.rs | 164 ++++++---- 25 files changed, 1531 insertions(+), 258 deletions(-) diff --git a/hyperdb-api-core/CHANGELOG.md b/hyperdb-api-core/CHANGELOG.md index 53b4f2ca..0d889b07 100644 --- a/hyperdb-api-core/CHANGELOG.md +++ b/hyperdb-api-core/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +- `Client::try_exec`, a non-blocking `exec` that returns `None` when the connection is busy (for destructors). +- `drain_to_ready()` on `QueryStream`, `PreparedQueryStream`, `AsyncQueryStream` and `AsyncPreparedQueryStream`, which consumes the rest of a result through `ReadyForQuery` so single-row callers can leave the connection in sync. - **`client::tls`: TLS for TCP connections**, negotiated with PostgreSQL's `SSLRequest` and libpq `sslmode` semantics. `TlsConfig` / `TlsMode` (with `ParseTlsModeError`) are set through `Config::with_tls` and read back with @@ -60,6 +62,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- A dropped `CopyInWriter` now cancels the COPY (`CopyFail`) instead of leaving the connection mid-COPY; `finish_copy`/`cancel_copy` are bounded by `POST_ERROR_DRAIN_CAP`, check connection health first, and mark the connection desynchronized on write/flush/read failure. The async COPY writer drop now warns when it cannot clean up. +- Dropping a sync `QueryStream` / `PreparedQueryStream` that still has a multi-chunk backlog no longer desynchronizes the connection: the post-cancel drain charges `DataRow`s against a separate ceiling (about 4.2 million rows) instead of the small message budget. The ceiling keeps a lost or ignored cancel from becoming an unbounded read inside a destructor; past it the connection is marked desynchronized. +- `Client::cancel` (and the async equivalent) now wait for the server to close the cancel connection before returning, so the cancel cannot land on the next query. A `CopyInWriter` dropped during panic unwinding no longer performs network I/O; it marks the connection desynchronized. A failed buffered write in `send_copy_data_direct` now also marks the connection desynchronized. - `protocol::escape::SqlIdentifier`, `format_table_name` and `escape_identifier` now always quote identifiers. They previously left lowercase names bare, so reserved words such as `order` produced invalid SQL. Output changes from diff --git a/hyperdb-api-core/src/client/async_client.rs b/hyperdb-api-core/src/client/async_client.rs index 75570f8c..7a689266 100644 --- a/hyperdb-api-core/src/client/async_client.rs +++ b/hyperdb-api-core/src/client/async_client.rs @@ -503,6 +503,16 @@ impl AsyncClient { ); Error::from_io(e) })?; + // Wait for the server's close (its "request processed" + // signal) so the cancel cannot land on the next query. + let close = async { + use tokio::io::AsyncReadExt; + stream.shutdown().await?; + let mut sink = [0u8; 64]; + while stream.read(&mut sink).await? > 0 {} + Ok::<(), std::io::Error>(()) + }; + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), close).await; } #[cfg(windows)] ConnectionEndpoint::NamedPipe { host, name } => { @@ -1247,6 +1257,16 @@ impl Drop for AsyncPreparedStatement { /// by queuing a `CopyFail` message in the connection's write buffer. The next /// async operation on the connection will flush and drain the cancel response, /// restoring the connection to a usable state. +/// +/// # Limitation +/// +/// If the connection mutex is held by another task at the moment of the drop, +/// the `CopyFail` cannot be queued and only a warning is logged. The +/// connection is then still inside the COPY sub-protocol, and later requests +/// on it will fail with a protocol error. The desynchronized flag lives +/// inside the same mutex, so `Drop` cannot set it without the lock; close and +/// replace such a connection. Always call [`finish()`](Self::finish) or +/// [`cancel()`](Self::cancel) explicitly. #[derive(Debug)] pub struct AsyncCopyInWriter<'a> { connection: &'a Mutex>, @@ -1358,6 +1378,13 @@ impl Drop for AsyncCopyInWriterOwned { // flush and drain. if let Ok(mut conn) = self.connection.try_lock() { conn.queue_copy_fail("AsyncCopyInWriterOwned dropped without finish/cancel"); + } else { + warn!( + target: "hyperdb_api_core::client", + "AsyncCopyInWriterOwned dropped without finish() or cancel(), \ + and the connection mutex was locked. The connection may be \ + left in an unusable COPY-IN state." + ); } } } diff --git a/hyperdb-api-core/src/client/async_prepared_stream.rs b/hyperdb-api-core/src/client/async_prepared_stream.rs index 104d1ae8..27ea1490 100644 --- a/hyperdb-api-core/src/client/async_prepared_stream.rs +++ b/hyperdb-api-core/src/client/async_prepared_stream.rs @@ -28,9 +28,9 @@ use super::statement::Column; /// triggers a best-effort server-side cancel and marks the connection /// desynchronized — same Drop semantics as /// [`AsyncQueryStream`](super::async_stream_query::AsyncQueryStream). -pub struct AsyncPreparedQueryStream<'a> { - conn: Option>>, - canceller: &'a dyn Cancellable, +pub struct AsyncPreparedQueryStream<'conn> { + conn: Option>>, + canceller: &'conn dyn Cancellable, finished: bool, chunk_size: usize, /// Columns captured at prepare time. @@ -47,10 +47,10 @@ impl std::fmt::Debug for AsyncPreparedQueryStream<'_> { } } -impl<'a> AsyncPreparedQueryStream<'a> { +impl<'conn> AsyncPreparedQueryStream<'conn> { pub(crate) fn new( - conn: MutexGuard<'a, AsyncRawConnection>, - canceller: &'a dyn Cancellable, + conn: MutexGuard<'conn, AsyncRawConnection>, + canceller: &'conn dyn Cancellable, chunk_size: usize, columns: Arc>, ) -> Self { @@ -122,6 +122,48 @@ impl<'a> AsyncPreparedQueryStream<'a> { } Ok(Some(rows)) } + + /// Reads and discards the rest of the response through `ReadyForQuery` + /// without building any rows, leaving the connection ready for the next + /// request. A no-op if the stream already finished. + /// + /// Use this after taking the rows you need from a stream that may hold + /// more data than one chunk: [`next_chunk`](Self::next_chunk) returns as + /// soon as it has `chunk_size` rows, *before* reading `CommandComplete` + /// and `ReadyForQuery`, so simply dropping the stream at that point + /// leaves the wire mid-response. + /// + /// # Errors + /// + /// - Returns `Error` (I/O) / `Error` (closed) if the transport fails + /// while reading. + /// - Returns `Error` (server) if the server sends an `ErrorResponse` + /// before `ReadyForQuery`; the error is still drained through + /// `ReadyForQuery` first. + pub async fn drain_to_ready(&mut self) -> Result<()> { + if self.finished { + return Ok(()); + } + let Some(conn) = self.conn.as_mut() else { + return Ok(()); + }; + loop { + match conn.read_message().await? { + Message::ReadyForQuery(_) => { + self.finished = true; + self.conn = None; + return Ok(()); + } + Message::ErrorResponse(body) => { + self.finished = true; + let err = conn.consume_error(&body).await; + self.conn = None; + return Err(err); + } + _ => {} + } + } + } } impl Drop for AsyncPreparedQueryStream<'_> { diff --git a/hyperdb-api-core/src/client/async_stream_query.rs b/hyperdb-api-core/src/client/async_stream_query.rs index 75985489..9653d2f9 100644 --- a/hyperdb-api-core/src/client/async_stream_query.rs +++ b/hyperdb-api-core/src/client/async_stream_query.rs @@ -28,14 +28,14 @@ use super::statement::{Column, ColumnFormat}; /// [`AsyncRawConnection`] locked via a `MutexGuard`. Dropping the stream /// before fully iterating triggers a server-side cancel (see [`Drop`] below) /// so the connection is returned to a usable state. -pub struct AsyncQueryStream<'a> { - conn: Option>>, +pub struct AsyncQueryStream<'conn> { + conn: Option>>, /// Best-effort cancel handle. For [`AsyncClient`](super::async_client::AsyncClient) /// this is a thin wrapper that opens a synchronous TCP/UDS/Named-Pipe /// connection and sends a `CancelRequest` — the same pattern used by the /// sync [`Client`](super::client::Client). Using a sync cancel path /// lets `Drop` issue cancels without needing a tokio runtime handle. - canceller: &'a dyn Cancellable, + canceller: &'conn dyn Cancellable, finished: bool, chunk_size: usize, schema: Option>, @@ -52,13 +52,13 @@ impl std::fmt::Debug for AsyncQueryStream<'_> { } } -impl<'a> AsyncQueryStream<'a> { +impl<'conn> AsyncQueryStream<'conn> { /// Constructs a new streaming result reader. The caller is responsible /// for having already issued the query bytes (see /// [`AsyncRawConnection::start_query_binary`]). pub(crate) fn new( - conn: MutexGuard<'a, AsyncRawConnection>, - canceller: &'a dyn Cancellable, + conn: MutexGuard<'conn, AsyncRawConnection>, + canceller: &'conn dyn Cancellable, chunk_size: usize, ) -> Self { Self { @@ -144,6 +144,48 @@ impl<'a> AsyncQueryStream<'a> { } Ok(Some(rows)) } + + /// Reads and discards the rest of the response through `ReadyForQuery` + /// without building any rows, leaving the connection ready for the next + /// request. A no-op if the stream already finished. + /// + /// Use this after taking the rows you need from a stream that may hold + /// more data than one chunk: [`next_chunk`](Self::next_chunk) returns as + /// soon as it has `chunk_size` rows, *before* reading `CommandComplete` + /// and `ReadyForQuery`, so simply dropping the stream at that point + /// leaves the wire mid-response. + /// + /// # Errors + /// + /// - Returns `Error` (I/O) / `Error` (closed) if the transport fails + /// while reading. + /// - Returns `Error` (server) if the server sends an `ErrorResponse` + /// before `ReadyForQuery`; the error is still drained through + /// `ReadyForQuery` first. + pub async fn drain_to_ready(&mut self) -> Result<()> { + if self.finished { + return Ok(()); + } + let Some(conn) = self.conn.as_mut() else { + return Ok(()); + }; + loop { + match conn.read_message().await? { + Message::ReadyForQuery(_) => { + self.finished = true; + self.conn = None; + return Ok(()); + } + Message::ErrorResponse(body) => { + self.finished = true; + let err = conn.consume_error(&body).await; + self.conn = None; + return Err(err); + } + _ => {} + } + } + } } impl Drop for AsyncQueryStream<'_> { diff --git a/hyperdb-api-core/src/client/cancel.rs b/hyperdb-api-core/src/client/cancel.rs index aad44928..f55e56f1 100644 --- a/hyperdb-api-core/src/client/cancel.rs +++ b/hyperdb-api-core/src/client/cancel.rs @@ -43,21 +43,19 @@ /// transport-level failures (e.g. the cancel connection can't be opened) /// are logged and swallowed. This keeps `cancel()` usable from `Drop` /// impls, which cannot propagate errors. -/// - `cancel()` is **idempotent and late-safe**. It is always possible -/// for the in-flight query to complete between the moment a caller -/// decides to cancel and the moment the cancel actually reaches the -/// server — the query's natural `ReadyForQuery` and the cancel -/// request race. Both PG wire and gRPC handle this correctly: the -/// PG wire `CancelRequest` travels on a *separate* connection and -/// only affects the query currently bound to the target connection's -/// process id, so a cancel that arrives after the query finished -/// targets nothing and is a harmless no-op; the gRPC `cancel_query` -/// RPC is similarly keyed on a server-assigned query id and -/// returns gracefully when the id corresponds to a completed query. -/// Connection-pool state is never affected by a late cancel, -/// because cancels never mutate the underlying connection's -/// protocol state — they only signal the server to abort work on -/// an already-separately-tracked query. +/// - `cancel()` is **late-safe, provided it is synchronous with the +/// server**. A cancel can arrive after the query it targeted finished (the +/// natural `ReadyForQuery` and the cancel race). Hyper does **not** discard +/// such a cancel: if it is processed while the next query is running, that +/// query fails with `57014 canceled` (reproduced by +/// `late_cancel_never_fails_the_next_query`). The PG wire implementation +/// therefore waits for the server to close the cancel connection (its +/// "request processed" signal, as libpq's `PQcancel` does) before +/// returning, which orders the cancel before anything the caller sends +/// next. Implementations must preserve that ordering. (Windows named +/// pipes do not wait today, so the race remains possible there.) The gRPC +/// `cancel_query` RPC is keyed on a server-assigned query id and returns +/// gracefully when the id corresponds to a completed query. /// /// # Writing an implementation /// diff --git a/hyperdb-api-core/src/client/client.rs b/hyperdb-api-core/src/client/client.rs index 02f7e6f0..aa136b09 100644 --- a/hyperdb-api-core/src/client/client.rs +++ b/hyperdb-api-core/src/client/client.rs @@ -638,6 +638,11 @@ impl Client { })?; stream.flush().map_err(Error::from_io)?; + // Wait for the server to close the cancel connection (its + // "request processed" signal) so the cancel cannot land on + // the caller's next query. See `tls::send_cancel_sync`. + stream.shutdown(std::net::Shutdown::Write).ok(); + tls::drain_sync(&mut stream); } #[cfg(windows)] ConnectionEndpoint::NamedPipe { host, name } => { @@ -864,11 +869,11 @@ impl Client { /// - Returns [`Error`] (server) or [`Error`] (I/O) if the initial /// `Parse`/`Bind`/`Execute` sequence for the streaming query /// fails on the server or on the wire. - pub fn query_streaming<'a>( - &'a self, + pub fn query_streaming<'conn>( + &'conn self, query: &str, chunk_size: usize, - ) -> Result> { + ) -> Result> { let mut conn = self.lock_connection()?; conn.start_query_binary(query)?; Ok(QueryStream { @@ -893,7 +898,35 @@ impl Client { /// server-side SQL errors, and wire-protocol I/O failures all /// surface as [`Error`]. pub fn exec(&self, query: &str) -> Result { - let mut conn = self.lock_connection()?; + let conn = self.lock_connection()?; + self.exec_locked(conn, query) + } + + /// Like [`exec`](Self::exec), but returns `Ok(None)` instead of blocking + /// when the connection is currently locked (a live result stream or COPY + /// writer, possibly on the calling thread itself). + /// + /// Intended for destructors, where blocking on a lock the same thread + /// already holds would hang forever. + /// + /// # Errors + /// + /// Same error modes as [`exec`](Self::exec) once the lock is obtained. + pub fn try_exec(&self, query: &str) -> Result> { + match self.connection.try_lock() { + Ok(conn) => self.exec_locked(conn, query).map(Some), + Err(std::sync::TryLockError::WouldBlock) => Ok(None), + Err(std::sync::TryLockError::Poisoned(_)) => { + Err(Error::connection("connection mutex poisoned")) + } + } + } + + fn exec_locked( + &self, + mut conn: MutexGuard<'_, RawConnection>, + query: &str, + ) -> Result { let messages = conn.simple_query(query)?; drop(conn); // Release lock before processing notices @@ -1183,7 +1216,10 @@ impl Client { ) -> Result> { let mut conn = self.lock_connection()?; conn.start_copy_in_with_format(table_name, columns, format)?; - Ok(CopyInWriter { connection: conn }) + Ok(CopyInWriter { + connection: conn, + completed: false, + }) } /// Starts a COPY IN operation from a raw SQL query string. @@ -1234,7 +1270,10 @@ impl Client { } let mut conn = self.lock_connection()?; conn.start_copy_in_raw(query)?; - Ok(CopyInWriter { connection: conn }) + Ok(CopyInWriter { + connection: conn, + completed: false, + }) } /// Returns true if the connection is alive (no error has occurred). @@ -1336,9 +1375,62 @@ impl Cancellable for Client { /// /// This struct holds the connection lock while sending data to ensure /// exclusive access during the COPY operation. +/// +/// # Drop behavior +/// +/// The server stays in the COPY IN sub-protocol until it sees `CopyDone` or +/// `CopyFail`, so simply releasing the connection lock would leave the next +/// request to fail with `unexpected protocol package` (and desynchronize the +/// extended protocol). Dropping a writer that was neither +/// [`finish`](Self::finish)ed nor [`cancel`](Self::cancel)led therefore sends +/// `CopyFail`, flushes, and drains to `ReadyForQuery` within a bounded +/// message budget. If that cannot be done the connection is marked +/// desynchronized so later calls fail fast instead of misparsing the wire. +/// +/// When the drop happens during panic unwinding +/// ([`std::thread::panicking`]) no network I/O is attempted: the connection is +/// marked desynchronized immediately, so a stalled peer cannot turn a panic +/// into a hang. Discard such a connection. #[derive(Debug)] -pub struct CopyInWriter<'a> { - connection: MutexGuard<'a, RawConnection>, +pub struct CopyInWriter<'conn> { + connection: MutexGuard<'conn, RawConnection>, + /// Set once `finish`/`cancel` has taken over wire-level cleanup. + completed: bool, +} + +impl Drop for CopyInWriter<'_> { + fn drop(&mut self) { + if self.completed { + return; + } + // Never do blocking network I/O while unwinding from a panic: the + // peer may stall and turn the panic into a hang. Leave the wire as + // it is and poison the connection instead. + if std::thread::panicking() { + warn!( + target: "hyperdb_api_core::client", + "CopyInWriter dropped during panic unwinding; skipping CopyFail and \ + marking the connection desynchronized", + ); + self.connection.mark_desynchronized(); + return; + } + warn!( + target: "hyperdb_api_core::client", + "CopyInWriter dropped without finish() or cancel(); sending CopyFail", + ); + if let Err(e) = self + .connection + .cancel_copy("CopyInWriter dropped without finish() or cancel()") + { + warn!( + target: "hyperdb_api_core::client", + error = %e, + "CopyFail on drop did not reach ReadyForQuery; connection marked desynchronized", + ); + self.connection.mark_desynchronized(); + } + } } impl CopyInWriter<'_> { @@ -1409,6 +1501,7 @@ impl CopyInWriter<'_> { /// failure (constraint violation, type coercion error, etc.) /// via an `ErrorResponse` after `CopyDone`. pub fn finish(mut self) -> Result { + self.completed = true; self.connection.finish_copy() } @@ -1419,9 +1512,12 @@ impl CopyInWriter<'_> { /// # Errors /// /// Returns [`Error`] (I/O) if writing the `CopyFail` frame or - /// flushing the transport fails, or [`Error`] (server) if the server - /// reports an unexpected status after the cancel. + /// flushing the transport fails, [`Error`] (closed) if the server drops + /// the connection before `ReadyForQuery`, or [`Error`] (protocol) if the + /// drain budget is exhausted. In every error case the connection is + /// marked desynchronized and must be discarded. pub fn cancel(mut self, reason: &str) -> Result<()> { + self.completed = true; self.connection.cancel_copy(reason) } } @@ -1449,15 +1545,15 @@ fn parse_affected_rows(tag: &str) -> Option { /// via a `MutexGuard`. Dropping the stream before fully iterating triggers /// a server-side cancel (see [`Drop`] below) so the connection is returned /// to the pool cleanly. -pub struct QueryStream<'a> { - conn: Option>>, +pub struct QueryStream<'conn> { + conn: Option>>, /// Best-effort cancel handle, used in [`Drop`] when the stream is /// abandoned before completion. For the current sync client this is /// the owning [`Client`] itself (which implements [`Cancellable`] via /// a PG wire `CancelRequest` on a fresh connection). When a gRPC /// equivalent lands it will plug in the same trait with a /// `cancel_query(query_id)` RPC implementation. - canceller: &'a dyn Cancellable, + canceller: &'conn dyn Cancellable, finished: bool, chunk_size: usize, schema: Option>, @@ -1492,20 +1588,23 @@ impl Drop for QueryStream<'_> { // cannot fail. self.canceller.cancel(); - // After cancel, the server stops producing new rows and emits - // `ErrorResponse(QueryCanceled) + ReadyForQuery` promptly. We still - // need to drain those trailing messages off the wire so the - // connection returns to the pool cleanly. The budget here is small - // because we only expect: (a) whatever rows the server had already - // flushed before seeing the cancel, (b) `ErrorResponse`, - // (c) `ReadyForQuery`. A well-behaved server reaches `ReadyForQuery` - // within a handful of messages. If that budget is somehow exceeded - // the bounded drain logs a warning and marks the connection - // desynchronized — downstream users will surface the desync as a - // transport-level failure and reconnect. + // After the cancel the server stops producing rows and emits + // `ErrorResponse(QueryCanceled) + ReadyForQuery`, but the cancel is + // not instantaneous: rows it flushed before observing it are still + // in the socket (hundreds of thousands for a fast, narrow result). + // `drain_abandoned_stream` therefore charges `DataRow`s against a + // separate, much larger ceiling and every other message (error, + // notices, `CommandComplete`) against the small budget below. If + // either is exceeded, e.g. because the cancel was lost, the drain + // logs a warning and marks the connection desynchronized rather than + // reading without bound inside a destructor that holds the + // connection lock. const POST_CANCEL_DRAIN_CAP: usize = 1024; if let Some(ref mut conn) = self.conn { - let _ok = conn.drain_until_ready_bounded(POST_CANCEL_DRAIN_CAP); + let _ok = conn.drain_abandoned_stream( + POST_CANCEL_DRAIN_CAP, + super::connection::ABANDONED_STREAM_MAX_DATA_ROWS, + ); } } } @@ -1601,6 +1700,48 @@ impl QueryStream<'_> { } Ok(Some(rows)) } + + /// Reads and discards the rest of the response through `ReadyForQuery` + /// without building any rows, leaving the connection ready for the next + /// request. A no-op if the stream already finished. + /// + /// Unlike dropping the stream (whose `Drop` cancels the query and + /// discards the backlog), this reads the *entire* remainder of the result + /// with no cancel and no limit, and reports a server error that appears + /// in the unread part. Prefer a plain drop when the rest of the result is + /// not wanted; use this when the remainder must be consumed. + /// + /// # Errors + /// + /// - Returns `Error` (I/O) / `Error` (closed) if the transport fails + /// while reading. + /// - Returns `Error` (server) if the server sends an `ErrorResponse` + /// before `ReadyForQuery`; the error is still drained through + /// `ReadyForQuery` first. + pub fn drain_to_ready(&mut self) -> Result<()> { + if self.finished { + return Ok(()); + } + let Some(conn) = self.conn.as_mut() else { + return Ok(()); + }; + loop { + match conn.read_message()? { + Message::ReadyForQuery(_) => { + self.finished = true; + self.conn = None; + return Ok(()); + } + Message::ErrorResponse(body) => { + self.finished = true; + let err = conn.consume_error(&body); + self.conn = None; + return Err(err); + } + _ => {} + } + } + } } #[cfg(test)] diff --git a/hyperdb-api-core/src/client/connection.rs b/hyperdb-api-core/src/client/connection.rs index a87c1aa7..1f0570b7 100644 --- a/hyperdb-api-core/src/client/connection.rs +++ b/hyperdb-api-core/src/client/connection.rs @@ -46,6 +46,17 @@ use super::auth::{self, AuthState}; use super::error::{Error, Result}; use super::statement::{ParamFormat, bind_format_codes}; +/// Hard ceiling on `DataRow`s an abandoned-stream drain will discard before +/// giving up and marking the connection desynchronized (64 full 65,536-row +/// chunks, about 4.2 million rows). +/// +/// A cancel is not instantaneous: a fast server keeps producing while the +/// cancel connection is opened (measured: ~700,000 rows of backlog for a +/// one-column result on loopback), so the ceiling has to sit well above that. +/// It still turns a lost or ignored cancel into bounded work instead of an +/// unbounded read inside a destructor holding the connection lock. +pub(crate) const ABANDONED_STREAM_MAX_DATA_ROWS: usize = 64 * 65_536; + /// Maximum number of messages [`RawConnection::consume_error`] (and its /// async sibling) will read while draining the tail of a failed request. /// @@ -667,12 +678,14 @@ where /// /// The canonical way to abort a running query is to open a *separate* /// connection and send `CancelRequest` with the original connection's - /// process id and secret. That is exactly what - /// [`QueryStream`](super::client::QueryStream)'s `Drop` impl does - /// (via the [`Cancellable`](super::cancel::Cancellable) trait) - /// before calling this bounded drain. Cancel-then-drain converges on - /// `ReadyForQuery` within a handful of messages because the server - /// stops producing new `DataRow`s once it observes the cancel. + /// process id and secret. [`QueryStream`](super::client::QueryStream)'s + /// `Drop` impl does that (via the + /// [`Cancellable`](super::cancel::Cancellable) trait) and then calls + /// the dedicated `drain_abandoned_stream` rather than this method, + /// because after a cancel the in-flight backlog is mostly `DataRow`s + /// that this method's flat message budget would wrongly count against + /// the stream. This method is for drains where *every* message is + /// expected to be short (error recovery, COPY teardown). /// /// # Poisoned connections /// @@ -726,6 +739,79 @@ where false } + /// Drains an abandoned result stream through its trailing `ReadyForQuery`. + /// + /// Unlike [`drain_until_ready_bounded`](Self::drain_until_ready_bounded), + /// `DataRow` messages are charged against a separate, much larger budget. + /// A cancel is fire-and-forget and races the server: rows it had already + /// produced are buffered in the socket (a result larger than one chunk + /// leaves tens of thousands of `DataRow`s in flight), and they must be + /// consumed. The server stops producing once it observes the cancel, so + /// that backlog is finite. + /// + /// Two budgets keep this safe to run inside a destructor while holding + /// the connection lock: + /// + /// - `max_other_messages` bounds every non-row message (the + /// `ErrorResponse`, notices, `CommandComplete`) and guards against a + /// server that never sends `ReadyForQuery`. + /// - `max_data_rows` is a hard ceiling on discarded `DataRow`s. It is + /// what stops the drain from reading an unbounded result when the + /// cancel was lost or not honoured (the cancel connection failed to + /// open, the server was at its connection limit, ...). + /// + /// Returns `true` on `ReadyForQuery`. On a read error or an exhausted + /// budget the connection is marked desynchronized and `false` is returned. + pub(crate) fn drain_abandoned_stream( + &mut self, + max_other_messages: usize, + max_data_rows: usize, + ) -> bool { + let mut other = 0usize; + let mut rows = 0usize; + loop { + match self.read_message() { + Ok(Message::ReadyForQuery(_)) => return true, + Ok(Message::DataRow(_)) => { + rows += 1; + if rows > max_data_rows { + warn!( + target: "hyperdb_api_core::client", + max_data_rows, + "drain_abandoned_stream: DataRow ceiling exceeded (cancel lost or \ + not honoured); connection marked desynchronized", + ); + self.desynchronized = true; + return false; + } + } + Ok(_) => { + other += 1; + if other > max_other_messages { + warn!( + target: "hyperdb_api_core::client", + max_other_messages, + "drain_abandoned_stream: no ReadyForQuery within the budget; \ + connection marked desynchronized and should not be reused", + ); + self.desynchronized = true; + return false; + } + } + Err(e) => { + warn!( + target: "hyperdb_api_core::client", + error = %e, + "drain_abandoned_stream: read error mid-drain; \ + connection marked desynchronized", + ); + self.desynchronized = true; + return false; + } + } + } + } + /// Convenience: parse a server [`Message::ErrorResponse`] body into an /// [`Error`] and drain the rest of the response through the trailing /// [`Message::ReadyForQuery`] so the connection is safe to reuse. @@ -958,7 +1044,10 @@ where pub fn send_copy_data_direct(&mut self, data: &[u8]) -> Result<()> { // First flush any pending buffered data if !self.write_buf.is_empty() { - self.stream.write_all(&self.write_buf)?; + // A failed write may leave a partial frame on the wire. + self.stream + .write_all(&self.write_buf) + .inspect_err(|_| self.desynchronized = true)?; self.write_buf.clear(); } @@ -968,8 +1057,17 @@ where .map_err(|_| Error::protocol("CopyData payload exceeds u32::MAX bytes"))?; let len_be = msg_len.to_be_bytes(); let header = [b'd', len_be[0], len_be[1], len_be[2], len_be[3]]; - self.stream.write_all(&header)?; - self.stream.write_all(data)?; + // A failure on either write may leave a partial `CopyData` frame on + // the wire. The server would then misparse whatever we send next as + // the remainder of that frame, so the connection is unusable. + if let Err(e) = self.stream.write_all(&header) { + self.desynchronized = true; + return Err(e.into()); + } + if let Err(e) = self.stream.write_all(data) { + self.desynchronized = true; + return Err(e.into()); + } Ok(()) } @@ -999,17 +1097,22 @@ where /// accumulated rows). /// - Returns [`Error`] (I/O) on wire-protocol read/write failure. pub fn finish_copy(&mut self) -> Result { + // A desynchronized wire (e.g. a partial CopyData frame) would make the + // read loop below block forever waiting for a reply that never comes. + self.ensure_healthy()?; // Ensure all data is sent - self.flush()?; + self.flush_or_desync()?; // Send CopyDone frontend::copy_done(&mut self.write_buf); - self.flush()?; + self.flush_or_desync()?; - // Wait for CommandComplete and ReadyForQuery + // Wait for CommandComplete and ReadyForQuery. A healthy server emits + // only a handful of messages here, so the read is bounded: a server + // that never sends ReadyForQuery must not hang the caller forever. let mut row_count = 0u64; - loop { - let msg = self.read_message()?; + for _ in 0..POST_ERROR_DRAIN_CAP { + let msg = self.read_message_or_desync()?; match msg { Message::CommandComplete(body) => { if let Ok(tag) = body.tag() { @@ -1032,28 +1135,39 @@ where } } } + self.desynchronized = true; + Err(Error::protocol( + "finish_copy: no ReadyForQuery within the drain budget; \ + connection marked desynchronized", + )) } /// Cancels a COPY IN operation. /// - /// This sends `CopyFail` and waits for the error response. + /// This sends `CopyFail` and waits for the error response. The wait is + /// bounded by [`POST_ERROR_DRAIN_CAP`] messages; if `ReadyForQuery` does + /// not arrive in that budget the connection is marked desynchronized. /// /// # Errors /// /// Returns [`Error`] (I/O) if flushing the buffer or writing the - /// `CopyFail` frame fails, or [`Error`] (closed) if the server - /// drops the connection before returning `ReadyForQuery`. + /// `CopyFail` frame fails, [`Error`] (closed) if the server drops the + /// connection before returning `ReadyForQuery`, or [`Error`] (protocol) + /// if the drain budget is exhausted. In every error case the connection + /// is marked desynchronized. pub fn cancel_copy(&mut self, reason: &str) -> Result<()> { + // See `finish_copy`: never wait on a desynchronized wire. + self.ensure_healthy()?; // Ensure buffer is clear - self.flush()?; + self.flush_or_desync()?; // Send CopyFail frontend::copy_fail(reason, &mut self.write_buf); - self.flush()?; + self.flush_or_desync()?; // Wait for ErrorResponse and ReadyForQuery - loop { - let msg = self.read_message()?; + for _ in 0..POST_ERROR_DRAIN_CAP { + let msg = self.read_message_or_desync()?; match msg { Message::ReadyForQuery(_) => { return Ok(()); @@ -1066,6 +1180,32 @@ where } } } + self.desynchronized = true; + Err(Error::protocol( + "cancel_copy: no ReadyForQuery within the drain budget; \ + connection marked desynchronized", + )) + } + + /// Marks the connection as desynchronized from the server. + /// + /// Used by RAII guards (for example `CopyInWriter`'s `Drop`) that cannot + /// bring the wire back to `ReadyForQuery`. + pub(crate) fn mark_desynchronized(&mut self) { + self.desynchronized = true; + } + + /// [`flush`](Self::flush), marking the connection desynchronized on failure + /// (a failed flush may leave a partial frame on the wire). + fn flush_or_desync(&mut self) -> Result<()> { + self.flush().inspect_err(|_| self.desynchronized = true) + } + + /// [`read_message`](Self::read_message), marking the connection + /// desynchronized on failure inside a COPY exchange. + fn read_message_or_desync(&mut self) -> Result { + self.read_message() + .inspect_err(|_| self.desynchronized = true) } /// Executes a COPY ... TO STDOUT query and returns all output data. diff --git a/hyperdb-api-core/src/client/prepared_stream.rs b/hyperdb-api-core/src/client/prepared_stream.rs index a024bdfb..85996ed9 100644 --- a/hyperdb-api-core/src/client/prepared_stream.rs +++ b/hyperdb-api-core/src/client/prepared_stream.rs @@ -38,11 +38,11 @@ const POST_CANCEL_DRAIN_CAP: usize = 1024; /// via a `MutexGuard`. Dropping the stream before fully iterating triggers /// a server-side cancel (see [`Drop`] below) so the connection is returned /// to a usable state. -pub struct PreparedQueryStream<'a> { - conn: Option>>, +pub struct PreparedQueryStream<'conn> { + conn: Option>>, /// Best-effort cancel handle — see /// [`QueryStream`](super::client::QueryStream) for the full rationale. - canceller: &'a dyn Cancellable, + canceller: &'conn dyn Cancellable, finished: bool, chunk_size: usize, /// Column metadata carried through from the statement's Describe pass. @@ -61,13 +61,13 @@ impl std::fmt::Debug for PreparedQueryStream<'_> { } } -impl<'a> PreparedQueryStream<'a> { +impl<'conn> PreparedQueryStream<'conn> { /// Constructs a new streaming reader. The caller must have already /// issued the Bind/Execute/Sync via /// [`RawConnection::start_execute_prepared`]. pub(crate) fn new( - conn: MutexGuard<'a, RawConnection>, - canceller: &'a dyn Cancellable, + conn: MutexGuard<'conn, RawConnection>, + canceller: &'conn dyn Cancellable, chunk_size: usize, columns: Arc>, ) -> Self { @@ -144,6 +144,48 @@ impl<'a> PreparedQueryStream<'a> { } Ok(Some(rows)) } + + /// Reads and discards the rest of the response through `ReadyForQuery` + /// without building any rows, leaving the connection ready for the next + /// request. A no-op if the stream already finished. + /// + /// Unlike dropping the stream (whose `Drop` cancels the query and + /// discards the backlog), this reads the *entire* remainder of the result + /// with no cancel and no limit, and reports a server error that appears + /// in the unread part. Prefer a plain drop when the rest of the result is + /// not wanted; use this when the remainder must be consumed. + /// + /// # Errors + /// + /// - Returns `Error` (I/O) / `Error` (closed) if the transport fails + /// while reading. + /// - Returns `Error` (server) if the server sends an `ErrorResponse` + /// before `ReadyForQuery`; the error is still drained through + /// `ReadyForQuery` first. + pub fn drain_to_ready(&mut self) -> Result<()> { + if self.finished { + return Ok(()); + } + let Some(conn) = self.conn.as_mut() else { + return Ok(()); + }; + loop { + match conn.read_message()? { + Message::ReadyForQuery(_) => { + self.finished = true; + self.conn = None; + return Ok(()); + } + Message::ErrorResponse(body) => { + self.finished = true; + let err = conn.consume_error(&body); + self.conn = None; + return Err(err); + } + _ => {} + } + } + } } impl Drop for PreparedQueryStream<'_> { @@ -158,7 +200,10 @@ impl Drop for PreparedQueryStream<'_> { self.canceller.cancel(); if let Some(ref mut conn) = self.conn { - let _ok = conn.drain_until_ready_bounded(POST_CANCEL_DRAIN_CAP); + let _ok = conn.drain_abandoned_stream( + POST_CANCEL_DRAIN_CAP, + super::connection::ABANDONED_STREAM_MAX_DATA_ROWS, + ); } // Belt-and-braces in case the drain exceeds its budget: the drain diff --git a/hyperdb-api-core/src/client/tls.rs b/hyperdb-api-core/src/client/tls.rs index 6b4d847f..672ddc23 100644 --- a/hyperdb-api-core/src/client/tls.rs +++ b/hyperdb-api-core/src/client/tls.rs @@ -755,7 +755,14 @@ pub(crate) fn send_cancel_sync( let Some(connector) = tls else { tcp.write_all(&buf).map_err(cancel_send_error)?; - return tcp.flush().map_err(Error::from_io); + tcp.flush().map_err(Error::from_io)?; + // Wait for the server to close the cancel connection, which is how + // it signals that the request has been processed (libpq's + // `PQcancel` does the same). Without this the cancel races the + // caller's next query on the main connection and can fail it. + tcp.shutdown(Shutdown::Write).ok(); + drain_sync(&mut tcp); + return Ok(()); }; let mut stream = match negotiate_sync(tcp, connector, Fallback::Deny, Some(CANCEL_TLS_TIMEOUT))? @@ -782,15 +789,35 @@ pub(crate) fn send_cancel_sync( Ok(()) } -fn drain_sync(sock: &mut TcpStream) { +/// A blocking socket the cancel-request path can wait on for the server's +/// close. +pub(crate) trait SyncCancelSocket: io::Read { + /// Sets the read timeout used while waiting for the close. + fn set_cancel_read_timeout(&self, timeout: Duration) -> io::Result<()>; +} + +impl SyncCancelSocket for TcpStream { + fn set_cancel_read_timeout(&self, timeout: Duration) -> io::Result<()> { + self.set_read_timeout(Some(timeout)) + } +} + +#[cfg(unix)] +impl SyncCancelSocket for std::os::unix::net::UnixStream { + fn set_cancel_read_timeout(&self, timeout: Duration) -> io::Result<()> { + self.set_read_timeout(Some(timeout)) + } +} + +pub(crate) fn drain_sync(sock: &mut S) { let deadline = Instant::now() + CANCEL_DRAIN_TIMEOUT; let mut sink = [0u8; 4096]; loop { let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() || sock.set_read_timeout(Some(remaining)).is_err() { + if remaining.is_zero() || sock.set_cancel_read_timeout(remaining).is_err() { return; } - match sock.read(&mut sink) { + match io::Read::read(sock, &mut sink) { Ok(0) | Err(_) => return, Ok(_) => {} } @@ -818,7 +845,17 @@ pub(crate) async fn send_cancel_async( let Some(connector) = tls else { tcp.write_all(&buf).await.map_err(cancel_send_error)?; - return tcp.flush().await.map_err(Error::from_io); + tcp.flush().await.map_err(Error::from_io)?; + // Wait for the server's close (its "request processed" signal) so + // the cancel cannot land on the caller's next query. + let close = async { + tcp.shutdown().await?; + let mut sink = [0u8; 64]; + while tcp.read(&mut sink).await? > 0 {} + Ok::<(), io::Error>(()) + }; + let _ = tokio::time::timeout(CANCEL_DRAIN_TIMEOUT, close).await; + return Ok(()); }; let mut stream = diff --git a/hyperdb-api-core/tests/copy_tests.rs b/hyperdb-api-core/tests/copy_tests.rs index 0b7138e8..f9bcc67b 100644 --- a/hyperdb-api-core/tests/copy_tests.rs +++ b/hyperdb-api-core/tests/copy_tests.rs @@ -413,3 +413,142 @@ fn test_copy_in_text_with_special_chars() { client.close().expect("Failed to close"); } + +/// Dropping a `CopyInWriter` without `finish()`/`cancel()` must not leave the +/// server in COPY IN: it sends `CopyFail` and drains to `ReadyForQuery`. +#[test] +fn test_copy_in_dropped_writer_leaves_connection_usable() { + let server = TestServer::new().expect("Failed to create test server"); + let client = server.connect().expect("Failed to connect"); + + client + .exec("CREATE TABLE test_copy_drop (id INT NOT NULL, name TEXT NOT NULL)") + .expect("Failed to create table"); + + let mut buf = BytesMut::new(); + copy::write_header(&mut buf); + for i in 0..10 { + copy::write_i32_not_null(&mut buf, i); + copy::write_varbinary_not_null(&mut buf, b"x"); + } + + { + let mut writer = client + .copy_in("test_copy_drop", &["id", "name"]) + .expect("Failed to start COPY"); + writer.send(&buf).expect("Failed to send data"); + writer.flush().expect("Failed to flush"); + // Push a large flushed volume (~6 MB of valid rows) through too, so the + // teardown has real in-flight COPY data to cope with. + let mut big = BytesMut::new(); + for i in 0..400_000 { + copy::write_i32_not_null(&mut big, i); + copy::write_varbinary_not_null(&mut big, b"x"); + } + writer.send(&big).expect("Failed to send large volume"); + writer.flush().expect("Failed to flush large volume"); + // writer dropped here without finish()/cancel() + } + + // Simple-protocol statement must work (previously: "unexpected protocol package: 'Q'"). + let rows = client + .query("SELECT COUNT(*) FROM test_copy_drop") + .expect("query after dropped COPY writer must succeed"); + assert_eq!( + rows[0].get_i64(0), + Some(0), + "aborted COPY must insert nothing" + ); + + // A second request must not be desynchronized by a stray ReadyForQuery. + let rows = client + .query("SELECT 41 + 1") + .expect("follow-up query must succeed"); + assert_eq!(rows[0].get_i32(0), Some(42)); + client + .exec("INSERT INTO test_copy_drop VALUES (1, 'a')") + .expect("follow-up command must succeed"); + + client.close().expect("Failed to close"); +} + +// ============================================================================= +// Server rejects data mid-COPY, then the writer is dropped / cancelled +// ============================================================================= + +/// Builds a HyperBinary payload for `(id INT NOT NULL, name TEXT NOT NULL)` +/// whose row `bad_row` carries invalid UTF-8, followed by +/// `total_rows - bad_row` valid rows. The server rejects the COPY once it +/// parses the bad row, while the client is still writing. +fn copy_payload_with_bad_row(total_rows: i32, bad_row: i32) -> BytesMut { + let mut buf = BytesMut::new(); + copy::write_header(&mut buf); + for i in 0..total_rows { + if i == bad_row { + // Invalid UTF-8 in a TEXT column: the server rejects it while + // parsing, long before the end of the stream. + copy::write_i32_not_null(&mut buf, i); + copy::write_varbinary_not_null(&mut buf, &[0xFF, 0xFE, 0xFD]); + } else { + copy::write_i32_not_null(&mut buf, i); + copy::write_varbinary_not_null(&mut buf, b"x"); + } + } + buf +} + +/// Runs a follow-up that detects a stray `ReadyForQuery` / `ErrorResponse`: +/// several sequential requests must each return their own answer. +fn assert_connection_in_sync(client: &hyperdb_api_core::client::Client, table: &str) { + for expected in [42, 43, 44] { + let rows = client + .query(&format!("SELECT {expected}")) + .expect("follow-up query must succeed"); + assert_eq!(rows[0].get_i32(0), Some(expected)); + } + let rows = client + .query(&format!("SELECT COUNT(*) FROM {table}")) + .expect("count query must succeed"); + assert_eq!( + rows[0].get_i64(0), + Some(0), + "rejected COPY must insert nothing" + ); +} + +fn copy_rejected_mid_stream(finish_with_cancel: bool) { + let server = TestServer::new().expect("Failed to create test server"); + let client = server.connect().expect("Failed to connect"); + client + .exec("CREATE TABLE test_copy_reject (id INT NOT NULL, name TEXT NOT NULL)") + .expect("Failed to create table"); + + // 64,001 rows with the corruption at row 10. + let payload = copy_payload_with_bad_row(64_001, 10); + + let mut writer = client + .copy_in("test_copy_reject", &["id", "name"]) + .expect("Failed to start COPY"); + // The server may reject mid-write; a send error here is acceptable, the + // point is the state afterwards. + let _ = writer.send(&payload); + let _ = writer.flush(); + if finish_with_cancel { + let _ = writer.cancel("test cancel"); + } else { + drop(writer); + } + + assert_connection_in_sync(&client, "test_copy_reject"); + client.close().expect("Failed to close"); +} + +#[test] +fn test_copy_in_rejected_mid_stream_then_drop_leaves_connection_in_sync() { + copy_rejected_mid_stream(false); +} + +#[test] +fn test_copy_in_rejected_mid_stream_then_cancel_leaves_connection_in_sync() { + copy_rejected_mid_stream(true); +} diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 7a76e601..8fac310a 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **Breaking:** `Inserter::execute`, `Inserter::cancel`, `AsyncInserter::execute`, `AsyncInserter::cancel` and the mapped-inserter equivalents now take `self` by value. An inserter is single-use; reusing one after `execute`/`cancel` is a compile error instead of a silent re-send of already-committed rows or a wedged COPY session. Note that `execute` returning an incomplete-row error now discards all rows, including chunks already auto-flushed to the server (nothing is committed). - **BREAKING:** the `arrow` dependency moved from **59** to **60**. Arrow types appear in this crate's public API, so consumers must move to `arrow` 60 in lockstep. No source change was needed on our side. `parquet` (used by @@ -123,6 +124,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- Cancelling or dropping an `Inserter` after an auto-flush no longer leaves the connection mid-COPY: `cancel` now sends `CopyFail` and returns the connection to `ReadyForQuery`. `import_text` also cancels the COPY when reading the source fails. +- `MappedInserter` no longer deadlocks on `cancel` after an auto-flush, and its staging table is dropped on every path (execute, cancel, error, drop). Staging tables are named `_hyper_staging_{pid}_{n}`, so two mapped inserters on one connection no longer collide. +- On the sync path, `fetch_one`, `fetch_optional`, `fetch_scalar`, `fetch_optional_scalar`, `query_count`, `first_row` and the `QueryAs`/`QueryScalar` single-row fetchers used to leave the unread remainder of a multi-chunk (more than 65,536 rows) result on the wire, so the next request failed with "connection is desynchronized". They now read only the first chunk and drop the stream, whose `Drop` cancels the query and discards the backlog. + Add `LIMIT 1` to avoid producing the extra rows. A server error that would only occur in the unread rows is not reported. +- Async prepared `fetch_one` / `fetch_optional` now stream and take the first row instead of materializing the whole result (a memory improvement), and read the remainder of the response so the connection stays in sync. +- A cancel request no longer races the caller's next query: the cancel connection is now held open until the server closes it, so a stray `57014 canceled` can no longer fail the query that follows an abandoned stream. +- `StagingTable` (the `MappedInserter` staging table) no longer runs blocking SQL in its destructor: if the connection is busy (for example the same thread still holds a live result) or the thread is panicking, the session-scoped temporary table is left to be reclaimed when the connection closes instead of deadlocking. - **Reserved-word table and column names** (`order`, `user`, `table`, ...) no longer break generated SQL. `TableDefinition::qualified_name`, `table_name`, `schema_name`, `database_name`, `to_drop_sql` and the `MappedInserter` diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index ab03868e..f966e7a1 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -311,6 +311,9 @@ impl AsyncConnection { /// Fetches a single row, erroring if the query returns zero rows. /// + /// Reads and discards the rest of the result to keep the connection + /// in sync, so add `LIMIT 1` to queries that match many rows. + /// /// # Errors /// /// - Returns the error from [`execute_query`](Self::execute_query) if @@ -326,6 +329,9 @@ impl AsyncConnection { /// Fetches a single row, returning `None` if the query is empty. /// + /// Reads and discards the rest of the result to keep the connection + /// in sync, so add `LIMIT 1` to queries that match many rows. + /// /// # Errors /// /// Returns the error from [`execute_query`](Self::execute_query) if the diff --git a/hyperdb-api/src/async_inserter.rs b/hyperdb-api/src/async_inserter.rs index 497013b6..dd641be6 100644 --- a/hyperdb-api/src/async_inserter.rs +++ b/hyperdb-api/src/async_inserter.rs @@ -58,9 +58,9 @@ use crate::table_definition::TableDefinition; /// # Single-use /// /// Like the sync `Inserter`, `AsyncInserter` is intended for one COPY session -/// per instance. After [`execute`](Self::execute) returns, the row counter is -/// zeroed so a stray second call returns `Ok(0)` rather than corrupting the -/// stream. To insert a second batch, construct a new `AsyncInserter`. +/// per instance. [`execute`](Self::execute) and [`cancel`](Self::cancel) take +/// `self` by value, so reusing a finished inserter is a compile error. To +/// insert a second batch, construct a new `AsyncInserter`. #[derive(Debug)] pub struct AsyncInserter<'conn> { connection: &'conn AsyncConnection, @@ -254,16 +254,21 @@ impl<'conn> AsyncInserter<'conn> { /// Sends any remaining buffered data and finishes the COPY operation. /// Returns the number of rows inserted. /// - /// The inserter is single-use: calling `execute` a second time returns - /// `Ok(0)`. Construct a new `AsyncInserter` to insert another batch. + /// Takes `self` by value: the inserter is single-use, so reuse after + /// `execute` is a compile error rather than a silent resend of the + /// already-committed rows. Construct a new `AsyncInserter` to insert + /// another batch. /// /// # Errors /// /// - Returns [`Error::InvalidTableDefinition`] if there's an incomplete row (partial column). + /// Because `execute` consumes the inserter, this discards **all** rows + /// added so far, including chunks already auto-flushed to the server + /// (nothing is committed). /// - Returns [`Error::Server`] if the server rejects the COPY or its data. /// - Returns [`Error::Connection`] on a transport I/O failure, or /// [`Error::Closed`] if the server closed the connection. - pub async fn execute(&mut self) -> Result { + pub async fn execute(mut self) -> Result { if self.chunk.column_index() != 0 { return Err(Error::invalid_table_definition( "Incomplete row at execute time", @@ -297,18 +302,17 @@ impl<'conn> AsyncInserter<'conn> { "async-inserter-end" ); - // Reset so a stray second execute() returns Ok(0). - self.row_count = 0; Ok(rows) } /// Cancels the insert and discards all buffered rows. /// - /// The Drop impl on `AsyncCopyInWriter` queues a `CopyFail` on the + /// Takes `self` by value so a cancelled inserter cannot be reused. The + /// Drop impl on `AsyncCopyInWriter` queues a `CopyFail` on the /// connection so the server tears the COPY down cleanly on the next /// connection use. - pub fn cancel(&mut self) { - self.writer = None; - self.row_count = 0; + pub fn cancel(self) { + // Dropping `self` drops the writer, which tears the COPY down. + drop(self); } } diff --git a/hyperdb-api/src/async_prepared.rs b/hyperdb-api/src/async_prepared.rs index 947d27de..442a9a7b 100644 --- a/hyperdb-api/src/async_prepared.rs +++ b/hyperdb-api/src/async_prepared.rs @@ -264,6 +264,19 @@ impl AsyncPreparedStatementOwned { /// `Execute`, or raises a runtime error while streaming. /// - Returns [`Error::Connection`] on transport-level I/O failures. pub async fn fetch_all(&self, params: &[&dyn ToSqlParam]) -> Result> { + self.with_rowset(params, async |rowset| rowset.collect_rows().await) + .await + } + + /// Starts the streaming execution and hands the resulting rowset to `f`. + /// + /// The rowset borrows the connection guard obtained here, so it cannot be + /// returned; the consumer runs inside the guard's scope instead. + async fn with_rowset( + &self, + params: &[&dyn ToSqlParam], + f: impl AsyncFnOnce(AsyncRowset<'_>) -> Result, + ) -> Result { let (encoded, formats) = encode_params(params); let client = async_tcp_client_arc(&self.connection)?; let stream = client @@ -274,8 +287,7 @@ impl AsyncPreparedStatementOwned { crate::result::DEFAULT_BINARY_CHUNK_SIZE, ) .await?; - let rowset = AsyncRowset::from_prepared(stream); - rowset.collect_rows().await + f(AsyncRowset::from_prepared(stream)).await } /// Executes the statement as a command; returns the affected-row count. @@ -294,29 +306,42 @@ impl AsyncPreparedStatementOwned { .await?) } + /// Streams the result and returns only its first row, reading the rest + /// of the response through `ReadyForQuery` (discarding it) so the + /// connection is left ready for the next request. Does not materialize + /// the whole result set. + async fn first_row(&self, params: &[&dyn ToSqlParam]) -> Result> { + self.with_rowset(params, async |rowset| rowset.first_row().await) + .await + } + /// Fetches exactly one row; errors on empty. /// + /// Reads (and discards) the rest of the result to keep the connection in + /// sync, so add `LIMIT 1` for statements that can return huge results. + /// /// # Errors /// - /// - Returns the error from [`fetch_all`](Self::fetch_all). + /// - Returns [`Error::FeatureNotSupported`] on gRPC transport. + /// - Returns [`Error::Server`] if the server rejects `Bind` or `Execute`. /// - Returns [`Error::Conversion`] with message `"Query returned no rows"` /// if the result is empty. pub async fn fetch_one(&self, params: &[&dyn ToSqlParam]) -> Result { - self.fetch_all(params) + self.first_row(params) .await? - .into_iter() - .next() .ok_or_else(|| crate::error::Error::conversion("Query returned no rows")) } /// Fetches at most one row; `None` on empty. /// + /// See [`fetch_one`](Self::fetch_one) for the full-read note. + /// /// # Errors /// - /// Returns the error from [`fetch_all`](Self::fetch_all); an empty - /// result yields `Ok(None)`. + /// As [`fetch_one`](Self::fetch_one), except an empty result yields + /// `Ok(None)`. pub async fn fetch_optional(&self, params: &[&dyn ToSqlParam]) -> Result> { - Ok(self.fetch_all(params).await?.into_iter().next()) + self.first_row(params).await } /// Fetches the first column of the first row as `T`. diff --git a/hyperdb-api/src/async_result.rs b/hyperdb-api/src/async_result.rs index 7c3a09d8..5401abad 100644 --- a/hyperdb-api/src/async_result.rs +++ b/hyperdb-api/src/async_result.rs @@ -172,6 +172,19 @@ impl<'conn> AsyncRowset<'conn> { self.schema_cache.clone() } + /// Reads and discards everything left in the result stream, through + /// `ReadyForQuery` on TCP, so the connection is ready for the next + /// request. See `Rowset::drain_remaining` for why this is needed after + /// a partial read. No-op for gRPC results. + pub(crate) async fn drain_remaining(&mut self) -> Result<()> { + match &mut self.inner { + AsyncRowsetInner::Tcp(stream) => stream.drain_to_ready().await?, + AsyncRowsetInner::Prepared(stream) => stream.drain_to_ready().await?, + AsyncRowsetInner::Arrow(_) => {} + } + Ok(()) + } + /// Returns the next chunk of rows, or `None` when the stream is /// exhausted. /// @@ -262,16 +275,26 @@ impl<'conn> AsyncRowset<'conn> { /// Returns the first row of the result set, or `None` if the /// result is empty. Consumes the rowset. /// + /// # Reads the whole result + /// + /// After taking the first row this reads (and discards) the rest of the + /// response through `ReadyForQuery`, so the connection is always left + /// ready for the next request. On a very large result that means the + /// whole stream is transferred; add `LIMIT 1` to the query when you only + /// want one row. + /// /// # Errors /// - /// Returns the error from [`next_chunk`](Self::next_chunk). An empty + /// Returns the error from [`next_chunk`](Self::next_chunk), or a server + /// or transport error hit while discarding the remainder. An empty /// result yields `Ok(None)`. pub async fn first_row(mut self) -> Result> { - if let Some(chunk) = self.next_chunk().await? { - Ok(chunk.into_iter().next()) - } else { - Ok(None) - } + let first = self + .next_chunk() + .await? + .and_then(|chunk| chunk.into_iter().next()); + self.drain_remaining().await?; + Ok(first) } /// Returns the first row, or an error if the result set is empty. diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index d8a11bd0..e3d76fb9 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -428,6 +428,13 @@ impl Connection { result } + /// Like [`execute_command`](Self::execute_command), but returns + /// `Ok(None)` instead of blocking when the connection is currently held + /// by a live result stream or COPY writer. For destructors only. + pub(crate) fn try_execute_command(&self, command: &str) -> Result> { + self.transport.try_execute_command(command) + } + /// Executes a SQL query and returns a streaming result set. /// /// Results are streamed in chunks (up to 65,536 rows over TCP), keeping memory @@ -617,6 +624,12 @@ impl Connection { /// /// Returns an error if the query returns no rows. /// + /// Only the first chunk of the result is read. If more rows remain, the + /// query is cancelled and the in-flight backlog discarded so the + /// connection stays in sync (a server error that would only appear in + /// the unread rows is not reported). Add `LIMIT 1` to queries that match + /// many rows to avoid producing them at all. + /// /// # Example /// /// ```no_run @@ -650,6 +663,12 @@ impl Connection { /// /// Returns `None` if the query returns no rows. /// + /// Only the first chunk of the result is read. If more rows remain, the + /// query is cancelled and the in-flight backlog discarded so the + /// connection stays in sync (a server error that would only appear in + /// the unread rows is not reported). Add `LIMIT 1` to queries that match + /// many rows to avoid producing them at all. + /// /// # Example /// /// ```no_run @@ -1055,7 +1074,9 @@ impl Connection { /// Fetches an optional scalar value from a query. /// - /// Returns `None` if the query returns no rows or NULL. + /// Returns `None` if the first cell is SQL `NULL`. A query that returns + /// zero rows is an error (see `# Errors`); use + /// [`fetch_optional`](Self::fetch_optional) to distinguish "no row". /// /// # Example /// diff --git a/hyperdb-api/src/copy.rs b/hyperdb-api/src/copy.rs index 2fc91a7f..be2d63d8 100644 --- a/hyperdb-api/src/copy.rs +++ b/hyperdb-api/src/copy.rs @@ -483,9 +483,23 @@ impl Connection { let chunk_size = options.chunk_size.unwrap_or(DEFAULT_IMPORT_CHUNK_SIZE); let mut buf = vec![0u8; chunk_size]; loop { - let n = reader - .read(&mut buf) - .map_err(|e| Error::connection_with_io("Failed to read import data", e))?; + let n = match reader.read(&mut buf) { + Ok(n) => n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + // Abort the COPY explicitly so the connection returns to + // ReadyForQuery and the caller sees the read error, not a + // later "unexpected protocol package". + if let Err(cancel_err) = writer.cancel("failed to read import data") { + tracing::warn!( + target: "hyperdb_api", + error = %cancel_err, + "import_text: aborting the COPY after a read error failed", + ); + } + return Err(Error::connection_with_io("Failed to read import data", e)); + } + }; if n == 0 { break; } diff --git a/hyperdb-api/src/inserter.rs b/hyperdb-api/src/inserter.rs index f6e75de1..57ee4073 100644 --- a/hyperdb-api/src/inserter.rs +++ b/hyperdb-api/src/inserter.rs @@ -28,7 +28,7 @@ use hyperdb_api_core::types::bytes::BytesMut; use hyperdb_api_core::types::{ Date, Geography, Interval, Numeric, OffsetTimestamp, Time, Timestamp, }; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use crate::catalog::Catalog; use crate::connection::Connection; @@ -295,7 +295,7 @@ impl<'conn> Inserter<'conn> { /// Returns the number of complete rows added so far, including rows already flushed to the server. /// - /// Reset to 0 by [`execute`](Self::execute) and [`cancel`](Self::cancel). + /// The inserter is consumed by [`execute`](Self::execute) and [`cancel`](Self::cancel). #[must_use] pub fn row_count(&self) -> u64 { self.row_count @@ -662,18 +662,32 @@ impl<'conn> Inserter<'conn> { /// This sends any remaining buffered data and finishes the COPY operation. /// Returns the number of rows inserted. /// - /// The inserter is single-use: calling `execute` a second time returns - /// `Ok(0)` because the internal row counter has been reset and no further - /// data has been added. To insert additional batches, create a new + /// Takes `self` by value: an inserter is single-use, so reusing it after + /// `execute` is a compile error rather than a silent resend of the + /// already-committed rows. To insert additional batches, create a new /// [`Inserter`]. /// + /// ```compile_fail,E0382 + /// # use hyperdb_api::{Connection, Inserter, Result, TableDefinition}; + /// # fn reuse(conn: &Connection, def: &TableDefinition) -> Result<()> { + /// let mut inserter = Inserter::new(conn, def)?; + /// inserter.execute()?; + /// inserter.execute()?; // error[E0382]: use of moved value: `inserter` + /// # Ok(()) + /// # } + /// ``` + /// /// # Errors /// /// - Returns [`Error::InvalidTableDefinition`] if a row is incomplete. + /// Because `execute` consumes the inserter, this discards **all** rows + /// added so far, including chunks already auto-flushed to the server + /// (the open COPY is aborted when the inserter is dropped, so nothing + /// is committed). /// - Returns [`Error::FeatureNotSupported`] on a gRPC connection. /// - Returns [`Error::Server`] if the server rejects the COPY or its data. /// - Returns [`Error::Connection`] or [`Error::Closed`] on transport failure. - pub fn execute(&mut self) -> Result { + pub fn execute(mut self) -> Result { if self.chunk.column_index() != 0 { return Err(Error::invalid_table_definition( "Incomplete row at execute time", @@ -735,19 +749,37 @@ impl<'conn> Inserter<'conn> { "inserter-end" ); - // Reset row counter so a stray second execute() call returns Ok(0) - // instead of attempting another COPY trailer on a finished writer. - self.row_count = 0; - Ok(rows) } /// Cancels the insert and discards all buffered rows. - pub fn cancel(&mut self) { - // Drop the in-progress writer (if any). The Drop impl on CopyInWriter - // sends a CopyFail to the server. - self.writer = None; - self.row_count = 0; + /// + /// If a COPY session is open it is aborted with `CopyFail` so the + /// connection returns to `ReadyForQuery` and stays usable; nothing sent + /// so far is committed. Takes `self` by value so the cancelled inserter + /// cannot be reused. + /// + /// A failure while aborting the COPY is logged at WARN level (the + /// connection is then marked desynchronized by the client layer) rather + /// than returned, so that cancelling never masks the error that + /// prompted it. + pub fn cancel(mut self) { + self.cancel_in_place(); + } + + /// Aborts any open COPY without consuming + /// `self` (used by [`MappedInserter`], which owns an `Inserter` field). + pub(crate) fn cancel_in_place(&mut self) { + if let Some(writer) = self.writer.take() + && let Err(e) = writer.cancel("Inserter cancelled") + { + warn!( + target: "hyperdb_api", + error = %e, + table = %self.table_def.qualified_name(), + "inserter-cancel: aborting the COPY session failed", + ); + } } } @@ -1135,14 +1167,94 @@ impl IntoValue for &Geography { /// [`Inserter::with_column_mappings`]. #[derive(Debug)] pub struct MappedInserter<'conn> { + // Field order matters: fields drop in declaration order, and the + // `inner` inserter's COPY writer must be gone (CopyFail sent, connection + // back at ReadyForQuery) before `staging` runs `DROP TABLE` on the same + // connection. Never run a statement while a `CopyInWriter` is alive. /// The underlying inserter for the staging table. inner: Inserter<'conn>, /// The target table name. target_table: crate::TableName, /// The column mappings. mappings: Vec, - /// The staging table name. - staging_table: String, + /// Owns the staging table and drops it when no longer needed. + staging: StagingTable<'conn>, +} + +/// Process-wide counter that keeps staging-table names unique across +/// [`MappedInserter`]s sharing a process (and a connection). +static STAGING_TABLE_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// RAII owner of a `MappedInserter` staging table. +/// +/// Drops the (temporary) table on [`Drop`] if it was not already dropped, +/// logging a WARN on failure instead of panicking or masking the original +/// error. +#[derive(Debug)] +struct StagingTable<'conn> { + connection: &'conn Connection, + name: String, + dropped: bool, +} + +impl StagingTable<'_> { + fn drop_sql(&self) -> String { + use hyperdb_api_core::protocol::escape::SqlIdentifier; + format!("DROP TABLE IF EXISTS {}", SqlIdentifier(&self.name)) + } + + /// Drops the staging table. Idempotent. + fn drop_now(&mut self) -> Result<()> { + if self.dropped { + return Ok(()); + } + // Set first: a failed DROP is not retried from `Drop`. + self.dropped = true; + self.connection + .execute_command(&self.drop_sql()) + .map(|_| ()) + } +} + +impl Drop for StagingTable<'_> { + /// Best-effort cleanup that never blocks. + /// + /// A destructor must not wait on the connection lock: if this thread + /// still holds a live result stream or COPY writer, that lock is + /// non-reentrant and waiting would hang forever. So the drop is attempted + /// with a try-lock; when the connection is busy, or the thread is + /// panicking, the table is left in place. It is session-scoped + /// (`CREATE TEMPORARY TABLE`), so the server reclaims it when the + /// connection closes. The explicit `execute`/`cancel` paths call + /// [`drop_now`](Self::drop_now) and do block. + fn drop(&mut self) { + if self.dropped { + return; + } + self.dropped = true; + if std::thread::panicking() { + warn!( + target: "hyperdb_api", + table = %self.name, + "MappedInserter dropped while panicking; leaving session-scoped staging table", + ); + return; + } + match self.connection.try_execute_command(&self.drop_sql()) { + Ok(Some(_)) => {} + Ok(None) => warn!( + target: "hyperdb_api", + table = %self.name, + "connection busy while dropping MappedInserter; leaving session-scoped staging table", + ), + Err(e) => warn!( + target: "hyperdb_api", + error = %e, + table = %self.name, + "failed to drop MappedInserter staging table", + ), + } + } } impl<'conn> MappedInserter<'conn> { @@ -1159,17 +1271,32 @@ impl<'conn> MappedInserter<'conn> { { let target_table = target_table.try_into()?; - // Create a unique staging table name - let staging_table = format!("_hyper_staging_{}", std::process::id()); + // Unique staging table name. The table is `TEMPORARY`, hence scoped to + // this session, so other processes never see it; what matters is the + // process-wide counter, which keeps two inserters on the same + // connection apart. The pid is only a debugging aid in the name. + let staging_table = format!( + "_hyper_staging_{}_{}", + std::process::id(), + STAGING_TABLE_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ); // Create the staging table definition (temporary) let mut staging_def = inserter_def.clone(); staging_def.name.clone_from(&staging_table); + staging_def.set_persistence(crate::Persistence::Temporary); - // Create the staging table + // Create the staging table (`CREATE TEMPORARY TABLE`) let create_sql = staging_def.to_create_sql(true)?; - let create_temp = create_sql.replace("CREATE TABLE", "CREATE TEMPORARY TABLE"); - connection.execute_command(&create_temp)?; + connection.execute_command(&create_sql)?; + + // From here on the guard owns cleanup, including if the inner + // inserter cannot be constructed. + let staging = StagingTable { + connection, + name: staging_table, + dropped: false, + }; // Create the inner inserter for the staging table let inner = Inserter::new(connection, &staging_def)?; @@ -1178,7 +1305,7 @@ impl<'conn> MappedInserter<'conn> { inner, target_table, mappings: mappings.to_vec(), - staging_table, + staging, }) } @@ -1301,44 +1428,46 @@ impl<'conn> MappedInserter<'conn> { /// to the target table is rejected (e.g. a mapping expression fails /// to evaluate). /// - Returns [`Error::Server`] if dropping the staging table fails. - pub fn execute(&mut self) -> Result { - let connection = self.inner.connection; - let staging_table = self.staging_table.clone(); + pub fn execute(self) -> Result { + let MappedInserter { + inner, + target_table, + mappings, + mut staging, + } = self; + let connection = inner.connection; - // Insert data into staging table - let _staging_rows = self.inner.execute()?; + // Insert data into staging table. `inner` is consumed here, so its + // COPY writer is finished (or aborted on error) before any further + // statement runs on the connection. On error, `staging` is dropped + // by its guard. + inner.execute()?; // Build the INSERT...SELECT statement use hyperdb_api_core::protocol::escape::SqlIdentifier; - let target_columns: Vec = self - .mappings + let target_columns: Vec = mappings .iter() .map(|m| format!("{}", SqlIdentifier(&m.column_name))) .collect(); - let select_items: Vec = self - .mappings - .iter() - .map(ColumnMapping::to_select_item) - .collect(); + let select_items: Vec = + mappings.iter().map(ColumnMapping::to_select_item).collect(); let sql = format!( "INSERT INTO {} ({}) SELECT {} FROM {}", - self.target_table, + target_table, target_columns.join(", "), select_items.join(", "), - SqlIdentifier(&staging_table), + SqlIdentifier(&staging.name), ); - // Execute the INSERT...SELECT (returns row count directly) + // Execute the INSERT...SELECT (returns row count directly). On + // failure the guard still drops the staging table. let row_count = connection.execute_command(&sql)?; // Drop the staging table - connection.execute_command(&format!( - "DROP TABLE IF EXISTS {}", - SqlIdentifier(&staging_table) - ))?; + staging.drop_now()?; // Return the number of rows inserted Ok(row_count) @@ -1346,27 +1475,26 @@ impl<'conn> MappedInserter<'conn> { /// Cancels the insert and drops the staging table. /// - /// This method handles cleanup failures gracefully by logging warnings - /// instead of returning errors. This prevents masking the original error - /// that caused the cancellation. + /// Any open COPY into the staging table is aborted first (the connection + /// is locked by the COPY writer until then), after which the staging + /// table is dropped. This method handles cleanup failures gracefully by + /// logging warnings instead of returning errors. This prevents masking + /// the original error that caused the cancellation. /// /// # Logging /// /// Cleanup failures are logged using the `tracing` crate at WARN level. - /// If `tracing` is not initialized, errors are written to stderr. - pub fn cancel(&mut self) { - let connection = self.inner.connection; - let staging_table = &self.staging_table; - - // Drop the staging table, but don't fail if cleanup fails - // This avoids masking the original error that caused cancellation - if let Err(e) = connection.execute_command(&format!( - "DROP TABLE IF EXISTS \"{}\"", - staging_table.replace('"', "\"\"") - )) { - // Log the cleanup failure for debugging - // In production, consider using a logging framework like `tracing` - eprintln!("Warning: Failed to drop staging table '{staging_table}' during cancel: {e}"); + pub fn cancel(mut self) { + // Abort the COPY first: statements cannot run on the connection + // while the inner inserter's writer holds it. + self.inner.cancel_in_place(); + if let Err(e) = self.staging.drop_now() { + warn!( + target: "hyperdb_api", + error = %e, + table = %self.staging.name, + "failed to drop staging table during cancel", + ); } } } diff --git a/hyperdb-api/src/prepared.rs b/hyperdb-api/src/prepared.rs index 5a7061a0..a15ee1b0 100644 --- a/hyperdb-api/src/prepared.rs +++ b/hyperdb-api/src/prepared.rs @@ -148,6 +148,11 @@ impl<'conn> PreparedStatement<'conn> { /// Fetches exactly one row; errors if the result is empty. /// + /// Only the first chunk is read; if more rows remain the query is + /// cancelled and the backlog discarded (see + /// [`Rowset::first_row`](crate::Rowset::first_row)). Add `LIMIT 1` to + /// avoid producing the extra rows. + /// /// # Errors /// /// - Returns the error from [`query`](Self::query). @@ -159,6 +164,11 @@ impl<'conn> PreparedStatement<'conn> { /// Fetches at most one row; returns `None` if the result is empty. /// + /// Only the first chunk is read; if more rows remain the query is + /// cancelled and the backlog discarded (see + /// [`Rowset::first_row`](crate::Rowset::first_row)). Add `LIMIT 1` to + /// avoid producing the extra rows. + /// /// # Errors /// /// Returns the error from [`query`](Self::query); an empty result diff --git a/hyperdb-api/src/query_as.rs b/hyperdb-api/src/query_as.rs index bcd3c095..db280ab9 100644 --- a/hyperdb-api/src/query_as.rs +++ b/hyperdb-api/src/query_as.rs @@ -59,6 +59,27 @@ fn as_refs(params: &[BoundParam]) -> Vec<&dyn ToSqlParam> { params.iter().map(|p| p as &dyn ToSqlParam).collect() } +/// Runs `sql` (binding `params` when present) and returns only its first +/// row. Any unread remainder is cancelled and discarded (see +/// [`Rowset::first_row`](crate::Rowset::first_row)) so the connection stays in +/// sync without materializing the whole result. +fn first_row(conn: &Connection, sql: &str, params: &[BoundParam]) -> Result> { + if params.is_empty() { + conn.fetch_optional(sql) + } else { + conn.query_params(sql, &as_refs(params))?.first_row() + } +} + +/// Maps one row into `T`, resolving column names against the row's schema. +fn map_row(row: &crate::Row) -> Result { + let indices = row + .schema() + .map(crate::row_accessor::RowAccessor::build_indices) + .unwrap_or_default(); + T::from_row(crate::RowAccessor::new(row, &indices)) +} + /// A compiled, type-safe query. Created by the `query_as!` macro. /// /// Call `.fetch_all(&conn)`, `.fetch_one(&conn)`, or `.fetch_optional(&conn)` @@ -115,7 +136,10 @@ impl QueryAs { /// /// Returns a `hyperdb_api::Error` on connection or SQL failure. pub fn fetch_optional(self, conn: &Connection) -> Result> { - Ok(self.fetch_rows(conn)?.into_iter().next()) + first_row(conn, &self.sql, &self.params)? + .as_ref() + .map(map_row::) + .transpose() } /// Runs the query, binding `$N` arguments when there are any. @@ -175,10 +199,7 @@ impl QueryScalar { /// /// Returns `Error::Conversion` if the query returns zero rows. pub fn fetch_one(self, conn: &Connection) -> Result { - let rows = self.fetch_rows(conn)?; - rows.into_iter() - .next() - .map(|r| r.0) + self.fetch_optional(conn)? .ok_or_else(|| crate::Error::Conversion("query_scalar!: query returned no rows".into())) } @@ -188,8 +209,11 @@ impl QueryScalar { /// /// Returns a `hyperdb_api::Error` on connection or SQL failure. pub fn fetch_optional(self, conn: &Connection) -> Result> { - let rows = self.fetch_rows(conn)?; - Ok(rows.into_iter().next().map(|r| r.0)) + first_row(conn, &self.sql, &self.params)? + .as_ref() + .map(map_row::>) + .transpose() + .map(|opt| opt.map(|r| r.0)) } /// Runs the query, binding `$N` arguments when there are any. diff --git a/hyperdb-api/src/result.rs b/hyperdb-api/src/result.rs index 42fb0a7a..8177eca9 100644 --- a/hyperdb-api/src/result.rs +++ b/hyperdb-api/src/result.rs @@ -1364,16 +1364,25 @@ impl<'conn> Rowset<'conn> { /// # } /// ``` /// + /// # Unread rows are cancelled + /// + /// Only the first chunk is read. If the result holds more rows, the + /// rowset is dropped with the remainder unread, and its `Drop` cancels the + /// query and discards the in-flight backlog so the connection is ready for + /// the next request. A server error that would only have surfaced in the + /// unread part of the result is therefore not reported. Add `LIMIT 1` to + /// the query to avoid producing the extra rows at all. + /// /// # Errors /// /// Returns the error from [`next_chunk`](Self::next_chunk). An empty /// result set yields `Ok(None)`, not an error. pub fn first_row(mut self) -> crate::error::Result> { - if let Some(chunk) = self.next_chunk()? { - Ok(chunk.into_iter().next()) - } else { - Ok(None) - } + // `self` is dropped on return; for an unfinished stream that cancels + // and drains (see `QueryStream`'s `Drop`). + Ok(self + .next_chunk()? + .and_then(|chunk| chunk.into_iter().next())) } /// Gets the first row or returns an error if no rows were found. diff --git a/hyperdb-api/src/transport.rs b/hyperdb-api/src/transport.rs index 526ea85a..ba2b951c 100644 --- a/hyperdb-api/src/transport.rs +++ b/hyperdb-api/src/transport.rs @@ -152,6 +152,17 @@ impl Transport { } } + /// Like [`execute_command`](Self::execute_command) but returns `Ok(None)` + /// instead of blocking when the connection is busy. For destructors. + pub(crate) fn try_execute_command(&self, sql: &str) -> Result> { + match self { + Transport::Tcp(tcp) => Ok(tcp.client.try_exec(sql)?), + Transport::Grpc(_) => Err(Error::feature_not_supported( + "gRPC transport is read-only. Write operations are not supported over gRPC.", + )), + } + } + /// Execute a query returning raw Arrow IPC bytes. pub(crate) fn execute_query_to_arrow(&self, sql: &str) -> Result { match self { diff --git a/hyperdb-api/tests/async_connection_tests.rs b/hyperdb-api/tests/async_connection_tests.rs index c2bba6f6..901f94ed 100644 --- a/hyperdb-api/tests/async_connection_tests.rs +++ b/hyperdb-api/tests/async_connection_tests.rs @@ -658,3 +658,47 @@ async fn string_constructors_connect_over_a_unix_socket() { .unwrap(); assert_eq!(conn.database(), db.to_str()); } + +/// First-row fetches over a result larger than one 65,536-row chunk must +/// drain the backlog so the connection stays in sync. +#[tokio::test(flavor = "current_thread")] +async fn first_row_fetches_on_multi_chunk_results_keep_connection_in_sync() { + for rows in [70_000_i64, 65_536] { + let (_hyper, conn) = fresh_async_conn("async_first_row_drain").await.unwrap(); + conn.execute_command("CREATE TABLE t (v INT)") + .await + .unwrap(); + conn.execute_command(&format!( + "INSERT INTO t SELECT * FROM generate_series(1, {rows})" + )) + .await + .unwrap(); + + let row = conn.fetch_one("SELECT v FROM t ORDER BY v").await.unwrap(); + assert_eq!(row.get::(0), Some(1)); + assert_eq!( + conn.query_count("SELECT COUNT(*) FROM t").await.unwrap(), + rows + ); + + let row = conn + .fetch_optional("SELECT v FROM t ORDER BY v") + .await + .unwrap(); + assert!(row.is_some()); + assert_eq!( + conn.query_count("SELECT COUNT(*) FROM t").await.unwrap(), + rows + ); + + let v: i32 = conn + .fetch_scalar("SELECT v FROM t ORDER BY v") + .await + .unwrap(); + assert_eq!(v, 1); + assert_eq!( + conn.query_count("SELECT COUNT(*) FROM t").await.unwrap(), + rows + ); + } +} diff --git a/hyperdb-api/tests/inserter_tests.rs b/hyperdb-api/tests/inserter_tests.rs index 05db97a2..7c63a7cc 100644 --- a/hyperdb-api/tests/inserter_tests.rs +++ b/hyperdb-api/tests/inserter_tests.rs @@ -374,3 +374,311 @@ fn test_inserter_geography_nullable() { assert!(rows[0].1.is_some(), "First row should have a geography"); assert!(rows[1].1.is_none(), "Second row should have NULL"); } + +// ============================================================================= +// COPY lifecycle: cancel / drop / reuse / staging-table cleanup +// ============================================================================= + +/// More rows than one auto-flush chunk (`CHUNK_ROW_LIMIT` = 64_000), so the +/// inserter has opened a COPY session before `cancel`/drop. +const ROWS_OVER_AUTOFLUSH: i32 = 64_001; + +fn create_int_table(test: &TestConnection, name: &str) -> TableDefinition { + let table_def = TableDefinition::new(name).add_nullable_column("a", SqlType::int()); + Catalog::new(&test.connection) + .create_table(&table_def) + .expect("Failed to create table"); + table_def +} + +fn count_rows(test: &TestConnection, table: &str) -> i64 { + test.count_tuples(table).expect("COUNT(*) must succeed") +} + +/// Cancelling an inserter whose COPY session is open must abort the COPY +/// (nothing committed) and leave the connection usable. +#[test] +fn test_inserter_cancel_after_autoflush_leaves_connection_usable() { + let test = TestConnection::new().expect("Failed to create test connection"); + let table_def = create_int_table(&test, "cancel_copy"); + + let mut inserter = + Inserter::new(&test.connection, &table_def).expect("Failed to create inserter"); + for i in 0..ROWS_OVER_AUTOFLUSH { + inserter.add_row(&[&i]).expect("add_row"); + } + inserter.cancel(); + + assert_eq!( + count_rows(&test, "cancel_copy"), + 0, + "cancel must commit nothing" + ); + assert_eq!(test.execute_scalar_i32("SELECT 41 + 1").unwrap(), 42); + + // A brand-new inserter on the same table sees none of the cancelled rows. + let mut fresh = Inserter::new(&test.connection, &table_def).expect("inserter"); + fresh.add_row(&[&7i32]).unwrap(); + fresh.add_row(&[&8i32]).unwrap(); + assert_eq!(fresh.execute().unwrap(), 2); + assert_eq!(count_rows(&test, "cancel_copy"), 2); +} + +/// Dropping an inserter mid-COPY (no execute/cancel) must also leave the +/// connection usable. +#[test] +fn test_inserter_dropped_mid_copy_leaves_connection_usable() { + let test = TestConnection::new().expect("Failed to create test connection"); + let table_def = create_int_table(&test, "drop_copy"); + + { + let mut inserter = + Inserter::new(&test.connection, &table_def).expect("Failed to create inserter"); + for i in 0..ROWS_OVER_AUTOFLUSH { + inserter.add_row(&[&i]).expect("add_row"); + } + } + + assert_eq!(count_rows(&test, "drop_copy"), 0); + assert_eq!(test.execute_scalar_i32("SELECT 41 + 1").unwrap(), 42); +} + +/// A finished inserter must not leak state into the next one on the same +/// table: 64_001 rows via execute, then a fresh inserter adds exactly its own. +/// (Reusing the *same* inserter is a compile error, see the `compile_fail` +/// doctest on `Inserter::execute`.) +#[test] +fn test_inserter_execute_then_fresh_inserter() { + let test = TestConnection::new().expect("Failed to create test connection"); + let table_def = create_int_table(&test, "exec_then_fresh"); + + let mut inserter = Inserter::new(&test.connection, &table_def).expect("inserter"); + for i in 0..ROWS_OVER_AUTOFLUSH { + inserter.add_row(&[&i]).unwrap(); + } + assert_eq!( + inserter.execute().unwrap(), + u64::try_from(ROWS_OVER_AUTOFLUSH).unwrap() + ); + + let mut fresh = Inserter::new(&test.connection, &table_def).expect("inserter"); + fresh.add_row(&[&1i32]).unwrap(); + assert_eq!(fresh.execute().unwrap(), 1); + assert_eq!( + count_rows(&test, "exec_then_fresh"), + i64::from(ROWS_OVER_AUTOFLUSH) + 1 + ); +} + +/// A reader that fails mid-stream: `import_text` must abort the COPY and +/// leave the connection usable. +#[test] +fn test_import_text_read_error_leaves_connection_usable() { + use hyperdb_api::copy::CopyOptions; + + struct FailingReader { + sent: bool, + } + impl std::io::Read for FailingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.sent { + return Err(std::io::Error::other("boom")); + } + self.sent = true; + let line = b"1\n2\n"; + buf[..line.len()].copy_from_slice(line); + Ok(line.len()) + } + } + + let test = TestConnection::new().expect("Failed to create test connection"); + let _ = create_int_table(&test, "import_fail"); + + let err = test + .connection + .import_text( + "import_fail", + &CopyOptions::tsv(), + FailingReader { sent: false }, + ) + .expect_err("import must report the read error"); + assert!(err.to_string().contains("read import data"), "got: {err}"); + + assert_eq!(count_rows(&test, "import_fail"), 0); + assert_eq!(test.execute_scalar_i32("SELECT 41 + 1").unwrap(), 42); +} + +/// Runs `body` on a thread and fails the test (instead of hanging forever) if +/// it does not finish within `secs`. The connection is moved into the thread; +/// the `HyperProcess` stays on the caller so a timeout kills `hyperd`. +fn run_with_timeout( + secs: u64, + body: impl FnOnce(hyperdb_api::Connection) + Send + 'static, +) -> hyperdb_api::HyperProcess { + let TestConnection { + hyper, connection, .. + } = TestConnection::new().expect("Failed to create test connection"); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + body(connection); + let _ = tx.send(()); + }); + rx.recv_timeout(std::time::Duration::from_secs(secs)) + .expect("body did not finish in time (deadlock?)"); + hyper +} + +/// `MappedInserter::cancel` used to run `DROP TABLE` while the inner +/// inserter's `CopyInWriter` still held the connection lock (a same-thread +/// self-deadlock). It must abort the COPY first, then drop staging. +#[test] +fn test_mapped_inserter_cancel_after_autoflush_does_not_deadlock() { + use hyperdb_api::ColumnMapping; + + let _hyper = run_with_timeout(30, |conn| { + conn.execute_command("CREATE TABLE mapped_target (a INT)") + .unwrap(); + let def = TableDefinition::new("ignored").add_nullable_column("a", SqlType::int()); + let mut inserter = Inserter::with_column_mappings( + &conn, + &def, + "mapped_target", + &[ColumnMapping::new("a")], + ) + .unwrap(); + for i in 0..ROWS_OVER_AUTOFLUSH { + inserter.add_row(&[&i]).unwrap(); + } + inserter.cancel(); + + assert_eq!( + conn.execute_scalar_query::("SELECT 1").unwrap(), + Some(1) + ); + let leftover = conn + .execute_scalar_query::( + "SELECT COUNT(*) FROM pg_catalog.pg_class WHERE relname LIKE '\\_hyper\\_staging\\_%'", + ) + .unwrap(); + assert_eq!(leftover, Some(0), "staging table must be dropped"); + assert_eq!( + conn.execute_scalar_query::("SELECT COUNT(*) FROM mapped_target") + .unwrap(), + Some(0) + ); + }); +} + +/// Two mapped inserters on one connection used a pid-only staging name and +/// collided on `CREATE TEMPORARY TABLE`. +#[test] +fn test_two_mapped_inserters_on_one_connection_do_not_collide() { + use hyperdb_api::ColumnMapping; + + let test = TestConnection::new().expect("Failed to create test connection"); + test.execute_command("CREATE TABLE mapped_two (a INT)") + .unwrap(); + let def = TableDefinition::new("ignored").add_nullable_column("a", SqlType::int()); + let maps = [ColumnMapping::new("a")]; + + let mut one = Inserter::with_column_mappings(&test.connection, &def, "mapped_two", &maps) + .expect("first mapped inserter"); + let mut two = Inserter::with_column_mappings(&test.connection, &def, "mapped_two", &maps) + .expect("second mapped inserter must not collide on the staging name"); + one.add_row(&[&1i32]).unwrap(); + two.add_row(&[&2i32]).unwrap(); + assert_eq!(one.execute().unwrap(), 1); + assert_eq!(two.execute().unwrap(), 1); + assert_eq!(count_rows(&test, "mapped_two"), 2); +} + +// ============================================================================= +// Server rejects a value mid-COPY (row 10 of an auto-flushed 64,001-row batch) +// ============================================================================= + +/// Creates `name (a VARCHAR(3))` in the database but describes it to the +/// inserter as `TEXT`, so a longer string is rejected by the *server* while +/// the COPY is streaming. +fn inserter_into_narrow_varchar<'a>(test: &'a TestConnection, name: &str) -> Inserter<'a> { + test.execute_command(&format!("CREATE TABLE {name} (a VARCHAR(3))")) + .expect("create table"); + let def = TableDefinition::new(name).add_nullable_column("a", SqlType::text()); + let mut inserter = Inserter::new(&test.connection, &def).expect("inserter"); + for i in 0..ROWS_OVER_AUTOFLUSH { + let value = if i == 10 { "toolong" } else { "ok" }; + inserter.add_row(&[&value]).expect("add_row"); + } + inserter +} + +fn assert_in_sync_after_rejected_copy(test: &TestConnection, table: &str) { + for expected in [42, 43, 44] { + assert_eq!( + test.execute_scalar_i32(&format!("SELECT {expected}")) + .unwrap(), + expected + ); + } + assert_eq!( + count_rows(test, table), + 0, + "rejected COPY must commit nothing" + ); +} + +#[test] +fn test_inserter_server_rejected_mid_copy_then_drop_leaves_connection_in_sync() { + let test = TestConnection::new().expect("Failed to create test connection"); + drop(inserter_into_narrow_varchar(&test, "reject_drop")); + assert_in_sync_after_rejected_copy(&test, "reject_drop"); +} + +#[test] +fn test_inserter_server_rejected_mid_copy_then_cancel_leaves_connection_in_sync() { + let test = TestConnection::new().expect("Failed to create test connection"); + inserter_into_narrow_varchar(&test, "reject_cancel").cancel(); + assert_in_sync_after_rejected_copy(&test, "reject_cancel"); +} + +#[test] +fn test_inserter_server_rejected_mid_copy_execute_errors_and_stays_in_sync() { + let test = TestConnection::new().expect("Failed to create test connection"); + let result = inserter_into_narrow_varchar(&test, "reject_exec").execute(); + assert!(result.is_err(), "server must reject the over-long value"); + assert_in_sync_after_rejected_copy(&test, "reject_exec"); +} + +/// `StagingTable`'s destructor used to run `DROP TABLE` through the +/// connection even while the same thread still held a live result stream +/// (which owns the non-reentrant connection lock), hanging forever. It must +/// now notice the busy connection, leak the session-scoped temp table and +/// return. +#[test] +fn test_mapped_inserter_dropped_while_result_stream_is_live_does_not_deadlock() { + use hyperdb_api::ColumnMapping; + + let _hyper = run_with_timeout(15, |conn| { + conn.execute_command("CREATE TABLE mapped_busy (a INT)") + .unwrap(); + let def = TableDefinition::new("ignored").add_nullable_column("a", SqlType::int()); + let inserter = + Inserter::with_column_mappings(&conn, &def, "mapped_busy", &[ColumnMapping::new("a")]) + .unwrap(); + + // A live stream holds the connection lock until it is dropped. + let mut rows = conn + .execute_query("SELECT v FROM generate_series(1, 200000) v") + .unwrap(); + let _ = rows.next_chunk().unwrap(); + + // Must return instead of deadlocking on the held lock. + drop(inserter); + drop(rows); + + // The connection is still usable afterwards. + assert_eq!( + conn.execute_scalar_query::("SELECT 1").unwrap(), + Some(1) + ); + }); +} diff --git a/hyperdb-api/tests/wire_desync_tests.rs b/hyperdb-api/tests/wire_desync_tests.rs index 02ccd415..9a2b06fc 100644 --- a/hyperdb-api/tests/wire_desync_tests.rs +++ b/hyperdb-api/tests/wire_desync_tests.rs @@ -137,18 +137,23 @@ fn query_after_failed_copy() -> Result<()> { /// Abandoning a large streaming result (many rows queued) and running a /// follow-up query must still work. This is the worst case for the Drop -/// drain path: the server has a large backlog to flush, and the bounded -/// drain must either catch up or yield the connection safely. +/// path: the server has a large backlog already flushed into the socket. +/// `Drop` cancels the query and then drains that backlog (`DataRow`s are +/// free against the drain's non-row budget), so the connection is handed +/// back clean. +/// +/// This test is a plain correctness check; it does not prove the cancel +/// fired (a 150,000-row backlog drains fine even without one). See +/// `cancel_fires_when_streaming_result_is_abandoned` for the test that +/// does. #[test] fn query_after_abandoning_large_stream() -> Result<()> { let tc = TestConnection::new()?; tc.execute_command("CREATE TABLE big (v INT)")?; - // Use a single INSERT ... VALUES expression so we don't hammer the - // server with 500 round-trips during test setup. - let values: Vec = (0..500).map(|i| format!("({i})")).collect(); - tc.execute_command(&format!("INSERT INTO big VALUES {}", values.join(",")))?; + // More than one 65,536-row chunk, so rows are genuinely still in flight + // after the first chunk is read. + tc.execute_command("INSERT INTO big SELECT * FROM generate_series(0, 149999)")?; - // Grab one chunk then drop, leaving hundreds of rows in flight. { let mut rs = tc .connection @@ -156,85 +161,68 @@ fn query_after_abandoning_large_stream() -> Result<()> { let _ = rs.next_chunk()?; } - // Connection must still produce correct results, regardless of whether - // the Drop drain caught the full backlog or the 4096-message cap - // triggered (either way the next query should succeed). let count = tc.execute_scalar_i64("SELECT COUNT(*) FROM big")?; - assert_eq!(count, 500); + assert_eq!(count, 150_000); Ok(()) } /// Exercises the cancel-on-`Drop` path for streaming result sets. /// -/// Without cancel-on-drop, abandoning a stream whose remaining backlog -/// exceeds the bounded drain budget (currently 1024 messages) would leave -/// the connection desynchronized and the follow-up query would fail or -/// return wrong data. With cancel-on-drop, the server observes the -/// `CancelRequest` on a separate connection, stops emitting rows, and our -/// drain reaches `ReadyForQuery` well within the budget. -/// -/// We use `generate_series(1, 100000)` so the server has ~100k rows of -/// `DataRows` ready to stream; that is comfortably larger than any -/// reasonable drain cap, so a passing follow-up query is strong evidence -/// that the cancel landed and short-circuited the stream. +/// Hyper computes a result and queues it for the socket quickly, so a +/// wall-clock bound cannot tell "cancelled" from "drained everything" +/// (measured: ~20 ms either way for a few hundred thousand rows). The +/// discriminating signal is the drain's hard ceiling on abandoned +/// `DataRow`s: a result far above that ceiling can only be handed back +/// clean if the server was told to stop producing. Without the +/// `CancelRequest` the drain exhausts the ceiling, marks the connection +/// desynchronized, and the follow-up query fails. #[test] fn cancel_fires_when_streaming_result_is_abandoned() -> Result<()> { let tc = TestConnection::new()?; - // Seed a table large enough that a full passive drain would far - // exceed the 1024-message post-cancel drain budget. We use a single - // `INSERT ... VALUES` expression so we don't care whether hyperd's - // `generate_series` caps out at 66k or any other implementation - // artifact: 3000 rows is deterministic and comfortably above 1024. - tc.execute_command("CREATE TABLE huge (v INT)")?; - let values: Vec = (0..3000).map(|i| format!("({i})")).collect(); - tc.execute_command(&format!("INSERT INTO huge VALUES {}", values.join(",")))?; - let seeded = tc.execute_scalar_i64("SELECT COUNT(*) FROM huge")?; - assert_eq!(seeded, 3000, "test setup: INSERT didn't seed all rows"); - - let start = std::time::Instant::now(); - - // Open a streaming query, read one chunk, then drop. - // - // Before cancel-on-drop: a 3000-row stream would almost certainly fit - // within the 4096-message bounded drain cap we had, so this test would - // still pass without the cancel — but the interesting property now is - // that the 1024-message *post-cancel* cap is reliably sufficient - // *because* cancel stops the server from producing more rows. If the - // cancel path regressed (e.g. someone removed the `CancelRequest` call - // in `QueryStream::Drop`), the bounded drain would blow its 1024 - // budget on ~3000 waiting DataRows and the connection would end up - // poisoned — the follow-up query would fail or return wrong data. + // 10,000,000 rows is more than twice the drain's DataRow ceiling + // (about 4.2 million); the cancelled backlog is typically well under a + // million. { let mut rs = tc .connection - .execute_query("SELECT * FROM huge ORDER BY v")?; - let _first = rs.next_chunk()?; + .execute_query("SELECT v FROM generate_series(1, 10000000) v")?; + let first = rs.next_chunk()?.expect("first chunk"); + assert_eq!(first.len(), 65_536); } - let drop_elapsed = start.elapsed(); - - // Follow-up query MUST succeed with the correct count: proof that the - // post-cancel drain caught `ReadyForQuery` before its budget ran out - // and the connection is healthy. - let count = tc.execute_scalar_i64("SELECT COUNT(*) FROM huge")?; - assert_eq!(count, 3000); - - // Timing assertion: Drop (cancel + bounded drain) should finish well - // under a second on loopback — locally we see ~1-10 ms. A 3-second - // threshold leaves ~100x headroom for noisy CI runners while still - // being tight enough to catch the regression class we actually care - // about: "cancel didn't fire and the drain is reading every row - // serially" would blow far past 3 seconds for 3000 rows. - // - // Using `<` (not `<=`) because equality at exactly the threshold is - // already the failure mode — something took suspiciously long. - const DROP_DEADLINE: std::time::Duration = std::time::Duration::from_secs(3); - assert!( - drop_elapsed < DROP_DEADLINE, - "stream Drop took {drop_elapsed:?} (deadline {DROP_DEADLINE:?}), \ - suggesting cancel did not fire", - ); + // The connection must be healthy and in sync. + let count = tc.execute_scalar_i64("SELECT COUNT(*) FROM generate_series(1, 10)")?; + assert_eq!(count, 10); + Ok(()) +} + +/// A `CancelRequest` travels on a separate connection and has no ordering +/// against the main socket, so it could in principle land on the *next* +/// query after the drain and fail it with `57014`. Hammer the +/// drop-then-query pattern, including a result of exactly 65,537 rows (one +/// full chunk plus a one-row tail, so the stream is unfinished when +/// dropped), and require every follow-up to succeed. +#[test] +fn late_cancel_never_fails_the_next_query() -> Result<()> { + let tc = TestConnection::new()?; + for i in 0..50_i64 { + let rows: i64 = match i % 3 { + 0 => 65_537, + 1 => 200_000, + _ => 65_536, + }; + { + let mut rs = tc + .connection + .execute_query(&format!("SELECT v FROM generate_series(1, {rows}) v"))?; + let _ = rs.next_chunk()?; + } + let n = tc + .execute_scalar_i64("SELECT COUNT(*) FROM generate_series(1, 1000)") + .unwrap_or_else(|e| panic!("iteration {i} ({rows} rows): follow-up failed: {e}")); + assert_eq!(n, 1000, "iteration {i} ({rows} rows)"); + } Ok(()) } @@ -261,3 +249,37 @@ fn begin_transaction_after_error_is_not_poisoned() -> Result<()> { assert_eq!(v, 7); Ok(()) } + +/// Seeds `t(v INT)` with `rows` rows via a single `generate_series` insert. +fn seed_series(tc: &TestConnection, rows: i64) -> Result<()> { + tc.execute_command("CREATE TABLE t (v INT)")?; + tc.execute_command(&format!( + "INSERT INTO t SELECT * FROM generate_series(1, {rows})" + ))?; + assert_eq!(tc.execute_scalar_i64("SELECT COUNT(*) FROM t")?, rows); + Ok(()) +} + +/// `fetch_one` / `fetch_optional` / `fetch_scalar` take the first row of a +/// result larger than one 65,536-row chunk; the unread backlog must be +/// drained so the next query is not answered with stale `DataRow`s. +#[test] +fn first_row_fetches_on_multi_chunk_results_keep_connection_in_sync() -> Result<()> { + for rows in [70_000_i64, 65_536] { + let tc = TestConnection::new()?; + seed_series(&tc, rows)?; + + let row = tc.connection.fetch_one("SELECT v FROM t ORDER BY v")?; + assert_eq!(row.get::(0), Some(1)); + assert_eq!(tc.execute_scalar_i64("SELECT COUNT(*) FROM t")?, rows); + + let row = tc.connection.fetch_optional("SELECT v FROM t ORDER BY v")?; + assert!(row.is_some()); + assert_eq!(tc.execute_scalar_i64("SELECT COUNT(*) FROM t")?, rows); + + let v: i32 = tc.connection.fetch_scalar("SELECT v FROM t ORDER BY v")?; + assert_eq!(v, 1); + assert_eq!(tc.execute_scalar_i64("SELECT COUNT(*) FROM t")?, rows); + } + Ok(()) +} From 84e5ce261d413ae13cb3465a01ca8a381f5f21a9 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 20:31:40 -0700 Subject: [PATCH 45/61] fix(mcp): stop data loss in merge, replace and copy paths and clean up scratch tables Create the merge staging guard before the staging load, skip cleanup when a CTAS fails with 42P07 so another session's table is not dropped, keep the original error code across the rename swap, parse modes explicitly, and run catalog bootstrap inside catch_unwind. --- hyperdb-mcp/CHANGELOG.md | 44 +- hyperdb-mcp/DEVELOPMENT.md | 20 +- hyperdb-mcp/examples/demo.rs | 3 +- hyperdb-mcp/src/daemon/discovery.rs | 250 +++++++ hyperdb-mcp/src/engine.rs | 67 +- hyperdb-mcp/src/ingest.rs | 676 +++++++++++-------- hyperdb-mcp/src/ingest_arrow.rs | 40 +- hyperdb-mcp/src/lakehouse.rs | 61 +- hyperdb-mcp/src/lib.rs | 1 + hyperdb-mcp/src/readme.rs | 11 +- hyperdb-mcp/src/schema.rs | 161 +++-- hyperdb-mcp/src/scratch.rs | 122 ++++ hyperdb-mcp/src/server.rs | 402 +++++++---- hyperdb-mcp/src/watcher.rs | 3 +- hyperdb-mcp/tests/daemon_tests.rs | 24 +- hyperdb-mcp/tests/end_to_end_mcp_tests.rs | 510 ++++++++++++++ hyperdb-mcp/tests/export_tests.rs | 10 +- hyperdb-mcp/tests/ingest_arrow_tests.rs | 61 +- hyperdb-mcp/tests/ingest_tests.rs | 178 +++-- hyperdb-mcp/tests/integration_tests.rs | 15 +- hyperdb-mcp/tests/lakehouse_tests.rs | 37 +- hyperdb-mcp/tests/per_tool_database_tests.rs | 45 +- hyperdb-mcp/tests/recovery_tests.rs | 23 +- hyperdb-mcp/tests/schema_tests.rs | 71 +- hyperdb-mcp/tests/transaction_tests.rs | 216 +++++- 25 files changed, 2296 insertions(+), 755 deletions(-) create mode 100644 hyperdb-mcp/src/scratch.rs diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index ebc272a4..4815e2a5 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -70,6 +70,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). against one state directory at the same time is not supported: an old client may overwrite `daemon.json` (not verified against the old code), in which case a new client no longer finds the daemon through the record. +- **BREAKING: `ingest::IngestOptions.mode` and `lakehouse::IcebergIngestOptions.mode` + are enums, and `IngestOptions.merge_key` is gone.** Use `IngestMode::{Replace, + Append, Merge { keys }}` and `IcebergMode::{Replace, Append}`; the merge keys + now live in the `Merge` variant. `IngestMode::parse` and `IcebergMode::parse` + accept only the exact lowercase names. Ingest paths that cannot merge return + `INVALID_ARGUMENT` for `Merge` instead of silently ignoring it. ### Removed @@ -138,12 +144,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **`query_data` / `query_file` no longer leak their scratch table on a failed query** — the temp table is now dropped whether the query succeeds or fails, so a bad query no longer leaves a `_tmp_*` table behind in `describe`. -- **Table listings (`describe`) now hide transient scratch tables** (`_tmp_*` - and `__hyperdb_merge_*`), not just `_hyperdb_*` internals. +- **Table listings (`describe`) now hide transient scratch tables** (`_tmp_*`, + `__hyperdb_merge_*` and `__hyperdb_copy_*`), not just `_hyperdb_*` internals. - **`query_data` / `query_file` table-name substitution is whole-word** — an alias like `data` no longer corrupts a column named `metadata` or `data_url`. - **Unknown-table SQL errors now suggest running `describe`** and name the default `data` alias, instead of the generic "check SQL syntax" hint. +- **An unrecognised `mode` no longer silently replaces data.** `load_data`, + `load_file`, `load_files`, `load_iceberg` and `copy_query` used to treat any + value other than the exact lowercase names (`Append`, `appendd`, `""`) as + `replace`. Each tool now parses `mode` once at the boundary and returns + `INVALID_ARGUMENT` with a did-you-mean hint; `load_files` rejects a bad entry + before ingesting anything. +- **A failed `load_file` merge no longer loses the matched rows.** The + DELETE and INSERT now run in one transaction, so an INSERT failure (for + example a NULL into a NOT NULL column) restores the deleted rows. A NOT NULL + target column missing from the incoming data is rejected up front, columns + added for the merge are dropped again on failure, and the staging table + (`__hyperdb_merge_*`, hidden from listings) is always cleaned up. +- **`copy_query` `mode=replace` no longer drops the target before the SELECT + succeeds.** The result is built in a hidden staging table + (`__hyperdb_copy_*`) and swapped in with two renames, so a failing SELECT + leaves the existing table untouched and the SELECT may read the target. A + `create`/`replace` into a missing target that fails at run time no longer + leaves an empty table behind. Hyper cannot roll back renames, so the swap + itself is not one atomic step. +- **An out-of-range `NUMERIC` no longer panics inside the engine lock.** A + `schema` override such as `NUMERIC(40,2)`, or an Arrow `Decimal256(50,2)` + column, now returns `INVALID_ARGUMENT` (precision must be 1..=38, scale + 0..=precision). Type strings are parsed strictly (`NUMERICX` and + `NUMERIC, x INT` are rejected) and stored in canonical form, so user text is + never spliced into DDL. A panic in any tool body is now caught and reported as + an internal error instead of poisoning the engine mutex; the engine is + discarded and the next call reconnects (in local mode this drops the whole + ephemeral workspace, so tables loaded earlier in the session are lost). `schema::map_hyper_type` now returns `Result`. +- **The daemon's `hyperd` endpoint is no longer trusted from `daemon.json` + alone.** The client confirms it with the authenticated daemon's `STATUS` reply + and accepts only a daemon-owned IPC channel (`/sockets/`, or a + local named pipe on Windows), so a forged record cannot point sessions at a + TCP listener of an attacker's choosing; otherwise the session runs in local + mode. ## [1.0.0-rc.4] - 2026-09-08 diff --git a/hyperdb-mcp/DEVELOPMENT.md b/hyperdb-mcp/DEVELOPMENT.md index 04944231..d37dd54c 100644 --- a/hyperdb-mcp/DEVELOPMENT.md +++ b/hyperdb-mcp/DEVELOPMENT.md @@ -232,16 +232,26 @@ Logs land next to the persistent file when one is supplied (so users find them i ## Daemon Mode Internals `Engine::new` defaults to *daemon mode* — it tries -`daemon::spawn::ensure_daemon(resolve_port_scan())` first, which discovers an +`daemon::spawn::ensure_daemon()` first, which discovers an existing daemon via `~/.hyperdb/daemon.json` (overridable via `HYPERDB_STATE_DIR` — see `daemon::state_perms`, which restricts that directory and the files in it to the owning user, and which needs the override to name a path that can carry those permissions: inside `%USERPROFILE%` on Windows, on a mode-supporting filesystem on Unix), else auto-spawns one as a detached -background process. The Engine then connects via TCP -(`Connection::connect(endpoint, …)`) without owning any `HyperProcess`, and -records the daemon's `health_endpoint` so the server's debounced `HEARTBEAT` -targets the discovered endpoint rather than re-resolving. +background process. The Engine then connects over the daemon's local IPC +channel (a Unix domain socket at `/sockets/hyper`, or a Windows +named pipe; `Connection::connect(endpoint, …)`) without owning any +`HyperProcess`, and records the daemon's `health_endpoint` so the server's +debounced `HEARTBEAT` targets the discovered endpoint rather than +re-resolving. + +`daemon.json` is a locator, not a credential, so the `hyperd` endpoint in it is +never dialed on its own. `daemon::discovery::trusted_hyperd_endpoint` asks the +daemon for `STATUS` over the (state-directory-pinned) health endpoint and +accepts the endpoint only if the reply equals the record's value and has the +shape of a daemon-owned IPC channel (`/sockets/` on Unix, a +local `\\.\pipe\` on Windows). TCP endpoints and paths outside the +state directory are refused and the engine falls back to local mode. Falls back to local mode (per-session `hyperd` via `HyperProcess::new`) when the daemon can't be reached (including a state directory that is not owned by the user or is group/other-writable), or always when `--no-daemon` is passed. diff --git a/hyperdb-mcp/examples/demo.rs b/hyperdb-mcp/examples/demo.rs index e1151063..fd9df847 100644 --- a/hyperdb-mcp/examples/demo.rs +++ b/hyperdb-mcp/examples/demo.rs @@ -238,9 +238,8 @@ fn main() -> Result<(), Box> { section("Step 2 · Ingest — schema is inferred, not declared"); let ingest_opts = IngestOptions { table: "coder_stats".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let ingest_result = ingest_csv_file(&mut engine, csv_path.to_str().unwrap(), &ingest_opts)?; diff --git a/hyperdb-mcp/src/daemon/discovery.rs b/hyperdb-mcp/src/daemon/discovery.rs index 04fcc73a..4c32e058 100644 --- a/hyperdb-mcp/src/daemon/discovery.rs +++ b/hyperdb-mcp/src/daemon/discovery.rs @@ -507,6 +507,77 @@ fn home_dir() -> Option { .map(PathBuf::from) } +/// Whether `endpoint` is a shape the daemon's own `hyperd` could have been +/// published at: a local IPC channel, never TCP. +/// +/// - Unix: the socket `/sockets/` the daemon creates (the +/// directory is made owner-only by `build_params`), written without `..` +/// or extra path components. +/// - Windows: a local named pipe, `\\.\pipe\` (host `.` only). +/// +/// A discovery record is a locator an attacker with write access to a +/// misconfigured state directory could forge; refusing anything outside this +/// shape stops a client from sending credentials and data to a fake `hyperd`. +#[must_use] +pub(crate) fn is_valid_hyperd_endpoint(endpoint: &str, state_dir: &Path) -> bool { + if cfg!(windows) { + is_local_pipe_endpoint(endpoint) + } else { + is_state_dir_socket_endpoint(endpoint, state_dir) + } +} + +fn is_state_dir_socket_endpoint(endpoint: &str, state_dir: &Path) -> bool { + use std::path::Component; + let path = Path::new(endpoint); + if !path.is_absolute() + || path + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + { + return false; + } + path.parent() == Some(state_dir.join("sockets").as_path()) && path.file_name().is_some() +} + +fn is_local_pipe_endpoint(endpoint: &str) -> bool { + endpoint + .strip_prefix(r"\\.\pipe\") + .is_some_and(|name| !name.is_empty() && !name.contains(['\\', '/'])) +} + +/// The `hyperd` endpoint a client may connect to, or `None` to fall back to +/// local mode. +/// +/// `daemon.json` is only a locator, so its `hyperd_endpoint` is not trusted on +/// its own. The daemon is asked for `STATUS` over its health endpoint (which +/// [`discover_in`] already proved is the one this state directory implies); +/// the endpoint it reports must equal the record's and must have the shape of +/// a daemon-owned IPC channel ([`is_valid_hyperd_endpoint`]). A mismatch +/// means a forged record, or a restart in progress, and local mode is the safe +/// answer either way. +#[must_use] +pub(crate) fn trusted_hyperd_endpoint( + info: &DaemonInfo, + health: &HealthEndpoint, + state_dir: &Path, +) -> Option { + let reply = super::health::send_command(health, "STATUS").ok()?; + let status: serde_json::Value = serde_json::from_str(reply.trim()).ok()?; + let reported = status.get("hyperd_endpoint")?.as_str()?; + if reported != info.hyperd_endpoint { + tracing::warn!( + "daemon record and daemon STATUS disagree on the hyperd endpoint; not trusting it" + ); + return None; + } + if !is_valid_hyperd_endpoint(reported, state_dir) { + tracing::warn!("daemon hyperd endpoint is not a daemon-owned IPC channel; not trusting it"); + return None; + } + Some(reported.to_owned()) +} + #[cfg(test)] mod tests { use std::ffi::{OsStr, OsString}; @@ -1569,4 +1640,183 @@ mod tests { remove_legacy_record(tmp.path(), judged); assert!(target.exists(), "the symlink target must be left alone"); } + + // --- hyperd endpoint trust (R1) -------------------------------------- + + /// A TCP listener standing in for an attacker's fake `hyperd`; counts connections. + #[cfg(unix)] + struct AttackerListener { + addr: String, + accepted: Arc, + quit: Arc, + thread: Option>, + } + + #[cfg(unix)] + impl AttackerListener { + fn start() -> Self { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let accepted = Arc::new(AtomicUsize::new(0)); + let quit = Arc::new(AtomicBool::new(false)); + let (a, q) = (Arc::clone(&accepted), Arc::clone(&quit)); + let thread = std::thread::spawn(move || { + while !q.load(Ordering::SeqCst) { + if listener.accept().is_ok() { + a.fetch_add(1, Ordering::SeqCst); + } else { + std::thread::sleep(Duration::from_millis(5)); + } + } + }); + Self { + addr, + accepted, + quit, + thread: Some(thread), + } + } + + fn connections(&self) -> usize { + self.accepted.load(std::sync::atomic::Ordering::SeqCst) + } + } + + #[cfg(unix)] + impl Drop for AttackerListener { + fn drop(&mut self) { + self.quit.store(true, std::sync::atomic::Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } + } + + #[cfg(unix)] + fn legit_socket(dir: &Path) -> String { + dir.join("sockets") + .join("hyper") + .to_string_lossy() + .into_owned() + } + + #[cfg(unix)] + #[test] + fn hyperd_endpoint_shape_accepts_only_the_state_dir_socket() { + let dir = short_state_dir(); + let d = dir.path(); + assert!(is_valid_hyperd_endpoint(&legit_socket(d), d)); + for bad in [ + "127.0.0.1:54321".to_string(), + "localhost:7485".to_string(), + "/tmp/evil.sock".to_string(), + d.join("hyper").to_string_lossy().into_owned(), + d.join("sockets") + .join("..") + .join("hyper") + .to_string_lossy() + .into_owned(), + d.join("sockets") + .join("sub") + .join("hyper") + .to_string_lossy() + .into_owned(), + d.join("sockets").to_string_lossy().into_owned(), + "sockets/hyper".to_string(), + String::new(), + ] { + assert!( + !is_valid_hyperd_endpoint(&bad, d), + "{bad:?} must be rejected" + ); + } + } + + #[cfg(windows)] + #[test] + fn hyperd_endpoint_shape_accepts_only_a_local_named_pipe() { + let d = Path::new(r"C:\state"); + assert!(is_valid_hyperd_endpoint(r"\\.\pipe\hyper-1234", d)); + for bad in [ + "127.0.0.1:54321", + r"\\evilhost\pipe\hyper", + r"\\.\pipe\", + r"\\.\pipe\a\b", + "", + ] { + assert!( + !is_valid_hyperd_endpoint(bad, d), + "{bad:?} must be rejected" + ); + } + } + + /// The attack: `daemon.json` names an attacker's TCP listener as the + /// `hyperd` endpoint while a genuine daemon answers on the control + /// endpoint. The record's endpoint must be refused and never connected to. + #[cfg(unix)] + #[test] + fn a_record_naming_an_attacker_listener_is_rejected_and_never_dialed() { + let dir = short_state_dir(); + let attacker = AttackerListener::start(); + // The genuine daemon (authenticated: it owns the state dir's socket). + let genuine = DaemonInfo { + hyperd_endpoint: legit_socket(dir.path()), + ..info_for(dir.path(), 4242) + }; + let _daemon = LiveDaemon::start(dir.path(), &genuine); + // The tampered record. + let tampered = DaemonInfo { + hyperd_endpoint: attacker.addr.clone(), + ..genuine.clone() + }; + let health = HealthEndpoint::from_record(&tampered.health_endpoint, dir.path()).unwrap(); + + assert_eq!( + trusted_hyperd_endpoint(&tampered, &health, dir.path()), + None + ); + std::thread::sleep(Duration::from_millis(100)); + assert_eq!( + attacker.connections(), + 0, + "the attacker listener was dialed" + ); + } + + /// Even an authenticated STATUS cannot smuggle in a non-IPC endpoint. + #[cfg(unix)] + #[test] + fn an_authenticated_status_naming_a_tcp_endpoint_is_rejected() { + let dir = short_state_dir(); + let attacker = AttackerListener::start(); + let lying = DaemonInfo { + hyperd_endpoint: attacker.addr.clone(), + ..info_for(dir.path(), 4242) + }; + let _daemon = LiveDaemon::start(dir.path(), &lying); + let health = HealthEndpoint::from_record(&lying.health_endpoint, dir.path()).unwrap(); + + assert_eq!(trusted_hyperd_endpoint(&lying, &health, dir.path()), None); + std::thread::sleep(Duration::from_millis(100)); + assert_eq!(attacker.connections(), 0); + } + + #[cfg(unix)] + #[test] + fn a_consistent_record_and_status_yield_the_endpoint() { + let dir = short_state_dir(); + let info = DaemonInfo { + hyperd_endpoint: legit_socket(dir.path()), + ..info_for(dir.path(), 4242) + }; + let _daemon = LiveDaemon::start(dir.path(), &info); + let health = HealthEndpoint::from_record(&info.health_endpoint, dir.path()).unwrap(); + assert_eq!( + trusted_hyperd_endpoint(&info, &health, dir.path()), + Some(legit_socket(dir.path())) + ); + } } diff --git a/hyperdb-mcp/src/engine.rs b/hyperdb-mcp/src/engine.rs index 4ee0e947..c933dece 100644 --- a/hyperdb-mcp/src/engine.rs +++ b/hyperdb-mcp/src/engine.rs @@ -273,17 +273,10 @@ fn create_table_statements( "No columns to create table from", )); } - for col in columns { - if crate::schema::map_hyper_type(&col.hyper_type).is_none() { - return Err(McpError::new( - ErrorCode::SchemaMismatch, - format!( - "Unknown type '{}' for column '{}'", - col.hyper_type, col.name - ), - )); - } - } + let type_sqls: Vec = columns + .iter() + .map(|col| crate::schema::column_type_sql(&col.hyper_type, &col.name)) + .collect::>()?; let quoted_table = match target_db { Some(db) => { @@ -296,14 +289,10 @@ fn create_table_statements( let col_defs: Vec = columns .iter() - .map(|c| { + .zip(&type_sqls) + .map(|(c, ty)| { let nullable = if c.nullable { "" } else { " NOT NULL" }; - format!( - "\"{}\" {}{}", - c.name.replace('"', "\"\""), - c.hyper_type, - nullable - ) + format!("\"{}\" {ty}{nullable}", c.name.replace('"', "\"\"")) }) .collect(); @@ -624,7 +613,16 @@ impl Engine { return Ok(None); }; - let endpoint = &info.hyperd_endpoint; + // `daemon.json` is a locator, not a credential: connect only to an + // endpoint the authenticated daemon itself confirms and that has the + // shape of its own IPC channel (see `trusted_hyperd_endpoint`). + let Some(trusted_endpoint) = daemon::discovery::state_dir().ok().and_then(|dir| { + daemon::discovery::trusted_hyperd_endpoint(&info, &health_endpoint, &dir) + }) else { + tracing::warn!("daemon hyperd endpoint was not confirmed, falling back to local mode"); + return Ok(None); + }; + let endpoint = &trusted_endpoint; // CreateAndReplace: same rationale as the local path — a per-pid // temp file from a crashed prior session shouldn't leak in. let connection = Connection::connect( @@ -657,7 +655,7 @@ impl Engine { Ok(Some(Self { hyper: None, - daemon_endpoint: Some(info.hyperd_endpoint), + daemon_endpoint: Some(trusted_endpoint), daemon_health_endpoint: Some(health_endpoint), connection, ephemeral_path: ephemeral_path.to_path_buf(), @@ -1479,17 +1477,10 @@ impl Engine { if cols.is_empty() { return Ok(()); } - for col in cols { - if crate::schema::map_hyper_type(&col.hyper_type).is_none() { - return Err(McpError::new( - ErrorCode::SchemaMismatch, - format!( - "Unknown type '{}' for column '{}'", - col.hyper_type, col.name - ), - )); - } - } + let type_sqls: Vec = cols + .iter() + .map(|col| crate::schema::column_type_sql(&col.hyper_type, &col.name)) + .collect::>()?; let quoted_table = match target_db { Some(db) => { let esc_db = db.replace('"', "\"\""); @@ -1500,13 +1491,8 @@ impl Engine { }; let add_clauses = cols .iter() - .map(|c| { - format!( - "ADD COLUMN \"{}\" {}", - c.name.replace('"', "\"\""), - c.hyper_type - ) - }) + .zip(&type_sqls) + .map(|(c, ty)| format!("ADD COLUMN \"{}\" {ty}", c.name.replace('"', "\"\""))) .collect::>() .join(", "); let sql = format!("ALTER TABLE {quoted_table} {add_clauses}"); @@ -2102,10 +2088,11 @@ pub const HYPERDB_INTERNAL_PREFIX: &str = "_hyperdb_"; /// Reserved prefixes for short-lived scratch tables the server creates and /// drops within a single tool call: `_tmp_` (`query_data` / `query_file` -/// staging) and `__hyperdb_merge_` (`load_file` merge staging). Filtered +/// staging), `__hyperdb_merge_` (`load_file` merge staging) and +/// `__hyperdb_copy_` (`copy_query` replace staging / backup). Filtered /// from user-facing listings so an in-flight — or, after a mid-call /// failure, leaked — scratch table never surfaces as if it were user data. -const INTERNAL_SCRATCH_PREFIXES: [&str; 2] = ["_tmp_", "__hyperdb_merge_"]; +const INTERNAL_SCRATCH_PREFIXES: [&str; 3] = ["_tmp_", "__hyperdb_merge_", "__hyperdb_copy_"]; /// Returns true when `name` is one of `HyperDB`'s own internal tables: /// a persistent internal table ([`HYPERDB_INTERNAL_PREFIX`]) or a diff --git a/hyperdb-mcp/src/ingest.rs b/hyperdb-mcp/src/ingest.rs index 05f4466d..c37fe84d 100644 --- a/hyperdb-mcp/src/ingest.rs +++ b/hyperdb-mcp/src/ingest.rs @@ -18,11 +18,14 @@ //! the same call are rolled back, so a failed ingest leaves zero additional //! rows behind — and the guard's `Drop` covers the panic path too. //! -//! Note that Hyper auto-commits DDL (`DROP TABLE`, `CREATE TABLE`) regardless -//! of the surrounding transaction. In `replace` mode, this means the original -//! table is already gone by the time inserts start — a mid-ingest failure -//! leaves the new table empty, not the original intact. In `append` mode, no -//! DDL runs (assuming the table already exists), so rollback is fully atomic. +//! Note that Hyper auto-commits DDL (`DROP TABLE`, `CREATE TABLE`) once a +//! transaction contains it, and refuses DDL after DML. In `replace` mode, this +//! means the original table is already gone by the time inserts start — a +//! mid-ingest failure leaves the new table empty, not the original intact. In +//! `append` mode, no DDL runs (assuming the table already exists), so rollback +//! is fully atomic. `merge` mode is atomic for rows and columns: its +//! `DELETE` + `INSERT` run in one transaction, and a failed merge drops any +//! column it added (see [`merge_via_temp_table`]). use crate::engine::Engine; use crate::error::{ErrorCode, McpError}; @@ -30,6 +33,7 @@ use crate::schema::{ ColumnSchema, apply_schema_override, infer_csv_schema, infer_json_schema, json_type_name, widen_csv_numeric_columns, }; +use crate::scratch::ScratchTableGuard; use crate::stats::{IngestStats, StatsTimer}; use hyperdb_api::AsyncConnection; use std::path::{Path, PathBuf}; @@ -124,19 +128,176 @@ fn read_text_sample(path: impl AsRef, max_bytes: u64) -> std::io::Result, + default: Option<&'static str>, + allowed: &[&'static str], + tool: &str, +) -> Result<&'static str, McpError> { + let Some(raw) = value else { + return default.ok_or_else(|| { + McpError::new( + ErrorCode::InvalidArgument, + format!( + "{tool}: `mode` is required (one of: {})", + allowed.join(", ") + ), + ) + }); + }; + if let Some(hit) = allowed.iter().find(|a| **a == raw) { + return Ok(hit); + } + let hint = allowed + .iter() + .find(|a| a.eq_ignore_ascii_case(raw.trim())) + .map(|a| format!(" (did you mean '{a}'? mode values are exact lowercase)")) + .unwrap_or_default(); + Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "{tool}: invalid mode '{raw}'; allowed values: {}{hint}", + allowed.join(", ") + ), + )) +} + +/// How an ingest populates its target table. Parsed once at the tool +/// boundary by [`IngestMode::parse`]; every ingest path matches on it +/// exhaustively so an unrecognised value can never fall through to a +/// destructive replace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IngestMode { + /// Drop any existing table and recreate it from the incoming data. + Replace, + /// Add rows to the existing table (creating it when absent). + Append, + /// Upsert rows keyed on `keys`. + Merge { + /// Column(s) matching incoming rows to existing rows. Non-empty. + keys: Vec, + }, +} + +impl IngestMode { + /// Parse a tool's `mode` / `merge_key` parameters. + /// + /// `allowed` lists the mode names the tool supports (a subset of + /// `replace`, `append`, `merge`); an omitted `mode` means `replace`. + /// `merge_key` is required (and non-empty) for `merge` and rejected for + /// every other mode. + /// + /// # Errors + /// + /// [`ErrorCode::InvalidArgument`] for an unknown, mis-cased or + /// unsupported mode, or an invalid `merge_key` combination. + pub fn parse( + mode: Option<&str>, + merge_key: Option>, + allowed: &[&'static str], + tool: &str, + ) -> Result { + match parse_mode_name(mode, Some("replace"), allowed, tool)? { + "merge" => match merge_key { + None => Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "{tool}: mode=merge requires merge_key (a column name or list of \ + column names)" + ), + )), + Some(keys) if keys.is_empty() || keys.iter().any(String::is_empty) => { + Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "{tool}: merge_key must be a non-empty list of non-empty column names" + ), + )) + } + Some(keys) => Ok(Self::Merge { keys }), + }, + other @ ("append" | "replace") => { + if merge_key.is_some() { + return Err(McpError::new( + ErrorCode::InvalidArgument, + format!("{tool}: merge_key is only valid with mode=merge"), + )); + } + Ok(if other == "append" { + Self::Append + } else { + Self::Replace + }) + } + other => unreachable!("parse_mode_name returned a mode outside the known set: {other}"), + } + } + + /// The canonical lowercase name (`replace` / `append` / `merge`). + #[must_use] + pub const fn as_str(&self) -> &'static str { + match self { + Self::Replace => "replace", + Self::Append => "append", + Self::Merge { .. } => "merge", + } + } + + /// Whether this is a merge. + #[must_use] + pub const fn is_merge(&self) -> bool { + matches!(self, Self::Merge { .. }) + } + + /// The merge keys, when this is a merge. + #[must_use] + pub fn merge_keys(&self) -> Option<&[String]> { + match self { + Self::Merge { keys } => Some(keys), + _ => None, + } + } + + /// Whether the load recreates the table (`true`) or appends to it + /// (`false`). Used by the per-format loaders after they have already + /// diverted merges to [`merge_via_temp_table`]. + /// + /// # Errors + /// + /// [`ErrorCode::InvalidArgument`] for [`IngestMode::Merge`], which the + /// calling path does not support (e.g. the async batch loaders). + pub fn replaces(&self) -> Result { + match self { + Self::Replace => Ok(true), + Self::Append => Ok(false), + Self::Merge { .. } => Err(McpError::new( + ErrorCode::InvalidArgument, + "mode=merge is not supported by this ingest path; use load_file for merge", + )), + } + } +} + /// Controls how data is loaded into a target table. #[derive(Debug)] pub struct IngestOptions { pub table: String, - /// `"replace"` drops the existing table first; `"append"` adds rows to - /// it; `"merge"` upserts rows by [`Self::merge_key`]. - pub mode: String, + /// Replace, append, or merge (with its keys). + pub mode: IngestMode, /// When set, bypasses schema inference and uses these exact column types. pub schema_override: Option>, - /// When `mode == "merge"`, the column(s) to match on for upsert. Required - /// in merge mode; rejected in any other mode (the per-call site validates - /// up-front so the lower ingest paths can stay format-agnostic). - pub merge_key: Option>, /// Resolved database alias for fully-qualified SQL. `None` means the /// primary (ephemeral); `Some("persistent")` or `Some("user_alias")` /// qualifies table references as `""."public"."
"`. @@ -166,140 +327,62 @@ pub struct IngestResult { pub stats: IngestStats, } -/// RAII guard that ensures a temp table is dropped on **every** scope -/// exit — `Ok`, `Err`, *and* panic-unwind. Used by -/// [`merge_via_temp_table`] so an orphan `__hyperdb_merge_*` table -/// can't leak into the workspace, even if a per-format ingest path -/// panics mid-load. -/// -/// Disarm by calling [`TempTableGuard::disarm`] when the temp has -/// been renamed away (so the guard isn't dropping a real -/// user-facing table by name). -struct TempTableGuard<'a> { - engine: &'a Engine, - name: String, - /// Target database the temp lives in. `None` → primary (unqualified - /// drop). `Some(alias)` → emit `"alias"."public".""` so the - /// drop lands in the same DB the temp was created in. - target_db: Option, - armed: bool, -} - -impl<'a> TempTableGuard<'a> { - fn new(engine: &'a Engine, name: String, target_db: Option) -> Self { - Self { - engine, - name, - target_db, - armed: true, - } - } - fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for TempTableGuard<'_> { - fn drop(&mut self) { - if !self.armed { - return; - } - // Detect whether the guard is being dropped during normal - // unwinding vs. as part of a panic. Including this in the log - // lets ops distinguish "expected cleanup" from "we wouldn't - // have learned about the temp leak otherwise" — the latter is - // the load-bearing case for this guard's existence. - let panicking = std::thread::panicking(); - let quoted = match &self.target_db { - Some(db) => format!( - "\"{}\".\"public\".\"{}\"", - db.replace('"', "\"\""), - self.name.replace('"', "\"\""), - ), - None => format!("\"{}\"", self.name.replace('"', "\"\"")), - }; - let drop_sql = format!("DROP TABLE IF EXISTS {quoted}"); - match self.engine.execute_command(&drop_sql) { - Ok(_) => { - if panicking { - // Successful cleanup during a panic-unwind is - // exactly what the guard exists for. Note it at - // info so it shows up in normal log review without - // requiring debug-level capture. - tracing::info!( - tmp = %self.name, - "merge_via_temp_table: dropped temp table during panic unwind" - ); - } - } - Err(e) => { - // Don't mask the original outcome's error with a - // cleanup error — log and continue. The temp table is - // benign; the user can drop it manually if it ever - // surfaces. `panicking` is included so ops can tell - // whether this was a normal-path cleanup failure or - // happened while another error was already - // propagating. - tracing::warn!( - tmp = %self.name, - panicking, - error = %e, - "merge_via_temp_table: failed to drop temp table on guard exit" - ); - } - } - } -} - /// Implement `mode = "merge"` for any format by reusing that format's -/// `replace`-mode load to populate a temp table, then upserting the -/// target by `merge_key` columns. +/// `replace`-mode load to populate a staging table, then upserting the +/// target by the merge key columns. /// /// `replace_load` is the format-specific entry point (e.g. `ingest_json`, /// `ingest_parquet_file`) called with a `replace`-mode [`IngestOptions`] -/// pointing at a temp table. It must return the [`IngestResult`] for -/// the temp-table load — caller stitches that into the final result. +/// pointing at a staging table. It must return the [`IngestResult`] for +/// the staging load — caller stitches that into the final result. /// /// Algorithm: /// -/// 1. Validate `merge_key` is set and non-empty. -/// 2. Load incoming data into a unique temp table via `replace_load`. -/// A `TempTableGuard` arms here and unwind-safely drops the temp -/// on every exit (`Ok` / `Err` / panic). Verify the temp actually -/// materialized — a no-op load would otherwise produce confusing -/// downstream errors. -/// 3. If target table doesn't exist, rename the temp table → target, -/// disarm the guard (the temp is now the target, do **not** drop -/// it), and return (degenerates to "create"). -/// 4. Read target + temp column metadata. Validate every merge key -/// exists in both with `types_compatible` type. Reject any -/// non-key shared column whose type differs. -/// 5. Auto-`ALTER TABLE ADD COLUMN` for any column present in temp but -/// not target (added as nullable). When this fires, set -/// [`IngestStats::schema_changed`] so the caller can issue a +/// 1. Take the merge keys from [`IngestMode::Merge`] (non-empty). +/// 2. Load incoming data into a unique staging table via `replace_load`. +/// A `ScratchTableGuard` arms here and unwind-safely drops the staging +/// table on every exit (`Ok` / `Err` / panic). Verify the staging table +/// actually materialized — a no-op load would otherwise produce +/// confusing downstream errors. +/// 3. If target table doesn't exist, rename the staging table → target, +/// disarm the guard (the staging table is now the target, do **not** +/// drop it), and return (degenerates to "create"). +/// 4. Read target + staging column metadata. Validate every merge key +/// exists in both with `types_compatible` type. Reject any non-key +/// shared column whose type differs. Reject up front (before any DDL) +/// a `NOT NULL` target column that the incoming data does not supply: +/// the `INSERT` would be refused by Hyper after the matched rows were +/// already deleted. +/// 5. `ALTER TABLE ADD COLUMN` (nullable) for any column present in +/// staging but not target, recording what was added. When this fires, +/// set [`IngestStats::schema_changed`] so the caller can issue a /// resource-list-changed notification. -/// 6. `DELETE FROM target USING temp WHERE ` then -/// `INSERT INTO target () SELECT FROM temp`. -/// Columns the target has but temp doesn't are deliberately omitted -/// from the projection so they fall through to NULL — standard -/// PostgreSQL semantics for partial inserts. -/// 7. The `TempTableGuard` drops the temp on scope exit. +/// 6. In **one transaction**: `DELETE FROM target USING staging WHERE +/// ` then `INSERT INTO target () SELECT +/// FROM staging`. Columns the target has but staging +/// doesn't are omitted from the projection so they fall through to NULL +/// (they are guaranteed nullable by the step-4 pre-flight). +/// 7. The guard drops the staging table on scope exit. /// -/// Atomicity: the DELETE+INSERT pair is **not** wrapped in a transaction. -/// Hyper auto-commits DDL inside transactions (see module-level note), -/// and the existing `replace` mode already accepts the same race window, -/// so this matches precedent rather than introducing a new exposure. +/// Atomicity: the DELETE+INSERT pair is a real transaction (Hyper rejects +/// DDL inside a transaction only once it contains DML, and no DDL runs in +/// step 6), so a failing `INSERT` restores the deleted rows. The step-5 +/// `ADD COLUMN` auto-commits and is outside that transaction; if step 6 +/// fails the columns added in step 5 are dropped again on a best-effort +/// basis (a failure to do so is logged), so a failed merge leaves the +/// target's rows and columns unchanged. /// /// # Errors /// -/// - [`ErrorCode::InvalidArgument`] when `merge_key` is missing/empty, -/// or when a key column is missing from target or temp, or when a -/// column type differs between target and temp on a shared column. +/// - [`ErrorCode::InvalidArgument`] when the merge keys are empty, a key +/// column is missing from target or staging, a column type differs +/// between target and staging on a shared column, or a `NOT NULL` +/// target column is missing from the incoming data. /// - [`ErrorCode::InternalError`] if `replace_load` returns `Ok` but -/// doesn't actually create the temp table (a contract violation). -/// - Propagates errors from `replace_load`, [`Engine::column_metadata`], -/// [`Engine::alter_table_add_columns`], or the underlying DELETE / -/// INSERT statements. The temp table is dropped before the error +/// doesn't actually create the staging table (a contract violation). +/// - Propagates errors from `replace_load`, [`Engine::column_metadata_in`], +/// [`Engine::alter_table_add_columns_in`], or the underlying DELETE / +/// INSERT statements. The staging table is dropped before the error /// propagates. pub fn merge_via_temp_table( engine: &mut Engine, @@ -309,25 +392,16 @@ pub fn merge_via_temp_table( where F: FnOnce(&mut Engine, &IngestOptions) -> Result, { - // Belt-and-suspenders contract check. Every per-format ingest only - // calls this helper when `opts.mode == "merge"`; if a future - // refactor adds a third mode that also recurses, this debug-only - // assertion catches the mistake before it manifests as silent - // misbehavior. The closure also passes `mode = "replace"` to the - // recursed call, breaking infinite recursion. - debug_assert_eq!( - opts.mode, "merge", - "merge_via_temp_table called with non-merge mode `{}`", - opts.mode - ); - - // Step 1 — validate merge_key. - let keys = opts.merge_key.as_ref().ok_or_else(|| { - McpError::new( - ErrorCode::InvalidArgument, - "merge mode requires merge_key (a column name or list of column names)", - ) - })?; + // Step 1 — the merge keys travel inside the mode, already validated + // non-empty by `IngestMode::parse`. Every per-format ingest only calls + // this helper for a merge; the staging load below passes + // `IngestMode::Replace`, which breaks the recursion. + let IngestMode::Merge { keys } = &opts.mode else { + return Err(McpError::new( + ErrorCode::InternalError, + "merge_via_temp_table called with a non-merge mode", + )); + }; if keys.is_empty() || keys.iter().any(String::is_empty) { return Err(McpError::new( ErrorCode::InvalidArgument, @@ -335,65 +409,49 @@ where )); } - // Step 2 — load incoming data into a per-call temp table. The name - // mixes PID + nanosecond timestamp + a process-local atomic counter. - // The counter is what makes this race-free: two parallel merges of - // the same target inside one process can land in the same nanosecond - // (sub-µs OS timer, or platform-error fallback to `0`), but the - // counter is monotonically unique per call. That's enough to keep - // each merge's temp table distinct so the `TempTableGuard`s don't - // cross-drop one another. - use std::sync::atomic::{AtomicU64, Ordering}; - static MERGE_COUNTER: AtomicU64 = AtomicU64::new(0); - let counter = MERGE_COUNTER.fetch_add(1, Ordering::Relaxed); - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_nanos()); - // Squash anything that isn't [A-Za-z0-9_] in the target name to `_` - // so the auto-generated identifier is always SQL-safe. The temp - // name is never user-visible after a successful merge. - let safe_target: String = opts - .table - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() || c == '_' { - c - } else { - '_' - } - }) - .collect(); - let tmp = format!( - "__hyperdb_merge_{safe_target}_{}_{nanos}_{counter}", - std::process::id(), - ); - - // The temp table lives in the same database as the target. That - // way every DML statement below (CREATE TABLE AS, DELETE-USING, - // INSERT-SELECT) stays within a single DB — no cross-DB DML — and - // the merge works the same regardless of whether the target is - // primary, persistent, or any user-attached writable database. + // Step 2 — load incoming data into a per-call staging table that lives + // in the same database as the target. That way every DML statement + // below (DELETE-USING, INSERT-SELECT) stays within a single DB — no + // cross-DB DML — and the merge works the same regardless of whether + // the target is primary, persistent, or any user-attached writable + // database. + let tmp = crate::scratch::scratch_name(crate::scratch::MERGE_PREFIX, &opts.table); let tmp_opts = IngestOptions { table: tmp.clone(), - mode: "replace".into(), + mode: IngestMode::Replace, schema_override: opts.schema_override.clone(), - merge_key: None, target_db: opts.target_db.clone(), }; - let tmp_result = replace_load(engine, &tmp_opts)?; - - // Arm the cleanup guard immediately after the load so any later - // failure (or panic) drops the temp table on unwind. The guard - // tracks `target_db` so the DROP lands in the same DB the temp - // was created in. - let mut guard = TempTableGuard::new(engine, tmp.clone(), opts.target_db.clone()); + // Arm the cleanup guard BEFORE the load: Hyper commits DDL immediately, + // so a staging load that creates the table and then fails (bad value, + // panic) would otherwise leak it. The guard's drop is `DROP TABLE IF + // EXISTS`, so it is harmless when the load never got as far as creating it. + let mut guard = ScratchTableGuard::new(engine, qualified_table(&tmp_opts), tmp.clone()); + let tmp_result = replace_load(guard.engine(), &tmp_opts)?; + let (result, renamed_into_target) = + merge_from_staging(guard.engine(), opts, keys, &tmp, tmp_result)?; + if renamed_into_target { + // The staging table is now the target; the guard must NOT drop it. + guard.disarm(); + } + Ok(result) +} +/// Steps 3–6 of [`merge_via_temp_table`]. Returns the result and whether +/// the staging table was renamed into the (previously absent) target. +fn merge_from_staging( + engine: &mut Engine, + opts: &IngestOptions, + keys: &[String], + tmp: &str, + tmp_result: IngestResult, +) -> Result<(IngestResult, bool), McpError> { // Belt-and-suspenders check: a per-format `replace_load` that returns // `Ok` without actually creating the table would surface as opaque // "table not found" errors from the catalog read in step 4. Catch // it here with a clear message so the contract violation is named // outright. - if !engine.table_exists_in(opts.target_db.as_deref(), &tmp)? { + if !engine.table_exists_in(opts.target_db.as_deref(), tmp)? { return Err(McpError::new( ErrorCode::InternalError, format!( @@ -403,43 +461,44 @@ where )); } - // Step 3 — if target doesn't exist, rename temp → target. + let staging_opts = IngestOptions { + table: tmp.to_string(), + mode: IngestMode::Replace, + schema_override: None, + target_db: opts.target_db.clone(), + }; + let quoted_tmp = qualified_table(&staging_opts); + + // Step 3 — if target doesn't exist, rename staging → target. if !engine.table_exists_in(opts.target_db.as_deref(), &opts.table)? { - let qualified_tmp_opts = IngestOptions { - table: tmp.clone(), - mode: "replace".into(), - schema_override: None, - merge_key: None, - target_db: opts.target_db.clone(), - }; - let quoted_tmp = qualified_table(&qualified_tmp_opts); // RENAME TO accepts an unqualified new name (Hyper / PostgreSQL // semantics — the schema/database stays the same as the source). let escaped_new = opts.table.replace('"', "\"\""); engine.execute_command(&format!( "ALTER TABLE {quoted_tmp} RENAME TO \"{escaped_new}\"" ))?; - // The temp is now the target; the guard must NOT try to drop it. - guard.disarm(); - return Ok(IngestResult { - rows: tmp_result.rows, - schema: tmp_result.schema, - stats: IngestStats { - operation: tmp_result.stats.operation, - rows: tmp_result.stats.rows, - elapsed_ms: tmp_result.stats.elapsed_ms, - bytes_read: tmp_result.stats.bytes_read, - bytes_stored: tmp_result.stats.bytes_stored, - schema_inference_ms: tmp_result.stats.schema_inference_ms, - table: opts.table.clone(), - file_format: tmp_result.stats.file_format, - warning: tmp_result.stats.warning, - // Target was just created from scratch — by definition - // the resource list "changed" relative to its prior - // absence, so signal a notify. - schema_changed: true, + return Ok(( + IngestResult { + rows: tmp_result.rows, + schema: tmp_result.schema, + stats: IngestStats { + operation: tmp_result.stats.operation, + rows: tmp_result.stats.rows, + elapsed_ms: tmp_result.stats.elapsed_ms, + bytes_read: tmp_result.stats.bytes_read, + bytes_stored: tmp_result.stats.bytes_stored, + schema_inference_ms: tmp_result.stats.schema_inference_ms, + table: opts.table.clone(), + file_format: tmp_result.stats.file_format, + warning: tmp_result.stats.warning, + // Target was just created from scratch — by definition + // the resource list "changed" relative to its prior + // absence, so signal a notify. + schema_changed: true, + }, }, - }); + true, + )); } // Step 4 — schema reconciliation. Read both schemas from the @@ -448,7 +507,7 @@ where // `column_metadata_in` which falls back to the qualified // `pg_catalog.pg_attribute` probe. let target_cols = engine.column_metadata_in(opts.target_db.as_deref(), &opts.table)?; - let tmp_cols = engine.column_metadata_in(opts.target_db.as_deref(), &tmp)?; + let tmp_cols = engine.column_metadata_in(opts.target_db.as_deref(), tmp)?; // Every key must exist in both, with matching type. for k in keys { let in_target = target_cols.iter().find(|c| c.name == *k); @@ -500,9 +559,29 @@ where )); } } + // Pre-flight: a NOT NULL target column the incoming data does not + // supply cannot be satisfied by the partial-column INSERT (Hyper + // refuses it), and by then the matched rows would be deleted. + // Refuse before any DDL or DML. + let missing_required: Vec<&str> = target_cols + .iter() + .filter(|t| !t.nullable && !tmp_cols.iter().any(|c| c.name == t.name)) + .map(|t| t.name.as_str()) + .collect(); + if !missing_required.is_empty() { + return Err(McpError::new( + ErrorCode::InvalidArgument, + format!( + "merge: NOT NULL column(s) of '{}' missing from the incoming data: {}. \ + Include them, or use mode=replace.", + opts.table, + missing_required.join(", ") + ), + )); + } - // Step 5 — auto-ALTER for new columns (in temp, not in target). - // `alter_table_add_columns` handles the empty-input case by + // Step 5 — auto-ALTER for new columns (in staging, not in target). + // `alter_table_add_columns_in` handles the empty-input case by // returning early without issuing SQL. let new_cols: Vec = tmp_cols .iter() @@ -517,18 +596,10 @@ where let schema_changed = !new_cols.is_empty(); engine.alter_table_add_columns_in(opts.target_db.as_deref(), &opts.table, &new_cols)?; - // Step 6 — DELETE matching rows by key, then INSERT all temp rows. - // Both target and temp share `opts.target_db`, so qualifying via - // `qualified_table` keeps the DML scoped to one DB. + // Step 6 — DELETE matching rows by key, then INSERT all staging rows, + // in one transaction. Both tables share `opts.target_db`, so + // qualifying via `qualified_table` keeps the DML scoped to one DB. let quoted_tgt = qualified_table(opts); - let qualified_tmp_opts = IngestOptions { - table: tmp.clone(), - mode: "replace".into(), - schema_override: None, - merge_key: None, - target_db: opts.target_db.clone(), - }; - let quoted_tmp = qualified_table(&qualified_tmp_opts); let key_eq = keys .iter() .map(|k| { @@ -538,13 +609,9 @@ where .collect::>() .join(" AND "); let delete_sql = format!("DELETE FROM {quoted_tgt} t USING {quoted_tmp} s WHERE {key_eq}"); - engine.execute_command(&delete_sql)?; - - // The INSERT projection is `tmp_cols` (not `target_cols`). Columns - // the target has but the incoming temp lacks are deliberately - // omitted — PostgreSQL fills them with NULL, which is the right - // widening semantic for an upsert that adds new rows for - // unmatched keys. + // The INSERT projection is the staging columns (not the target's). + // Target-only columns are omitted — PostgreSQL fills them with NULL, + // which the pre-flight above guarantees they accept. let cols_csv = tmp_cols .iter() .map(|c| format!("\"{}\"", c.name.replace('"', "\"\""))) @@ -552,28 +619,56 @@ where .join(", "); let insert_sql = format!("INSERT INTO {quoted_tgt} ({cols_csv}) SELECT {cols_csv} FROM {quoted_tmp}"); - let inserted = engine.execute_command(&insert_sql)?; + let merged = engine.execute_in_transaction(|txn| { + txn.execute_command(&delete_sql)?; + txn.execute_command(&insert_sql) + }); + let inserted = match merged { + Ok(n) => n, + Err(e) => { + // The transaction rolled the rows back; undo the auto-committed + // ADD COLUMNs so the failed merge leaves the target's shape + // unchanged too. + for col in &new_cols { + let drop_sql = format!( + "ALTER TABLE {quoted_tgt} DROP COLUMN \"{}\"", + col.name.replace('"', "\"\"") + ); + if let Err(drop_err) = engine.execute_command(&drop_sql) { + tracing::warn!( + table = %opts.table, + column = %col.name, + error = %drop_err, + "merge: failed to drop column added by a merge that then failed" + ); + } + } + return Err(e); + } + }; // Re-read target schema for the result so callers see the post-merge shape. let final_schema = engine.column_metadata_in(opts.target_db.as_deref(), &opts.table)?; - // The guard drops the temp table when it falls out of scope here. - Ok(IngestResult { - rows: inserted, - schema: final_schema, - stats: IngestStats { - operation: tmp_result.stats.operation, + Ok(( + IngestResult { rows: inserted, - elapsed_ms: tmp_result.stats.elapsed_ms, - bytes_read: tmp_result.stats.bytes_read, - bytes_stored: tmp_result.stats.bytes_stored, - schema_inference_ms: tmp_result.stats.schema_inference_ms, - table: opts.table.clone(), - file_format: tmp_result.stats.file_format, - warning: tmp_result.stats.warning, - schema_changed, + schema: final_schema, + stats: IngestStats { + operation: tmp_result.stats.operation, + rows: inserted, + elapsed_ms: tmp_result.stats.elapsed_ms, + bytes_read: tmp_result.stats.bytes_read, + bytes_stored: tmp_result.stats.bytes_stored, + schema_inference_ms: tmp_result.stats.schema_inference_ms, + table: opts.table.clone(), + file_format: tmp_result.stats.file_format, + warning: tmp_result.stats.warning, + schema_changed, + }, }, - }) + false, + )) } /// Compare two Hyper type strings for merge compatibility. @@ -586,7 +681,7 @@ where /// [`crate::schema::map_hyper_type`] collapses all known aliases /// into a single [`hyperdb_api::SqlType`]. /// -/// If *either* side fails to parse (returns `None`), we err on the +/// If *either* side fails to parse (returns `Err`), we err on the /// side of permissive: treat the pair as compatible and let Hyper /// itself enforce at INSERT time. This avoids false-rejecting /// types the Rust mapper doesn't know about (future Hyper types, @@ -601,7 +696,7 @@ fn types_compatible(a: &str, b: &str) -> bool { crate::schema::map_hyper_type(a), crate::schema::map_hyper_type(b), ) { - (Some(sa), Some(sb)) => sa == sb, + (Ok(sa), Ok(sb)) => sa == sb, _ => true, } } @@ -627,7 +722,7 @@ pub fn ingest_json( json_str: &str, opts: &IngestOptions, ) -> Result { - if opts.mode == "merge" { + if opts.mode.is_merge() { return merge_via_temp_table(engine, opts, |engine, tmp_opts| { ingest_json(engine, json_str, tmp_opts) }); @@ -654,7 +749,7 @@ pub fn ingest_json( // All mutations run inside a transaction so a partial failure leaves // zero side effects. - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; let qualified = qualified_table(opts); let row_count = engine.execute_in_transaction(|txn| { txn.create_table_in(&opts.table, &columns, is_replace, opts.target_db.as_deref())?; @@ -729,7 +824,7 @@ pub fn ingest_csv( csv_text: &str, opts: &IngestOptions, ) -> Result { - if opts.mode == "merge" { + if opts.mode.is_merge() { return merge_via_temp_table(engine, opts, |engine, tmp_opts| { ingest_csv(engine, csv_text, tmp_opts) }); @@ -790,9 +885,9 @@ pub fn ingest_csv( hyperdb_api::escape_string_literal(canonical_temp.to_str().unwrap_or("")) ); - // Create table + COPY inside one transaction so that a COPY failure also - // unwinds the table creation. - let is_replace = opts.mode != "append"; + // Create table + COPY inside one transaction. Hyper commits DDL + // immediately, so a COPY failure rolls back the rows but not the CREATE. + let is_replace = opts.mode.replaces()?; let row_count = engine.execute_in_transaction(|txn| { txn.create_table_in(&opts.table, &columns, is_replace, opts.target_db.as_deref())?; txn.execute_command(©_sql) @@ -846,7 +941,7 @@ pub fn ingest_csv_file( path: &str, opts: &IngestOptions, ) -> Result { - if opts.mode == "merge" { + if opts.mode.is_merge() { return merge_via_temp_table(engine, opts, |engine, tmp_opts| { ingest_csv_file(engine, path, tmp_opts) }); @@ -886,7 +981,7 @@ pub fn ingest_csv_file( hyperdb_api::escape_string_literal(canonical.to_str().unwrap_or("")) ); - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; let row_count = engine.execute_in_transaction(|txn| { txn.create_table_in(&opts.table, &columns, is_replace, opts.target_db.as_deref())?; txn.execute_command(©_sql) @@ -974,7 +1069,7 @@ pub async fn ingest_csv_file_async( hyperdb_api::escape_string_literal(canonical.to_str().unwrap_or("")) ); - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; let txn = conn.transaction().await.map_err(McpError::from)?; let inner: Result = async { @@ -1121,7 +1216,7 @@ pub async fn ingest_json_async( return Err(McpError::new(ErrorCode::EmptyData, "JSON array is empty")); } - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; let qualified = qualified_table(opts); let txn = conn.transaction().await.map_err(McpError::from)?; @@ -1260,17 +1355,10 @@ pub(crate) async fn create_table_async( "No columns to create table from", )); } - for col in columns { - if crate::schema::map_hyper_type(&col.hyper_type).is_none() { - return Err(McpError::new( - ErrorCode::SchemaMismatch, - format!( - "Unknown type '{}' for column '{}'", - col.hyper_type, col.name - ), - )); - } - } + let type_sqls: Vec = columns + .iter() + .map(|col| crate::schema::column_type_sql(&col.hyper_type, &col.name)) + .collect::>()?; let quoted_table = match target_db { Some(db) => { @@ -1288,14 +1376,10 @@ pub(crate) async fn create_table_async( let col_defs: Vec = columns .iter() - .map(|c| { + .zip(&type_sqls) + .map(|(c, ty)| { let nullable = if c.nullable { "" } else { " NOT NULL" }; - format!( - "\"{}\" {}{}", - c.name.replace('"', "\"\""), - c.hyper_type, - nullable - ) + format!("\"{}\" {ty}{nullable}", c.name.replace('"', "\"\"")) }) .collect(); let create_sql = format!( diff --git a/hyperdb-mcp/src/ingest_arrow.rs b/hyperdb-mcp/src/ingest_arrow.rs index 65f21aed..c31d5a3c 100644 --- a/hyperdb-mcp/src/ingest_arrow.rs +++ b/hyperdb-mcp/src/ingest_arrow.rs @@ -221,20 +221,30 @@ fn infer_parquet_schema(path: &str) -> Result, McpError> { /// wrapped in a `::TYPE AS name` cast so the target table receives the /// narrowed/widened type. Non-overridden columns pass through by name. /// Column names are quoted to tolerate case, spaces, and reserved words. -fn parquet_select_projection(inferred: &[ColumnSchema], final_columns: &[ColumnSchema]) -> String { - inferred +/// +/// # Errors +/// +/// An overridden column whose type does not parse (or whose `NUMERIC` +/// precision/scale is out of range) is rejected here, so only the canonical +/// type text is ever spliced into the cast. +fn parquet_select_projection( + inferred: &[ColumnSchema], + final_columns: &[ColumnSchema], +) -> Result { + let parts = inferred .iter() .zip(final_columns.iter()) .map(|(orig, col)| { let quoted = format!("\"{}\"", col.name.replace('"', "\"\"")); if orig.hyper_type == col.hyper_type { - quoted + Ok(quoted) } else { - format!("{quoted}::{ty} AS {quoted}", ty = col.hyper_type) + let ty = crate::schema::column_type_sql(&col.hyper_type, &col.name)?; + Ok(format!("{quoted}::{ty} AS {quoted}")) } }) - .collect::>() - .join(", ") + .collect::, McpError>>()?; + Ok(parts.join(", ")) } /// Build the SQL statement hyperd will run to ingest the Parquet file. @@ -355,7 +365,7 @@ pub fn ingest_parquet_file( path: &str, opts: &IngestOptions, ) -> Result { - if opts.mode == "merge" { + if opts.mode.is_merge() { return crate::ingest::merge_via_temp_table(engine, opts, |engine, tmp_opts| { ingest_parquet_file(engine, path, tmp_opts) }); @@ -368,9 +378,9 @@ pub fn ingest_parquet_file( Some(s) => crate::schema::apply_schema_override(inferred.clone(), s)?, None => inferred.clone(), }; - let projection = parquet_select_projection(&inferred, &final_columns); + let projection = parquet_select_projection(&inferred, &final_columns)?; - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; let sql = build_parquet_ingest_sql( &opts.table, &absolute_path, @@ -460,8 +470,8 @@ pub async fn ingest_parquet_file_async( .await .map_err(|e| McpError::new(ErrorCode::InternalError, format!("Task join error: {e}")))??; - let projection = parquet_select_projection(&inferred, &final_columns); - let is_replace = opts.mode != "append"; + let projection = parquet_select_projection(&inferred, &final_columns)?; + let is_replace = opts.mode.replaces()?; let sql = build_parquet_ingest_sql( &opts.table, &absolute_path, @@ -657,7 +667,7 @@ pub fn ingest_arrow_ipc_file( path: &str, opts: &IngestOptions, ) -> Result { - if opts.mode == "merge" { + if opts.mode.is_merge() { return crate::ingest::merge_via_temp_table(engine, opts, |engine, tmp_opts| { ingest_arrow_ipc_file(engine, path, tmp_opts) }); @@ -676,7 +686,7 @@ pub fn ingest_arrow_ipc_file( let (columns, batches) = read_arrow_ipc_file(path)?; - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; // Arrow IPC uses the binary COPY protocol which resolves table names via // the search path. When targeting a non-primary database, temporarily // redirect the search path for the duration of the transaction. The @@ -767,7 +777,7 @@ pub async fn ingest_arrow_ipc_file_async( .await .map_err(|e| McpError::new(ErrorCode::InternalError, format!("Task join error: {e}")))??; - let is_replace = opts.mode != "append"; + let is_replace = opts.mode.replaces()?; let table_def = crate::schema::build_table_def(&opts.table, &columns)?; let txn = conn.transaction().await.map_err(McpError::from)?; @@ -884,7 +894,7 @@ mod tests { /// would look like it worked. #[test] fn null_is_not_a_resolvable_override_type() { - assert!(crate::schema::map_hyper_type("NULL").is_none()); + assert!(crate::schema::map_hyper_type("NULL").is_err()); } /// Footer inspection must reject a physically `NullType` Parquet column, diff --git a/hyperdb-mcp/src/lakehouse.rs b/hyperdb-mcp/src/lakehouse.rs index 72a35957..27ddf2d3 100644 --- a/hyperdb-mcp/src/lakehouse.rs +++ b/hyperdb-mcp/src/lakehouse.rs @@ -22,6 +22,44 @@ use crate::stats::{IngestStats, StatsTimer}; use hyperdb_api::escape_string_literal; use std::path::Path; +/// How an Iceberg ingest populates its target table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IcebergMode { + /// Drop any existing table and recreate it (`CREATE TABLE AS`). + Replace, + /// `INSERT INTO ... SELECT` into the existing table. + Append, +} + +impl IcebergMode { + /// Parse the `load_iceberg` `mode` parameter; omitted means `replace`. + /// + /// # Errors + /// + /// [`ErrorCode::InvalidArgument`] for any value other than the exact + /// lowercase `replace` / `append`. + pub fn parse(mode: Option<&str>) -> Result { + match crate::ingest::parse_mode_name( + mode, + Some("replace"), + &["replace", "append"], + "load_iceberg", + )? { + "append" => Ok(Self::Append), + _ => Ok(Self::Replace), + } + } + + /// The canonical lowercase name. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Replace => "replace", + Self::Append => "append", + } + } +} + /// Parameters for an Iceberg ingest. Mirrors `LoadFileParams` but with /// Iceberg-specific knobs and no schema-override support (hyperd derives /// the schema from the Iceberg metadata, so overrides would have no @@ -30,8 +68,8 @@ use std::path::Path; pub struct IcebergIngestOptions { /// Target table name. pub table: String, - /// `"replace"` (drops + CTAS) or `"append"` (INSERT INTO ... SELECT). - pub mode: String, + /// Replace (drops + CTAS) or append (INSERT INTO ... SELECT). + pub mode: IcebergMode, /// Optional specific metadata filename to pin a snapshot, e.g. /// `"v2.metadata.json"`. If omitted, hyperd uses whatever /// `metadata/version-hint.text` (or the latest `vN.metadata.json`) @@ -186,7 +224,10 @@ pub fn ingest_iceberg_table( let timer = StatsTimer::start(); let absolute_path = resolve_iceberg_path(path)?; - let is_replace = opts.mode != "append"; + let is_replace = match opts.mode { + IcebergMode::Replace => true, + IcebergMode::Append => false, + }; let sql = build_iceberg_ingest_sql(&opts.table, &absolute_path, opts, is_replace); // Same tx-shape as parquet ingest. The COUNT(*) *must* run outside @@ -237,7 +278,7 @@ mod tests { fn builds_replace_sql_with_minimal_options() { let opts = IcebergIngestOptions { table: "my_table".into(), - mode: "replace".into(), + mode: IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; @@ -252,7 +293,7 @@ mod tests { fn builds_append_sql_with_metadata_filename() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "append".into(), + mode: IcebergMode::Append, metadata_filename: Some("v2.metadata.json".into()), version_as_of: None, }; @@ -267,7 +308,7 @@ mod tests { fn builds_sql_with_version_as_of() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: IcebergMode::Replace, metadata_filename: None, version_as_of: Some(7), }; @@ -282,7 +323,7 @@ mod tests { fn builds_sql_with_both_metadata_and_version() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: IcebergMode::Replace, metadata_filename: Some("v3.metadata.json".into()), version_as_of: Some(42), }; @@ -297,7 +338,7 @@ mod tests { fn escapes_single_quotes_in_path() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; @@ -313,7 +354,7 @@ mod tests { fn escapes_single_quotes_in_metadata_filename() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: IcebergMode::Replace, metadata_filename: Some("v1.metadata'.json".into()), version_as_of: None, }; @@ -328,7 +369,7 @@ mod tests { fn escapes_quotes_in_table_name() { let opts = IcebergIngestOptions { table: "weird\"name".into(), - mode: "replace".into(), + mode: IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; diff --git a/hyperdb-mcp/src/lib.rs b/hyperdb-mcp/src/lib.rs index db6987e2..b0c4c37e 100644 --- a/hyperdb-mcp/src/lib.rs +++ b/hyperdb-mcp/src/lib.rs @@ -95,6 +95,7 @@ pub mod readme; pub mod saved_queries; #[doc(hidden)] pub mod schema; +pub(crate) mod scratch; #[doc(hidden)] pub mod server; mod sql_classify; diff --git a/hyperdb-mcp/src/readme.rs b/hyperdb-mcp/src/readme.rs index 5f3e56cb..567b8371 100644 --- a/hyperdb-mcp/src/readme.rs +++ b/hyperdb-mcp/src/readme.rs @@ -91,7 +91,7 @@ Every successful database-routed tool response carries canonical `resolved_datab one query, temporary table; format auto-detected (`[`/`{` → JSON, else CSV) - `query_file`(path, sql, table_name?="data", schema?, json_extract_path?) — same for a CSV|JSON|JSONL| Parquet|Arrow IPC file; fastest answer to "what's in this file?" -- `schema`: partial override `{"col": "BIGINT"}`, rest inferred · `json_extract_path`: dot path to a +- `schema`: partial override `{"col": "BIGINT"}`, rest inferred · `NUMERIC(p,s)` needs p 1..=38, s 0..=p (else INVALID_ARGUMENT) · `json_extract_path`: dot path to a nested JSON array, numeric segments index arrays (`content.0`), stringified JSON parsed en route ### Load @@ -219,13 +219,16 @@ Give precision and scale: `::numeric(20,10)`. - Persistent-file contention: only a reserved persistent attachment lock is reported as `RESOURCE_BUSY`. Run `hyperdb-mcp doctor`, compare client/daemon identities, close the possible owner (Hyper, Tableau, or another process) or copy/select another `.hyper` file, then retry. +- `mode` values are exact lowercase; anything else (`Append`, `appendd`, `""`, `merge` where unsupported) + → INVALID_ARGUMENT instead of falling back to replace. - `copy_query` mode: create (target must not exist; CREATE TABLE AS) | append (must exist; INSERT - INTO ... SELECT) | replace (drop and recreate, atomically). + INTO ... SELECT) | replace (result built in staging first; a failing SELECT leaves the target intact). - `load_file` merge (needs `merge_key`): key match → row deleted, incoming row inserted (target columns absent from the file → NULL; duplicate incoming keys all insert) · no match → INSERT · new file column → `ALTER TABLE ADD COLUMN` (nullable, existing rows NULL) · type change on an existing column - → rejected (use replace or a `schema` override). ⚠ DELETE+INSERT is not transactional (Hyper - auto-commits DDL): a mid-run failure leaves partial state, same as replace. + → rejected (use replace or a `schema` override). DELETE+INSERT runs in one transaction: a failed + merge leaves the target's rows and columns unchanged. A NOT NULL target column missing from the + incoming data is rejected up front. ## SQL dialect quick-reference diff --git a/hyperdb-mcp/src/schema.rs b/hyperdb-mcp/src/schema.rs index c741c56e..c7420f62 100644 --- a/hyperdb-mcp/src/schema.rs +++ b/hyperdb-mcp/src/schema.rs @@ -76,12 +76,7 @@ pub fn build_table_def( } let mut def = TableDefinition::new(table_name); for col in columns { - let sql_type = map_hyper_type(&col.hyper_type).ok_or_else(|| { - McpError::new( - ErrorCode::SchemaMismatch, - format!("Unknown type: {}", col.hyper_type), - ) - })?; + let sql_type = map_hyper_type(&col.hyper_type)?; if col.nullable { def = def.add_nullable_column(&col.name, sql_type); } else { @@ -94,44 +89,85 @@ pub fn build_table_def( /// Map a user-facing type name string (e.g. `"INT"`, `"NUMERIC(12,2)"`) to a /// [`hyperdb_api::SqlType`]. Accepts `PostgreSQL` aliases (`INT4`, `FLOAT8`, `VARCHAR`) /// so schema overrides from diverse sources work without normalization. -#[must_use] -pub fn map_hyper_type(type_name: &str) -> Option { +/// +/// Parsing is strict: the whole string must be one known type name, optionally +/// `NUMERIC`/`DECIMAL` followed by `(precision[, scale])`. Anything trailing +/// (`NUMERIC, x INT`, `NUMERICX`) is rejected, because the resulting type is +/// rendered into DDL by [`canonical_type_sql`]. +/// +/// # Errors +/// +/// - [`ErrorCode::InvalidArgument`] when a `NUMERIC` precision/scale is out of +/// range (precision `1..=38`, scale `0..=precision`). This is checked with the +/// non-panicking [`SqlType::try_numeric`] so untrusted input can never +/// panic while the engine mutex is held. +/// - [`ErrorCode::SchemaMismatch`] for any other unrecognised type string. +pub fn map_hyper_type(type_name: &str) -> Result { let upper = type_name.trim().to_uppercase(); match upper.as_str() { - "SMALLINT" | "INT2" => Some(SqlType::small_int()), - "INT" | "INTEGER" | "INT4" => Some(SqlType::int()), - "BIGINT" | "INT8" => Some(SqlType::big_int()), - "FLOAT" | "FLOAT4" | "REAL" => Some(SqlType::double()), - "DOUBLE" | "DOUBLE PRECISION" | "FLOAT8" => Some(SqlType::double()), - "TEXT" | "VARCHAR" | "STRING" => Some(SqlType::text()), - "BOOL" | "BOOLEAN" => Some(SqlType::bool()), - "DATE" => Some(SqlType::date()), - "TIME" => Some(SqlType::time()), - "TIMESTAMP" => Some(SqlType::timestamp()), - "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => Some(SqlType::timestamp_tz()), - "BYTEA" | "BYTES" => Some(SqlType::bytes()), - _ if upper.starts_with("NUMERIC") => { - // Parse NUMERIC(p,s) or default to NUMERIC(38,0) - if let Some(inner) = upper - .strip_prefix("NUMERIC(") - .and_then(|s| s.strip_suffix(')')) - { - let parts: Vec<&str> = inner.split(',').collect(); - let precision = parts - .first() - .and_then(|p| p.trim().parse().ok()) - .unwrap_or(38); - let scale = parts - .get(1) - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0); - Some(SqlType::numeric(precision, scale)) - } else { - Some(SqlType::numeric(38, 0)) - } - } - _ => None, + "SMALLINT" | "INT2" => Ok(SqlType::small_int()), + "INT" | "INTEGER" | "INT4" => Ok(SqlType::int()), + "BIGINT" | "INT8" => Ok(SqlType::big_int()), + "FLOAT" | "FLOAT4" | "REAL" => Ok(SqlType::double()), + "DOUBLE" | "DOUBLE PRECISION" | "FLOAT8" => Ok(SqlType::double()), + "TEXT" | "VARCHAR" | "STRING" => Ok(SqlType::text()), + "BOOL" | "BOOLEAN" => Ok(SqlType::bool()), + "DATE" => Ok(SqlType::date()), + "TIME" => Ok(SqlType::time()), + "TIMESTAMP" => Ok(SqlType::timestamp()), + "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => Ok(SqlType::timestamp_tz()), + "BYTEA" | "BYTES" => Ok(SqlType::bytes()), + _ => parse_numeric(&upper).unwrap_or_else(|| { + Err(McpError::new( + ErrorCode::SchemaMismatch, + format!("Unknown type '{}'", type_name.trim()), + )) + }), + } +} + +/// Strictly parse `NUMERIC`/`DECIMAL` with an optional `(p[,s])` suffix from an +/// already upper-cased, trimmed string. `None` means "not a numeric type +/// string at all" (so the caller reports an unknown type); `Some(Err)` is a +/// numeric spelling with bad or out-of-range arguments. +fn parse_numeric(upper: &str) -> Option> { + let rest = upper + .strip_prefix("NUMERIC") + .or_else(|| upper.strip_prefix("DECIMAL"))? + .trim_start(); + if rest.is_empty() { + return Some(Ok(SqlType::numeric(38, 0))); + } + let inner = rest.strip_prefix('(')?.strip_suffix(')')?; + let mut parts = inner.split(','); + let precision: u32 = parts.next()?.trim().parse().ok()?; + let scale: u32 = match parts.next() { + Some(s) => s.trim().parse().ok()?, + None => 0, + }; + if parts.next().is_some() { + return None; } + Some(SqlType::try_numeric(precision, scale).ok_or_else(|| { + McpError::new( + ErrorCode::InvalidArgument, + format!( + "NUMERIC precision must be 1..=38 and scale 0..=precision; got ({precision},{scale})" + ), + ) + })) +} + +/// Canonical SQL text for a user- or file-derived type name: parses it with +/// [`map_hyper_type`] and renders the resulting [`SqlType`]. This is the only +/// form that may be spliced into DDL, so no caller-controlled text ever +/// reaches a `CREATE`/`ALTER TABLE` statement. +/// +/// # Errors +/// +/// Same as [`map_hyper_type`]. +pub fn canonical_type_sql(type_name: &str) -> Result { + map_hyper_type(type_name).map(|t| t.to_string()) } // --- Tier 2: JSON Schema Inference --- @@ -602,6 +638,23 @@ fn infer_csv_field_type(field: &str) -> InferredType { infer_string_type(trimmed) } +/// Canonicalise `type_name` ([`canonical_type_sql`]) for column `column`, adding the column name to an +/// error so an LLM can self-correct. +/// +/// # Errors +/// +/// Same as [`map_hyper_type`]. +pub fn column_type_sql(type_name: &str, column: &str) -> Result { + canonical_type_sql(type_name).map_err(|mut e| { + if e.code == ErrorCode::SchemaMismatch { + e.message = format!("Unknown type '{type_name}' for column '{column}'"); + } else { + e.message = format!("column '{column}': {}", e.message); + } + e + }) +} + /// Parse a user-provided schema override (`{"column_name": "TYPE", ...}`) into /// a `Vec`. Validates each type name against [`map_hyper_type`] and /// rejects unknown types early. All override columns are marked nullable. @@ -617,7 +670,8 @@ fn infer_csv_field_type(field: &str) -> InferredType { /// - Returns [`ErrorCode::SchemaMismatch`] if any value in `schema` is /// not a string. /// - Returns [`ErrorCode::SchemaMismatch`] if any type name does not -/// resolve via [`map_hyper_type`]. +/// resolve via [`map_hyper_type`], or [`ErrorCode::InvalidArgument`] for an +/// out-of-range `NUMERIC(p,s)`. pub fn parse_schema_override( schema: &serde_json::Map, ) -> Result, McpError> { @@ -629,15 +683,10 @@ pub fn parse_schema_override( format!("Schema type for '{name}' must be a string"), ) })?; - if map_hyper_type(type_name).is_none() { - return Err(McpError::new( - ErrorCode::SchemaMismatch, - format!("Unknown type '{type_name}' for column '{name}'"), - )); - } + let hyper_type = column_type_sql(type_name, name)?; columns.push(ColumnSchema { name: name.clone(), - hyper_type: type_name.to_uppercase(), + hyper_type, nullable: true, }); } @@ -759,6 +808,7 @@ pub fn apply_schema_override( // Validate: every override key must match a real inferred column and every // override value must be a known type string. let known: std::collections::HashSet<&str> = inferred.iter().map(|c| c.name.as_str()).collect(); + let mut canonical: std::collections::HashMap<&str, String> = std::collections::HashMap::new(); for (name, type_val) in override_map { if !known.contains(name.as_str()) { let real: Vec<&str> = inferred.iter().map(|c| c.name.as_str()).collect(); @@ -773,18 +823,15 @@ pub fn apply_schema_override( format!("Schema type for '{name}' must be a string"), ) })?; - if map_hyper_type(type_name).is_none() { - return Err(McpError::new( - ErrorCode::SchemaMismatch, - format!("Unknown type '{type_name}' for column '{name}'"), - )); - } + canonical.insert(name.as_str(), column_type_sql(type_name, name)?); } // Apply: overlay types by column name. Column order is the inferred order. + // The canonical rendering (not the raw user string) is what gets stored, + // because `hyper_type` is later spliced into DDL. for col in &mut inferred { - if let Some(v) = override_map.get(&col.name).and_then(|v| v.as_str()) { - col.hyper_type = v.trim().to_uppercase(); + if let Some(ty) = canonical.remove(col.name.as_str()) { + col.hyper_type = ty; } } Ok(inferred) diff --git a/hyperdb-mcp/src/scratch.rs b/hyperdb-mcp/src/scratch.rs new file mode 100644 index 00000000..e523621e --- /dev/null +++ b/hyperdb-mcp/src/scratch.rs @@ -0,0 +1,122 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Shared plumbing for short-lived staging tables: collision-free names and +//! an RAII guard that drops the table on every scope exit. +//! +//! Used by the `load_file` merge path (`__hyperdb_merge_*`) and the +//! `copy_query` replace path (`__hyperdb_copy_*`). Both prefixes are listed +//! in `engine::INTERNAL_SCRATCH_PREFIXES`, so a staging table that is in +//! flight (or leaked by a hard crash) never shows up as user data. + +use crate::engine::Engine; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Prefix of `load_file` merge staging tables. +pub(crate) const MERGE_PREFIX: &str = "__hyperdb_merge_"; +/// Prefix of `copy_query` replace staging tables (new result and the +/// renamed-aside backup of the old target). +pub(crate) const COPY_PREFIX: &str = "__hyperdb_copy_"; + +/// Build a unique scratch-table name: `___`. +/// +/// The name mixes the PID, a nanosecond timestamp and a process-local atomic +/// counter. The counter is what makes it race-free: two parallel calls for +/// the same target inside one process can land in the same nanosecond, but +/// never share a counter value, so their guards cannot cross-drop each +/// other's tables. Anything outside `[A-Za-z0-9_]` in `target` is squashed to +/// `_` so the identifier is always SQL-safe. +pub(crate) fn scratch_name(prefix: &str, target: &str) -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let counter = COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let safe_target: String = target + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' { + c + } else { + '_' + } + }) + .collect(); + format!( + "{prefix}{safe_target}_{}_{nanos}_{counter}", + std::process::id() + ) +} + +/// RAII guard that drops a staging table on **every** scope exit — `Ok`, +/// `Err`, *and* panic-unwind — so an orphan scratch table can't leak into +/// the workspace. +/// +/// The guard owns the exclusive engine borrow so the guarded code can run +/// `&mut self` engine operations (e.g. `execute_in_transaction`) through +/// [`Self::engine`]. Disarm with [`Self::disarm`] once the table has been +/// renamed into its final role and must no longer be dropped by name. +pub(crate) struct ScratchTableGuard<'engine> { + engine: &'engine mut Engine, + /// Fully-qualified (or unqualified) quoted identifier to drop. + qualified: String, + /// Bare table name, for logging. + name: String, + armed: bool, +} + +impl<'engine> ScratchTableGuard<'engine> { + /// Arm a guard for `qualified` (an already-quoted identifier such as + /// `"db"."public"."name"` or `"name"`); `name` is only used in logs. + pub(crate) fn new(engine: &'engine mut Engine, qualified: String, name: String) -> Self { + Self { + engine, + qualified, + name, + armed: true, + } + } + + /// The guarded engine. + pub(crate) fn engine(&mut self) -> &mut Engine { + self.engine + } + + /// Stop the guard from dropping the table (it was renamed away). + pub(crate) fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for ScratchTableGuard<'_> { + fn drop(&mut self) { + if !self.armed { + return; + } + // Distinguish expected cleanup from "we wouldn't have learned about + // the leak otherwise" (panic-unwind) in the log. + let panicking = std::thread::panicking(); + let drop_sql = format!("DROP TABLE IF EXISTS {}", self.qualified); + match self.engine.execute_command(&drop_sql) { + Ok(_) => { + if panicking { + tracing::info!( + tmp = %self.name, + "scratch table dropped during panic unwind" + ); + } + } + Err(e) => { + // Don't mask the original outcome's error with a cleanup + // error — log and continue. The scratch table is benign + // (hidden by its prefix); the user can drop it manually. + tracing::warn!( + tmp = %self.name, + panicking, + error = %e, + "failed to drop scratch table on guard exit" + ); + } + } + } +} diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index 92ae0757..4a78864e 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -23,7 +23,7 @@ use crate::engine::{ use crate::error::{ErrorCode, McpError}; use crate::export::{ExportOptions, export_to_file}; use crate::ingest::{ - InferredFileFormat, IngestOptions, detect_file_format, ingest_csv, ingest_csv_file, + InferredFileFormat, IngestMode, IngestOptions, detect_file_format, ingest_csv, ingest_csv_file, ingest_csv_file_async, ingest_json, ingest_json_file, ingest_json_file_async, }; use crate::ingest_arrow::{ @@ -309,6 +309,15 @@ impl<'de> Deserialize<'de> for MergeKey { } impl MergeKey { + /// The column names as given (possibly empty); validated by + /// [`IngestMode::parse`]. + pub fn into_names(self) -> Vec { + match self { + Self::Single(s) => vec![s], + Self::Multi(v) => v, + } + } + /// Materialize as a non-empty `Vec`, or return `None` for the /// empty case so callers can convert it into an `InvalidArgument` /// error with a context-appropriate message. @@ -368,38 +377,6 @@ pub struct LoadFilesParams { pub persist: Option, } -/// Validate the (`mode`, `merge_key`) combination at the tool boundary. -/// Returns the normalized `Vec` for merge mode (or `None` for -/// replace/append). Rejects: -/// -/// - `mode = "merge"` without `merge_key` → `InvalidArgument`. -/// - `mode = "merge"` with empty / blank-element `merge_key` → -/// `InvalidArgument`. -/// - `mode != "merge"` with `merge_key` set → `InvalidArgument` -/// (catches "I added merge_key but forgot mode" mistakes loudly). -fn validate_merge_args( - mode: &str, - merge_key: Option, -) -> Result>, McpError> { - match (mode, merge_key) { - ("merge", None) => Err(McpError::new( - ErrorCode::InvalidArgument, - "mode=merge requires merge_key (a column name or list of column names)", - )), - ("merge", Some(mk)) => mk.into_vec().map(Some).ok_or_else(|| { - McpError::new( - ErrorCode::InvalidArgument, - "merge_key must be a non-empty list of non-empty column names", - ) - }), - (_, Some(_)) => Err(McpError::new( - ErrorCode::InvalidArgument, - "merge_key is only valid with mode=merge", - )), - (_, None) => Ok(None), - } -} - /// Parameters for the local-only `load_iceberg` tool. /// /// An Iceberg table on disk is a *directory* containing a `metadata/` @@ -810,7 +787,9 @@ pub struct CopyQueryParams { /// the query's result schema via `CREATE TABLE AS`. /// * `"append"` — error if the target does not exist; rows are /// appended via `INSERT INTO ... SELECT`. - /// * `"replace"` — drop (if any) and recreate, atomically. + /// * `"replace"` — build the result in a hidden staging table, then swap it in; a + /// failing SELECT leaves the existing target untouched. (Hyper cannot roll + /// back renames, so the swap itself is two back-to-back RENAMEs, not one atomic step.) pub mode: String, /// Alias of the destination database. `None` and `"local"` both /// mean the server's local database. Any other value must refer @@ -1658,8 +1637,36 @@ impl HyperMcpServer { // runs *inside* `with_engine` (not `ensure_engine`) so the // catalog SQL can see errors classified via the normal error // path. No-op in read-only mode. - self.ensure_catalog_ready(engine); - let result = f(engine); + // + // A panic inside a tool body (or the catalog bootstrap) must not + // unwind through the engine mutex (poisoning it and wedging every + // later call). Catch it, turn it into an `InternalError` reply + // and discard the engine: the connection may be mid-protocol, so + // the next call reconnects rather than trusting it. + let result = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.ensure_catalog_ready(engine); + f(engine) + })) { + Ok(result) => result, + Err(payload) => { + let detail = payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "non-string panic payload".to_string()); + tracing::error!(panic = %detail, "tool call panicked; discarding engine"); + *guard = None; + if let Ok(mut ready) = self.catalog_ready.lock() { + *ready = false; + } + Err(McpError::new( + ErrorCode::InternalError, + format!( + "internal error: tool call panicked ({detail}); the connection was reset, retry" + ), + )) + } + }; let connection_lost = result .as_ref() .is_err_and(|e| e.code == ErrorCode::ConnectionLost); @@ -1943,9 +1950,8 @@ impl HyperMcpServer { let schema_override = crate::schema::normalize_schema_param(params.schema.as_ref())?; let opts = IngestOptions { table: temp_table.clone(), - mode: "replace".into(), + mode: IngestMode::Replace, schema_override, - merge_key: None, target_db: None, }; @@ -1994,9 +2000,8 @@ impl HyperMcpServer { let schema_override = crate::schema::normalize_schema_param(params.schema.as_ref())?; let opts = IngestOptions { table: temp_table.clone(), - mode: "replace".into(), + mode: IngestMode::Replace, schema_override, - merge_key: None, target_db: None, }; @@ -2061,7 +2066,16 @@ impl HyperMcpServer { // existing one), which is a resource-list-changing event; append // mode only changes row content. Captured before the move-into // closure so we can pick the right notifications after success. - let mode = params.mode.clone().unwrap_or_else(|| "replace".into()); + let mode = match IngestMode::parse( + params.mode.as_deref(), + None, + &["replace", "append"], + "load_data", + ) { + Ok(m) => m, + Err(e) => return Self::err_content(e), + }; + let mode_name = mode.as_str(); let result = self.with_engine(|engine| { let target_db = self.resolve_db(engine, params.database.as_deref(), params.persist, true)?; @@ -2069,9 +2083,8 @@ impl HyperMcpServer { let schema_override = crate::schema::normalize_schema_param(params.schema.as_ref())?; let opts = IngestOptions { table: params.table.clone(), - mode: mode.clone(), + mode, schema_override, - merge_key: None, target_db: target_db.clone(), }; @@ -2099,7 +2112,7 @@ impl HyperMcpServer { // get their own. { let load_params = serde_json::to_string(&json!({ - "mode": mode, + "mode": mode_name, "format": fmt, "database": target_db.as_deref().unwrap_or("local"), })) @@ -2127,7 +2140,7 @@ impl HyperMcpServer { match result { Ok(val) => { self.notify_table_changed(&table_name); - if mode == "replace" { + if mode_name == "replace" { // Replace either created a new table or recreated an // existing one — either way the resource catalog // moved. @@ -2151,14 +2164,21 @@ impl HyperMcpServer { return Self::err_content(e); } let table_name = params.table.clone(); - let mode = params.mode.clone().unwrap_or_else(|| "replace".into()); - // Validate (mode, merge_key) combination at the tool boundary so the - // ingest layer can stay focused on the load mechanics. `merge_key` - // is required for merge and rejected for replace/append. - let merge_key_vec = match validate_merge_args(&mode, params.merge_key) { - Ok(v) => v, + // Parse (mode, merge_key) once at the tool boundary: only exact + // lowercase names are accepted, `merge_key` is required for merge + // and rejected otherwise, and nothing unrecognised can fall + // through to a destructive replace. + let mode = match IngestMode::parse( + params.mode.as_deref(), + params.merge_key.map(MergeKey::into_names), + &["replace", "append", "merge"], + "load_file", + ) { + Ok(m) => m, Err(e) => return Self::err_content(e), }; + let mode_name = mode.as_str(); + let merge_key_vec = mode.merge_keys().map(<[String]>::to_vec); // The closure returns `(payload, schema_changed)` so the // notify branch below can fire correctly for merges that ran // an `ALTER TABLE ADD COLUMN`. `replace` always changes shape; @@ -2172,7 +2192,6 @@ impl HyperMcpServer { table: params.table.clone(), mode: mode.clone(), schema_override, - merge_key: merge_key_vec.clone(), target_db: target_db.clone(), }; @@ -2218,7 +2237,7 @@ impl HyperMcpServer { { let load_params = serde_json::to_string(&json!({ "source_path": params.path, - "mode": mode, + "mode": mode_name, "schema": params.schema, "json_extract_path": params.json_extract_path, "merge_key": merge_key_vec, @@ -2257,7 +2276,7 @@ impl HyperMcpServer { // circuit). A merge that only updated existing rows // leaves the schema untouched, so we skip the // broadcast — same precedent as `append`. - if mode == "replace" || schema_changed { + if mode_name == "replace" || schema_changed { self.notify_resource_list_changed(); } Self::ok_content(val) @@ -2295,13 +2314,13 @@ impl HyperMcpServer { // carry the merge-via-temp-table branch, and rejecting per-entry // would produce a confusing N-rejection result for a uniform // merge call. One top-level error is the clearer contract. + let mut entry_modes: Vec = Vec::with_capacity(params.files.len()); for (idx, entry) in params.files.iter().enumerate() { if let Err(mut e) = crate::attach::validate_input_path(&entry.path, "data file") { e.message = format!("entry {idx} (table '{}'): {}", entry.table, e.message); return Self::err_content(e); } - let mode = entry.mode.as_deref().unwrap_or("replace"); - if mode == "merge" || entry.merge_key.is_some() { + if entry.merge_key.is_some() { return Self::err_content(McpError::new( ErrorCode::InvalidArgument, format!( @@ -2311,6 +2330,18 @@ impl HyperMcpServer { ), )); } + match IngestMode::parse( + entry.mode.as_deref(), + None, + &["replace", "append"], + "load_files", + ) { + Ok(m) => entry_modes.push(m), + Err(mut e) => { + e.message = format!("entry {idx} (table '{}'): {}", entry.table, e.message); + return Self::err_content(e); + } + } } // Resolve hyperd endpoint + the workspace path matching the @@ -2396,12 +2427,11 @@ impl HyperMcpServer { let outcomes: Vec = tokio::task::block_in_place(|| { rt.block_on(async { let mut set = tokio::task::JoinSet::new(); - for (idx, entry) in params.files.into_iter().enumerate() { + for ((idx, entry), mode) in params.files.into_iter().enumerate().zip(entry_modes) { let pool = Arc::clone(&pool); let entry_target_db = target_db.clone(); set.spawn(async move { - let mode = entry.mode.clone().unwrap_or_else(|| "replace".into()); - let replace_mode = mode == "replace"; + let replace_mode = mode == IngestMode::Replace; let mut out = EntryOutcome { table: entry.table.clone(), replace_mode, @@ -2431,9 +2461,8 @@ impl HyperMcpServer { let _ = &entry_target_db; let opts = IngestOptions { table: entry.table.clone(), - mode: mode.clone(), + mode, schema_override, - merge_key: None, target_db: None, }; @@ -2646,10 +2675,13 @@ impl HyperMcpServer { return Self::err_content(e); } let table_name = params.table.clone(); - let mode = params.mode.clone().unwrap_or_else(|| "replace".into()); + let mode = match crate::lakehouse::IcebergMode::parse(params.mode.as_deref()) { + Ok(m) => m, + Err(e) => return Self::err_content(e), + }; let opts = crate::lakehouse::IcebergIngestOptions { table: params.table.clone(), - mode: mode.clone(), + mode, metadata_filename: params.metadata_filename.clone(), version_as_of: params.version_as_of, }; @@ -2674,7 +2706,7 @@ impl HyperMcpServer { let load_params = serde_json::to_string(&json!({ "source_path": params.path, - "mode": mode, + "mode": mode.as_str(), "format": "iceberg", "metadata_filename": params.metadata_filename, "version_as_of": params.version_as_of, @@ -2699,7 +2731,7 @@ impl HyperMcpServer { match result { Ok(val) => { self.notify_table_changed(&table_name); - if mode == "replace" { + if mode == crate::lakehouse::IcebergMode::Replace { self.notify_resource_list_changed(); } Self::ok_content(val) @@ -4137,16 +4169,9 @@ impl HyperMcpServer { if let Err(e) = self.check_writable("copy_query") { return Self::err_content(e); } - let mode = match params.mode.as_str() { - "create" | "append" | "replace" => params.mode.clone(), - other => { - return Self::err_content(McpError::new( - ErrorCode::InvalidArgument, - format!( - "copy_query mode '{other}' is not supported. Use 'create', 'append', or 'replace'." - ), - )); - } + let mode = match CopyMode::parse(¶ms.mode) { + Ok(m) => m, + Err(e) => return Self::err_content(e), }; if !is_read_only_sql(¶ms.sql) { return Self::err_content(McpError::new( @@ -4205,7 +4230,7 @@ impl HyperMcpServer { let target_table = params.target_table.clone(); let sql_body = params.sql.clone(); let load_params = serde_json::to_string(&json!({ - "mode": mode, + "mode": mode.as_str(), "target_database": params.target_database, "target_table": target_table, "sql": Self::fmt_sql(&sql_body), @@ -4231,7 +4256,7 @@ impl HyperMcpServer { // Phase 2: run the actual copy inside a helper so the // cleanup path is unified. - let copy_outcome = perform_copy(engine, &mode, target_db, &target_table, &sql_body); + let copy_outcome = perform_copy(engine, mode, target_db, &target_table, &sql_body); // Phase 3: always detach the temp attachments, even on // error — they were installed only for the duration of @@ -4281,7 +4306,7 @@ impl HyperMcpServer { self.notify_table_changed(&target_table); } self.notify_workspace_changed(); - if mode != "append" { + if mode != CopyMode::Append { // `create` / `replace` add or recreate the table, // which is a resource-list-changing event. self.notify_resource_list_changed(); @@ -5599,13 +5624,146 @@ fn prepare_temp_attachments( Ok(out) } +/// The `copy_query` landing mode. Parsed once at the tool boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CopyMode { + /// Target must not exist. + Create, + /// Target must exist; rows are inserted. + Append, + /// Target is replaced by the query result. + Replace, +} + +impl CopyMode { + /// Parse the (required) `mode` parameter; exact lowercase names only. + fn parse(value: &str) -> Result { + match crate::ingest::parse_mode_name( + Some(value), + None, + &["create", "append", "replace"], + "copy_query", + )? { + "create" => Ok(Self::Create), + "append" => Ok(Self::Append), + "replace" => Ok(Self::Replace), + other => unreachable!("parse_mode_name returned a mode outside the known set: {other}"), + } + } + + const fn as_str(self) -> &'static str { + match self { + Self::Create => "create", + Self::Append => "append", + Self::Replace => "replace", + } + } +} + +/// `CREATE TABLE AS ` for a target known to be absent. +/// +/// Hyper can fail a CTAS at run time (e.g. division by zero, 22012) *after* +/// it created the table, leaving an empty relation behind. The target did not +/// exist before this call, so on failure it is dropped again (best effort) +/// and the original error is returned. +fn create_table_as(engine: &Engine, qualified: &str, sql_body: &str) -> Result<(), McpError> { + if let Err(e) = engine.execute_command(&format!("CREATE TABLE {qualified} AS {sql_body}")) { + // 42P07 = duplicate_table: the target already existed (e.g. created + // concurrently), so it is not ours to drop. + if e.message.starts_with("server error (42P07)") { + return Err(e); + } + if let Err(cleanup) = engine.execute_command(&format!("DROP TABLE IF EXISTS {qualified}")) { + tracing::warn!( + table = %qualified, + error = %cleanup, + "failed to drop partially created copy_query target" + ); + } + return Err(e); + } + Ok(()) +} + +/// Replace an existing target with the result of `sql_body` without ever +/// dropping the old data until the new result is fully built. +/// +/// Hyper renames are auto-committed and cannot be rolled back, so a truly +/// atomic swap is impossible; the sequence below keeps the window in which +/// the target name is absent to two back-to-back RENAMEs, and a failure of the +/// SELECT (the realistic failure) leaves the target untouched: +/// +/// 1. build the result into a hidden staging table (`__hyperdb_copy_*`); +/// 2. rename the old target aside to a hidden backup; +/// 3. rename staging to the target name (on failure, rename the backup back); +/// 4. drop the backup. +fn replace_via_staging( + engine: &mut Engine, + target_db: Option<&str>, + target_table: &str, + sql_body: &str, +) -> Result<(), McpError> { + use crate::scratch::{COPY_PREFIX, ScratchTableGuard, scratch_name}; + + let qualified = qualified_name(engine, target_db, target_table); + let staging_name = scratch_name(COPY_PREFIX, target_table); + let backup_name = scratch_name(&format!("{COPY_PREFIX}bak_"), target_table); + let staging_q = qualified_name(engine, target_db, &staging_name); + let backup_q = qualified_name(engine, target_db, &backup_name); + let quote = |n: &str| format!("\"{}\"", n.replace('"', "\"\"")); + + // Step 1: the guard drops staging on every exit until it is renamed away. + let mut guard = ScratchTableGuard::new(engine, staging_q.clone(), staging_name.clone()); + guard + .engine() + .execute_command(&format!("CREATE TABLE {staging_q} AS {sql_body}"))?; + + // Step 2: move the old target aside. + guard.engine().execute_command(&format!( + "ALTER TABLE {qualified} RENAME TO {}", + quote(&backup_name) + ))?; + + // Step 3: move staging into place; restore the backup if that fails. + if let Err(swap_err) = guard.engine().execute_command(&format!( + "ALTER TABLE {staging_q} RENAME TO {}", + quote(target_table) + )) { + return match guard.engine().execute_command(&format!( + "ALTER TABLE {backup_q} RENAME TO {}", + quote(target_table) + )) { + Ok(_) => Err(swap_err), + // Keep the original error code (e.g. ConnectionLost) so the + // caller's reconnect logic still fires; add the backup hint. + Err(restore_err) => Err(McpError::new( + swap_err.code, + format!( + "copy_query replace failed ({swap_err}) and the original table could not be restored ({restore_err}); \ + your data is preserved in hidden table '{backup_name}'" + ), + )), + }; + } + guard.disarm(); + + // Step 4: the old data is no longer needed. + if let Err(e) = guard + .engine() + .execute_command(&format!("DROP TABLE IF EXISTS {backup_q}")) + { + tracing::warn!(backup = %backup_name, error = %e, "failed to drop copy_query backup table"); + } + Ok(()) +} + /// Execute the chosen copy mode against the fully-qualified target /// and return a JSON summary. Extracted from the `copy_query` handler /// so the caller can run it inside the temp-attach cleanup wrapper /// without re-duplicating the match arms. fn perform_copy( - engine: &Engine, - mode: &str, + engine: &mut Engine, + mode: CopyMode, target_db: Option<&str>, target_table: &str, sql_body: &str, @@ -5615,45 +5773,34 @@ fn perform_copy( let timer = crate::stats::StatsTimer::start(); match mode { - "create" => { + CopyMode::Create => { if exists { return Err(McpError::new( ErrorCode::InvalidArgument, format!( - "Target '{target_table}' already exists. Use mode='append' to add rows or mode='replace' to drop and recreate." + "Target '{target_table}' already exists. Use mode='append' to add rows or mode='replace' to replace it." ), )); } - engine.execute_command(&format!("CREATE TABLE {qualified} AS {sql_body}"))?; + create_table_as(engine, &qualified, sql_body)?; } - "append" => { + CopyMode::Append => { if !exists { return Err(McpError::new( ErrorCode::InvalidArgument, format!( - "Target '{target_table}' does not exist. Use mode='create' to create it from the query or mode='replace' to drop and recreate." + "Target '{target_table}' does not exist. Use mode='create' to create it from the query or mode='replace' to replace it." ), )); } engine.execute_command(&format!("INSERT INTO {qualified} {sql_body}"))?; } - "replace" => { - // Hyper auto-commits DDL even inside transactions, so - // DROP+CREATE isn't atomic across the statement boundary - // (same caveat documented on `execute_in_transaction`). - // We still issue them in order — the `IF EXISTS` guard - // prevents an error when the target is absent, and the - // follow-up `CREATE TABLE AS` either succeeds or leaves - // the workspace with a dropped target, which is the - // expected replace semantics. - engine.execute_command(&format!("DROP TABLE IF EXISTS {qualified}"))?; - engine.execute_command(&format!("CREATE TABLE {qualified} AS {sql_body}"))?; - } - other => { - return Err(McpError::new( - ErrorCode::InvalidArgument, - format!("copy_query mode '{other}' is not supported"), - )); + CopyMode::Replace => { + if exists { + replace_via_staging(engine, target_db, target_table, sql_body)?; + } else { + create_table_as(engine, &qualified, sql_body)?; + } } } @@ -5662,7 +5809,7 @@ fn perform_copy( Ok(json!({ "target_table": target_table, "target_database": target_db.unwrap_or(LOCAL_ALIAS), - "mode": mode, + "mode": mode.as_str(), "row_count": row_count, "stats": { "operation": "copy_query", "elapsed_ms": elapsed_ms }, })) @@ -6139,27 +6286,26 @@ mod engine_mutex_poison_tests { // First call: warm the engine, then panic inside the `with_engine` // closure to simulate a bug in a tool handler (e.g. an `unwrap()` on - // `None`). `catch_unwind` at the `std::panic` boundary lets the test - // observe the aftermath without aborting the whole test binary — the - // real MCP server survives this the same way, because each tool call - // already runs on its own task. - let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - server.with_engine(|engine| -> Result<(), McpError> { - engine.execute_command("CREATE TABLE poison_probe (v INT)")?; - panic!("simulated tool handler bug"); - }) - })); + // `None`). `with_engine` catches the unwind itself: the caller gets an + // `InternalError` reply, the engine is discarded (its connection may be + // mid-protocol) and the mutex is never poisoned. + let outcome = server.with_engine(|engine| -> Result<(), McpError> { + engine.execute_command("CREATE TABLE poison_probe (v INT)")?; + panic!("simulated tool handler bug"); + }); + let err = outcome.expect_err("a panicking tool body must become an error reply"); + assert_eq!(err.code, ErrorCode::InternalError); + assert!( + err.message.contains("simulated tool handler bug"), + "{}", + err.message + ); assert!( - outcome.is_err(), - "panic should propagate out of with_engine" + !server.engine.is_poisoned(), + "with_engine must contain the panic before it poisons the engine mutex" ); - // Second call: without the fix, the panic above dropped the - // `MutexGuard` mid-unwind and poisoned `self.engine`, so - // `ensure_engine`'s `.lock()` returns `Err` forever after — every - // subsequent tool call fails with "Lock poisoned" and the server is - // bricked until the process restarts. With the fix, the poisoned - // engine is discarded and rebuilt transparently. + // Second call: the server keeps answering on a rebuilt engine. let rows = server .with_engine(|engine| engine.execute_query_to_json("SELECT 1 AS one")) .expect("server must recover from a panicking tool call, not stay poisoned forever"); @@ -6254,16 +6400,14 @@ mod engine_mutex_poison_tests { fn status_recovers_a_poisoned_engine_rather_than_reporting_it_busy() { let server = test_server(); + // `with_engine` now contains tool-body panics, so poison the engine + // mutex the way any other panic while it is held would: unwind with + // its guard alive. let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - server.with_engine(|engine| -> Result<(), McpError> { - engine.execute_command("CREATE TABLE poison_probe (v INT)")?; - panic!("simulated tool handler bug"); - }) + let _engine = server.engine.lock().expect("engine mutex is fresh"); + panic!("simulated panic while holding the engine mutex"); })); - assert!( - outcome.is_err(), - "panic should propagate out of with_engine" - ); + assert!(outcome.is_err(), "panic should propagate"); assert!( server.engine.is_poisoned(), "the panic must have poisoned the engine mutex" diff --git a/hyperdb-mcp/src/watcher.rs b/hyperdb-mcp/src/watcher.rs index 8ce6e03d..33e7ba84 100644 --- a/hyperdb-mcp/src/watcher.rs +++ b/hyperdb-mcp/src/watcher.rs @@ -765,9 +765,8 @@ async fn ingest_one_ready_file( // identifier routes into the pool's primary. let opts = IngestOptions { table: table.to_string(), - mode: "append".into(), + mode: crate::ingest::IngestMode::Append, schema_override: None, - merge_key: None, target_db: None, }; let data_str = data_path diff --git a/hyperdb-mcp/tests/daemon_tests.rs b/hyperdb-mcp/tests/daemon_tests.rs index 404f4239..782f5cb0 100644 --- a/hyperdb-mcp/tests/daemon_tests.rs +++ b/hyperdb-mcp/tests/daemon_tests.rs @@ -685,12 +685,22 @@ fn run_bounded_engine_report_child() { fn run_engine_report_child() { let state_dir = discovery::state_dir().expect("child state directory"); + // An IPC endpoint of the shape a real daemon publishes (the engine refuses + // anything else before dialing), but nothing listens there. That makes the + // public Engine constructor enter the daemon connection-error branch + // without racing another process for a recently released port. + let unreachable_hyperd = if cfg!(windows) { + r"\\.\pipe\hyperdb-mcp-test-nobody-home".to_string() + } else { + state_dir + .join("sockets") + .join("hyper") + .to_string_lossy() + .into_owned() + }; let mut health = RunningHealth::start_with_info(&state_dir, |endpoint| DaemonInfo { pid: 616_161, - // Port zero is never a reachable TCP server endpoint. It makes the - // public Engine constructor enter the daemon connection-error branch - // without racing another process for a recently released port. - hyperd_endpoint: "127.0.0.1:0".to_string(), + hyperd_endpoint: unreachable_hyperd.clone(), health_endpoint: endpoint.as_str().to_string(), started_at: "2026-08-14T12:34:56Z".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), @@ -712,9 +722,9 @@ fn run_engine_report_child() { let restart_requested = health.state.consume_restart_request(); health.stop_and_join(); assert!( - error - .message - .contains("Failed to connect to daemon hyperd at 127.0.0.1:0"), + error.message.contains(&format!( + "Failed to connect to daemon hyperd at {unreachable_hyperd}" + )), "public Engine constructor must fail in try_daemon_mode, got: {}", error.message ); diff --git a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs index 7eabd814..537b64a4 100644 --- a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs +++ b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs @@ -4266,3 +4266,513 @@ async fn export_and_chart_refuse_an_existing_file_by_default() -> TestResult { ); Ok(()) } + +/// Row count of `table` via the `query` tool. +async fn count_rows_via_tool( + h: &TestHarness, + table: &str, +) -> Result> { + let q = call_tool( + &h.client, + "query", + serde_json::json!({ "sql": format!("SELECT COUNT(*) AS n FROM \"{table}\"") }), + ) + .await?; + let body = all_text(&q); + let start = body + .find("\"n\":") + .ok_or_else(|| format!("no count in {body}"))? + + 4; + let digits: String = body[start..] + .trim_start() + .chars() + .take_while(char::is_ascii_digit) + .collect(); + Ok(digits.parse()?) +} + +/// Regression (D3): `mode: "Append"` used to fall through to replace and +/// silently wipe the table. It must be rejected and leave the data alone. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_data_rejects_miscased_mode_without_data_loss() -> TestResult { + let h = TestHarness::start(false, true).await?; + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ "table": "m", "data": r#"[{"id":1},{"id":2}]"#, "format": "json" }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + + for bad in ["Append", "appendd", "", "merge"] { + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ "table": "m", "data": r#"[{"id":3}]"#, "format": "json", "mode": bad }), + ) + .await?; + assert!( + is_error(&r), + "mode {bad:?} must be rejected: {:?}", + first_text(&r) + ); + assert!( + all_text(&r).contains("INVALID_ARGUMENT"), + "{}", + all_text(&r) + ); + assert_eq!( + count_rows_via_tool(&h, "m").await?, + 2, + "mode {bad:?} lost data" + ); + } + h.shutdown().await +} + +/// Regression (D3): `load_file` rejects a mis-cased mode before touching the table. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_file_rejects_miscased_mode_without_data_loss() -> TestResult { + let h = TestHarness::start(false, true).await?; + let temp = TempDir::new()?; + let csv = temp.path().join("a.csv"); + std::fs::write(&csv, b"id\n1\n2\n")?; + let r = call_tool( + &h.client, + "load_file", + serde_json::json!({ "path": csv.to_string_lossy(), "table": "f" }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + let r = call_tool( + &h.client, + "load_file", + serde_json::json!({ "path": csv.to_string_lossy(), "table": "f", "mode": "APPEND" }), + ) + .await?; + assert!( + is_error(&r), + "APPEND must be rejected: {:?}", + first_text(&r) + ); + assert_eq!(count_rows_via_tool(&h, "f").await?, 2); + h.shutdown().await +} + +/// Regression (D3): a `load_files` entry with a bad mode fails the whole call +/// up front (top-level error) and nothing is ingested. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_files_rejects_bad_entry_mode() -> TestResult { + let h = TestHarness::start(false, true).await?; + let temp = TempDir::new()?; + let csv = temp.path().join("a.csv"); + std::fs::write(&csv, b"id\n1\n2\n")?; + let r = call_tool( + &h.client, + "load_files", + serde_json::json!({ "files": [ + { "path": csv.to_string_lossy(), "table": "ok_t" }, + { "path": csv.to_string_lossy(), "table": "bad_t", "mode": "appendd" }, + ]}), + ) + .await?; + assert!( + is_error(&r), + "bad entry mode must be a top-level error: {:?}", + first_text(&r) + ); + assert!(all_text(&r).contains("appendd"), "{}", all_text(&r)); + h.shutdown().await +} + +/// Regression (D3): `load_iceberg` with `merge` is rejected and no table is dropped. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_iceberg_rejects_merge_mode_without_dropping() -> TestResult { + let h = TestHarness::start(false, true).await?; + let temp = TempDir::new()?; + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ "table": "ice", "data": r#"[{"id":1},{"id":2}]"#, "format": "json" }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + let r = call_tool( + &h.client, + "load_iceberg", + serde_json::json!({ "table": "ice", "path": temp.path().to_string_lossy(), "mode": "merge" }), + ) + .await?; + assert!(is_error(&r), "merge must be rejected: {:?}", first_text(&r)); + assert!( + all_text(&r).contains("INVALID_ARGUMENT"), + "{}", + all_text(&r) + ); + assert_eq!(count_rows_via_tool(&h, "ice").await?, 2); + h.shutdown().await +} + +/// Pin: `copy_query` stays strict about mode casing (target untouched). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_copy_query_rejects_miscased_mode() -> TestResult { + let h = TestHarness::start(false, true).await?; + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ "table": "cq", "data": r#"[{"id":1},{"id":2}]"#, "format": "json" }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "Replace", "target_table": "cq", "sql": "SELECT 9 AS id" }), + ) + .await?; + assert!( + is_error(&r), + "Replace must be rejected: {:?}", + first_text(&r) + ); + assert!( + all_text(&r).contains("INVALID_ARGUMENT"), + "{}", + all_text(&r) + ); + assert_eq!(count_rows_via_tool(&h, "cq").await?, 2); + h.shutdown().await +} + +/// Regression (D1): a `load_file` merge whose INSERT fails (NULL into a +/// NOT NULL column) must report the error and leave the target's rows and +/// columns exactly as they were. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_file_failed_merge_leaves_target_untouched() -> TestResult { + let h = TestHarness::start(false, true).await?; + let temp = TempDir::new()?; + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ "table": "mg", "format": "json", + "data": r#"[{"id":1,"name":"a"},{"id":2,"name":"b"}]"# }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + + let upd = temp.path().join("upd.json"); + std::fs::write( + &upd, + br#"[{"id":1,"name":null,"extra":5},{"id":3,"name":"c"}]"#, + )?; + let r = call_tool( + &h.client, + "load_file", + serde_json::json!({ "path": upd.to_string_lossy(), "table": "mg", + "mode": "merge", "merge_key": "id" }), + ) + .await?; + assert!(is_error(&r), "merge must fail: {:?}", first_text(&r)); + + let q = call_tool( + &h.client, + "query", + serde_json::json!({ "sql": "SELECT * FROM mg ORDER BY id" }), + ) + .await?; + let body = all_text(&q); + assert!( + body.contains("\"name\":\"a\"") || body.contains("\"name\": \"a\""), + "{body}" + ); + assert!( + body.contains("\"name\":\"b\"") || body.contains("\"name\": \"b\""), + "{body}" + ); + assert!( + !body.contains("extra"), + "failed merge left a column behind: {body}" + ); + assert_eq!(count_rows_via_tool(&h, "mg").await?, 2); + h.shutdown().await +} + +// ===================================================================== +// copy_query mode=replace builds the result in staging first (D2): a +// failing SELECT no longer destroys the target. +// ===================================================================== + +/// Run one DDL/DML statement through the `execute` tool. +async fn exec_ok(h: &TestHarness, sql: &str) -> TestResult { + let r = call_tool(&h.client, "execute", serde_json::json!({ "sql": [sql] })).await?; + assert!(!is_error(&r), "{sql} failed: {:?}", first_text(&r)); + Ok(()) +} + +/// Names of `__hyperdb_copy_*` tables in `db`'s public schema (`None` = primary). +fn leaked_copy_tables(h: &TestHarness, db: Option<&str>) -> Vec { + let guard = h.engine_handle.lock().unwrap(); + let engine = guard.as_ref().expect("engine present"); + let from = match db { + Some(d) => format!("\"{d}\".pg_catalog.pg_tables"), + None => "pg_catalog.pg_tables".to_string(), + }; + engine + .execute_query_to_json(&format!( + "SELECT tablename FROM {from} WHERE schemaname = 'public'" + )) + .unwrap() + .iter() + .filter_map(|v| v["tablename"].as_str().map(str::to_string)) + .filter(|n| n.starts_with("__hyperdb_copy_")) + .collect() +} + +async fn seed_r(h: &TestHarness, prefix: &str) -> TestResult { + exec_ok(h, &format!("CREATE TABLE {prefix}r (id INT)")).await?; + exec_ok(h, &format!("INSERT INTO {prefix}r VALUES (1),(2),(3)")).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_copy_query_replace_failing_select_keeps_target() -> TestResult { + let h = TestHarness::start(false, true).await?; + seed_r(&h, "").await?; + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "replace", "target_table": "r", + "sql": "SELECT id / (id - id) AS q FROM r" }), + ) + .await?; + assert!( + is_error(&r), + "division by zero must fail: {:?}", + first_text(&r) + ); + assert_eq!( + count_rows_via_tool(&h, "r").await?, + 3, + "target was destroyed" + ); + let q = call_tool( + &h.client, + "query", + serde_json::json!({ "sql": "SELECT id FROM r" }), + ) + .await?; + assert!( + !is_error(&q), + "column id must still exist: {:?}", + first_text(&q) + ); + assert_eq!(leaked_copy_tables(&h, None), Vec::::new()); + h.shutdown().await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_copy_query_replace_may_read_the_target() -> TestResult { + let h = TestHarness::start(false, true).await?; + seed_r(&h, "").await?; + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "replace", "target_table": "r", + "sql": "SELECT id FROM r WHERE id > 1" }), + ) + .await?; + assert!( + !is_error(&r), + "self-referencing replace failed: {:?}", + first_text(&r) + ); + assert_eq!(count_rows_via_tool(&h, "r").await?, 2); + assert_eq!(leaked_copy_tables(&h, None), Vec::::new()); + h.shutdown().await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_copy_query_replace_on_missing_target_creates_it() -> TestResult { + let h = TestHarness::start(false, true).await?; + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "replace", "target_table": "fresh", + "sql": "SELECT 1 AS id UNION ALL SELECT 2" }), + ) + .await?; + assert!(!is_error(&r), "replace-create failed: {:?}", first_text(&r)); + assert_eq!(count_rows_via_tool(&h, "fresh").await?, 2); + assert_eq!(leaked_copy_tables(&h, None), Vec::::new()); + h.shutdown().await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_copy_query_create_failing_select_leaves_no_table() -> TestResult { + let h = TestHarness::start(false, true).await?; + seed_r(&h, "").await?; + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "create", "target_table": "nope", + "sql": "SELECT id / (id - id) AS q FROM r" }), + ) + .await?; + assert!(is_error(&r), "{:?}", first_text(&r)); + let exists = { + let guard = h.engine_handle.lock().unwrap(); + guard.as_ref().unwrap().table_exists("nope")? + }; + assert!(!exists, "failed create must not leave a table"); + assert_eq!(leaked_copy_tables(&h, None), Vec::::new()); + h.shutdown().await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_copy_query_replace_in_attached_database() -> TestResult { + let h = TestHarness::start(false, false).await?; + let dir = TempDir::new()?; + let r = call_tool( + &h.client, + "attach_database", + serde_json::json!({ "alias": "dst", "kind": "local_file", + "path": dir.path().join("dst.hyper").to_string_lossy(), + "writable": true, "on_missing": "create" }), + ) + .await?; + assert!(!is_error(&r), "attach failed: {:?}", first_text(&r)); + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "create", "target_database": "dst", "target_table": "r", + "sql": "SELECT 1 AS id UNION ALL SELECT 2 UNION ALL SELECT 3" }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + + // A failing SELECT keeps the target. + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "replace", "target_database": "dst", "target_table": "r", + "sql": "SELECT id / (id - id) AS q FROM \"dst\".\"public\".\"r\"" }), + ) + .await?; + assert!(is_error(&r), "{:?}", first_text(&r)); + let count = |h: &TestHarness| { + let guard = h.engine_handle.lock().unwrap(); + let rows = guard + .as_ref() + .unwrap() + .execute_query_to_json("SELECT COUNT(*) AS n FROM \"dst\".\"public\".\"r\"") + .unwrap(); + rows[0]["n"].as_i64().unwrap() + }; + assert_eq!(count(&h), 3, "attached target was destroyed"); + + // A self-referencing replace succeeds. + let r = call_tool( + &h.client, + "copy_query", + serde_json::json!({ "mode": "replace", "target_database": "dst", "target_table": "r", + "sql": "SELECT id FROM \"dst\".\"public\".\"r\" WHERE id > 1" }), + ) + .await?; + assert!(!is_error(&r), "{:?}", first_text(&r)); + assert_eq!(count(&h), 2); + assert_eq!(leaked_copy_tables(&h, Some("dst")), Vec::::new()); + h.shutdown().await +} + +// --- P1: out-of-range NUMERIC must not panic under the engine mutex --------- + +/// Regression (P1): a `NUMERIC(40,2)` schema override used to panic inside +/// `SqlType::numeric` while the engine mutex was held, poisoning it and +/// killing every later tool call. It must be a clean INVALID_ARGUMENT, the +/// existing table must survive, and the server must keep answering. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_data_out_of_range_numeric_override_is_clean_error() -> TestResult { + let h = TestHarness::start(false, true).await?; + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ "table": "t1", "data": r#"[{"v":1}]"#, "format": "json" }), + ) + .await?; + assert!(!is_error(&r), "seed failed: {:?}", first_text(&r)); + + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ + "table": "t2", "data": r#"[{"v":1}]"#, "format": "json", + "schema": { "v": "NUMERIC(40,2)" } + }), + ) + .await?; + assert!(is_error(&r), "must fail: {:?}", first_text(&r)); + assert!( + all_text(&r).contains("INVALID_ARGUMENT"), + "{}", + all_text(&r) + ); + + // Same table name as the seed, replace mode: the old table must survive. + let r = call_tool( + &h.client, + "load_data", + serde_json::json!({ + "table": "t1", "data": r#"[{"v":1}]"#, "format": "json", + "schema": { "v": "NUMERIC(40,2)" } + }), + ) + .await?; + assert!(is_error(&r)); + assert_eq!(count_rows_via_tool(&h, "t1").await?, 1); + h.shutdown().await +} + +/// Regression (P1): an Arrow IPC file with a `Decimal256(50,2)` column (no +/// override involved) must be rejected, not panic. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tool_load_file_arrow_decimal256_out_of_range_is_clean_error() -> TestResult { + use arrow::array::{Array, Decimal256Array}; + use arrow::datatypes::{DataType, Field, Schema, i256}; + use arrow::ipc::writer::FileWriter; + use arrow::record_batch::RecordBatch; + + let h = TestHarness::start(false, true).await?; + let temp = TempDir::new()?; + let path = temp.path().join("wide.arrow"); + let schema = Arc::new(Schema::new(vec![Field::new( + "amount", + DataType::Decimal256(50, 2), + false, + )])); + let col = + Decimal256Array::from(vec![i256::from_i128(12345)]).with_precision_and_scale(50, 2)?; + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(col) as Arc])?; + let mut writer = FileWriter::try_new(std::fs::File::create(&path)?, &schema)?; + writer.write(&batch)?; + writer.finish()?; + + let r = call_tool( + &h.client, + "load_file", + serde_json::json!({ "path": path.to_string_lossy(), "table": "wide" }), + ) + .await?; + assert!(is_error(&r), "must fail: {:?}", first_text(&r)); + // The server survives and still answers a query. + let r = call_tool( + &h.client, + "query", + serde_json::json!({ "sql": "SELECT 1 AS one" }), + ) + .await?; + assert!( + !is_error(&r), + "server dead after error: {:?}", + first_text(&r) + ); + h.shutdown().await +} diff --git a/hyperdb-mcp/tests/export_tests.rs b/hyperdb-mcp/tests/export_tests.rs index ea7d69f8..2256dada 100644 --- a/hyperdb-mcp/tests/export_tests.rs +++ b/hyperdb-mcp/tests/export_tests.rs @@ -477,7 +477,7 @@ fn iceberg_export_round_trips_through_load_iceberg() { // Reload into a fresh table and verify rows. let ingest_opts = IcebergIngestOptions { table: "test_reloaded".into(), - mode: "replace".into(), + mode: hyperdb_mcp::lakehouse::IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; @@ -581,7 +581,7 @@ fn iceberg_export_overwrite_replaces_directory() { iceberg_str, &IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::lakehouse::IcebergMode::Replace, metadata_filename: None, version_as_of: None, }, @@ -692,9 +692,8 @@ fn parquet_export_round_trips_through_load_file() { path_str, &IngestOptions { table: "pq_export_reloaded".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }, ) @@ -790,9 +789,8 @@ fn arrow_ipc_export_round_trips_through_load_file() { path_str, &IngestOptions { table: "arrow_reloaded".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }, ) diff --git a/hyperdb-mcp/tests/ingest_arrow_tests.rs b/hyperdb-mcp/tests/ingest_arrow_tests.rs index 054571dc..bfe43fa7 100644 --- a/hyperdb-mcp/tests/ingest_arrow_tests.rs +++ b/hyperdb-mcp/tests/ingest_arrow_tests.rs @@ -115,9 +115,8 @@ fn ingest_parquet() { let opts = IngestOptions { table: "pq_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_parquet_file(&mut te.engine, path_str, &opts).unwrap(); @@ -182,9 +181,8 @@ fn ingest_parquet_decimal128_not_null_preserves_values() { let opts = IngestOptions { table: "decimal_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_parquet_file(&mut te.engine, path_str, &opts).unwrap(); @@ -246,9 +244,8 @@ fn ingest_parquet_reports_accurate_row_count_above_131072() { let opts = IngestOptions { table: "big_table".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_parquet_file(&mut te.engine, path_str, &opts).unwrap(); @@ -280,9 +277,8 @@ fn ingest_parquet_append_adds_to_existing_rows() { let opts_replace = IngestOptions { table: "append_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let r1 = ingest_parquet_file(&mut te.engine, path_str, &opts_replace).unwrap(); @@ -290,9 +286,8 @@ fn ingest_parquet_append_adds_to_existing_rows() { let opts_append = IngestOptions { table: "append_data".into(), - mode: "append".into(), + mode: hyperdb_mcp::ingest::IngestMode::Append, schema_override: None, - merge_key: None, target_db: None, }; let r2 = ingest_parquet_file(&mut te.engine, path_str, &opts_append).unwrap(); @@ -322,9 +317,8 @@ fn ingest_parquet_applies_schema_override() { override_map.insert("id".to_string(), serde_json::Value::String("BIGINT".into())); let opts = IngestOptions { table: "override_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(override_map), - merge_key: None, target_db: None, }; let result = ingest_parquet_file(&mut te.engine, path_str, &opts).unwrap(); @@ -363,9 +357,8 @@ fn ingest_arrow_ipc() { let opts = IngestOptions { table: "arrow_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_arrow_ipc_file(&mut te.engine, path_str, &opts).unwrap(); @@ -418,9 +411,8 @@ fn ingest_arrow_ipc_decimal128_not_null_preserves_values() { let opts = IngestOptions { table: "ipc_decimal".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_arrow_ipc_file(&mut te.engine, path_str, &opts).unwrap(); @@ -457,9 +449,8 @@ fn ingest_arrow_ipc_append_adds_to_existing_rows() { let opts_replace = IngestOptions { table: "ipc_append".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let r1 = ingest_arrow_ipc_file(&mut te.engine, path_str, &opts_replace).unwrap(); @@ -467,9 +458,8 @@ fn ingest_arrow_ipc_append_adds_to_existing_rows() { let opts_append = IngestOptions { table: "ipc_append".into(), - mode: "append".into(), + mode: hyperdb_mcp::ingest::IngestMode::Append, schema_override: None, - merge_key: None, target_db: None, }; let r2 = ingest_arrow_ipc_file(&mut te.engine, path_str, &opts_append).unwrap(); @@ -497,9 +487,8 @@ fn ingest_arrow_ipc_rejects_schema_override() { override_map.insert("x".to_string(), serde_json::Value::String("BIGINT".into())); let opts = IngestOptions { table: "ipc_override".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(override_map), - merge_key: None, target_db: None, }; let Err(err) = ingest_arrow_ipc_file(&mut te.engine, path_str, &opts) else { @@ -583,9 +572,8 @@ async fn load_files_runs_parallel_parquet_ingests() { let mut conn = pool.get().await.unwrap(); let opts = IngestOptions { table, - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_parquet_file_async(&mut conn, &path, &opts) @@ -651,9 +639,8 @@ fn ingest_arrow_ipc_accepts_stream_format() { let opts = IngestOptions { table: "stream_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_arrow_ipc_file(&mut te.engine, path_str, &opts).unwrap(); @@ -681,9 +668,8 @@ fn ingest_parquet_merge_basic() { create_test_parquet(initial_path.to_str().unwrap()); let opts_replace = IngestOptions { table: "pq_merge".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let r1 = ingest_parquet_file( @@ -720,9 +706,10 @@ fn ingest_parquet_merge_basic() { let opts_merge = IngestOptions { table: "pq_merge".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; ingest_parquet_file(&mut te.engine, merge_path.to_str().unwrap(), &opts_merge).unwrap(); @@ -751,9 +738,8 @@ fn ingest_arrow_ipc_merge_basic() { create_test_arrow_ipc(initial_path.to_str().unwrap()); let opts_replace = IngestOptions { table: "ipc_merge".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let r1 = ingest_arrow_ipc_file( @@ -790,9 +776,10 @@ fn ingest_arrow_ipc_merge_basic() { let opts_merge = IngestOptions { table: "ipc_merge".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["x".into()], + }, schema_override: None, - merge_key: Some(vec!["x".into()]), target_db: None, }; ingest_arrow_ipc_file(&mut te.engine, merge_path.to_str().unwrap(), &opts_merge).unwrap(); @@ -834,9 +821,8 @@ fn ingest_parquet_null_type_column_fails_before_issuing_sql() { let opts = IngestOptions { table: "nulltype_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let err = ingest_parquet_file(&mut te.engine, path_str, &opts) @@ -894,9 +880,8 @@ fn ingest_parquet_null_type_column_rejected_even_with_schema_override() { ); let opts = IngestOptions { table: "nulltype_override".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(override_map), - merge_key: None, target_db: None, }; let err = ingest_parquet_file(&mut te.engine, path_str, &opts) diff --git a/hyperdb-mcp/tests/ingest_tests.rs b/hyperdb-mcp/tests/ingest_tests.rs index 22b09f95..583da365 100644 --- a/hyperdb-mcp/tests/ingest_tests.rs +++ b/hyperdb-mcp/tests/ingest_tests.rs @@ -21,9 +21,8 @@ fn ingest_json_basic() { let data = r#"[{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]"#; let opts = IngestOptions { table: "users".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json(&mut te.engine, data, &opts).unwrap(); @@ -46,16 +45,14 @@ fn ingest_json_append_mode() { let data2 = r#"[{"id": 2}]"#; let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let opts_append = IngestOptions { table: "t".into(), - mode: "append".into(), + mode: hyperdb_mcp::ingest::IngestMode::Append, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, data1, &opts_replace).unwrap(); @@ -78,9 +75,8 @@ fn ingest_csv_basic() { let csv_text = "id,name,score\n1,Alice,95.5\n2,Bob,88.0\n"; let opts = IngestOptions { table: "scores".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_csv(&mut te.engine, csv_text, &opts).unwrap(); @@ -107,9 +103,8 @@ fn ingest_json_with_schema_override() { ); let opts = IngestOptions { table: "orders".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(schema), - merge_key: None, target_db: None, }; let result = ingest_json(&mut te.engine, data, &opts).unwrap(); @@ -124,9 +119,8 @@ fn ingest_json_empty_returns_error() { let data = "[]"; let opts = IngestOptions { table: "empty".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json(&mut te.engine, data, &opts); @@ -167,9 +161,8 @@ fn ingest_json_file_loads_json_array() { ); let opts = IngestOptions { table: "people".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json_file(&mut te.engine, &path, &opts).unwrap(); @@ -198,9 +191,8 @@ fn ingest_json_file_loads_jsonl() { let (path, _keep) = tmp_with_ext("jsonl", jsonl); let opts = IngestOptions { table: "events".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json_file(&mut te.engine, &path, &opts).unwrap(); @@ -227,9 +219,8 @@ fn ingest_json_file_reports_bad_jsonl_line() { let (path, _keep) = tmp_with_ext("jsonl", bad); let opts = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let Err(err) = ingest_json_file(&mut te.engine, &path, &opts) else { @@ -256,9 +247,8 @@ fn ingest_csv_empty_cells_become_null() { let csv_text = "id,name,age\n1,Alice,30\n2,Bob,\n3,Carol,42\n"; let opts = IngestOptions { table: "u".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_csv(&mut te.engine, csv_text, &opts).unwrap(); @@ -394,9 +384,8 @@ fn ingest_json_file_handles_log_extension_via_content_sniff() { let opts = IngestOptions { table: "events".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json_file(&mut te.engine, &path, &opts).unwrap(); @@ -413,9 +402,8 @@ fn ingest_csv_file_empty_cells_become_null() { let (path, _keep) = tmp_with_ext("csv", b"code,label\nAAA,first\n,middle\nBBB,\n"); let opts = IngestOptions { table: "lookup".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_csv_file(&mut te.engine, &path, &opts).unwrap(); @@ -567,9 +555,8 @@ fn extract_json_path_then_ingest() { let opts = IngestOptions { table: "splunk_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json(&mut te.engine, &extracted, &opts).unwrap(); @@ -606,18 +593,18 @@ fn ingest_json_merge_basic() { let opts_replace = IngestOptions { table: "users".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, initial, &opts_replace).unwrap(); let opts_merge = IngestOptions { table: "users".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; let merge_result = ingest_json(&mut te.engine, updates, &opts_merge).unwrap(); @@ -657,18 +644,18 @@ fn ingest_json_merge_adds_new_column() { let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, initial, &opts_replace).unwrap(); let opts_merge = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; let merge_result = ingest_json(&mut te.engine, with_host, &opts_merge).unwrap(); @@ -707,9 +694,10 @@ fn ingest_json_merge_target_does_not_exist() { let data = r#"[{"id": 1, "name": "Alice"}]"#; let opts_merge = IngestOptions { table: "fresh".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; let result = ingest_json(&mut te.engine, data, &opts_merge).unwrap(); @@ -738,9 +726,8 @@ fn ingest_json_merge_missing_key_param() { let data = r#"[{"id": 1}]"#; let opts = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { keys: vec![] }, schema_override: None, - merge_key: None, target_db: None, }; let err = ingest_json(&mut te.engine, data, &opts).unwrap_err(); @@ -761,18 +748,18 @@ fn ingest_json_merge_key_not_in_target() { // Target has only `id`. Merge attempts to key on `not_a_col`. let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, r#"[{"id": 1}]"#, &opts_replace).unwrap(); let opts_merge = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["not_a_col".into()], + }, schema_override: None, - merge_key: Some(vec!["not_a_col".into()]), target_db: None, }; let err = ingest_json( @@ -809,9 +796,8 @@ fn ingest_json_merge_key_type_mismatch() { so.insert("id".into(), serde_json::json!("BIGINT")); let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(so), - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, r#"[{"id": 1, "name": "a"}]"#, &opts_replace).unwrap(); @@ -819,9 +805,10 @@ fn ingest_json_merge_key_type_mismatch() { // Incoming has id as quoted string, no override → inferred TEXT. let opts_merge = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; let err = @@ -843,9 +830,8 @@ fn ingest_json_merge_existing_column_type_mismatch() { so.insert("score".into(), serde_json::json!("DOUBLE PRECISION")); let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(so), - merge_key: None, target_db: None, }; ingest_json( @@ -858,9 +844,10 @@ fn ingest_json_merge_existing_column_type_mismatch() { // Incoming has score as quoted text. let opts_merge = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; let err = ingest_json( @@ -885,18 +872,18 @@ fn ingest_json_merge_no_orphan_tmp_on_failure() { // Force a key-not-in-target failure (cheap, deterministic). let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, r#"[{"id": 1}]"#, &opts_replace).unwrap(); let opts_merge = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["bogus_key".into()], + }, schema_override: None, - merge_key: Some(vec!["bogus_key".into()]), target_db: None, }; let _ = ingest_json( @@ -947,9 +934,8 @@ fn ingest_json_merge_multi_key() { ]"#; let opts_replace = IngestOptions { table: "sales".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, initial, &opts_replace).unwrap(); @@ -961,9 +947,10 @@ fn ingest_json_merge_multi_key() { ]"#; let opts_merge = IngestOptions { table: "sales".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["region".into(), "year".into()], + }, schema_override: None, - merge_key: Some(vec!["region".into(), "year".into()]), target_db: None, }; ingest_json(&mut te.engine, updates, &opts_merge).unwrap(); @@ -1007,9 +994,8 @@ fn ingest_json_merge_type_canonicalization_does_not_false_reject() { so_int.insert("id".into(), serde_json::json!("INT")); let opts_replace = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(so_int), - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, r#"[{"id": 1, "name": "a"}]"#, &opts_replace).unwrap(); @@ -1033,9 +1019,10 @@ fn ingest_json_merge_type_canonicalization_does_not_false_reject() { so_integer.insert("id".into(), serde_json::json!("INTEGER")); let opts_merge = IngestOptions { table: "t".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: Some(so_integer), - merge_key: Some(vec!["id".into()]), target_db: None, }; let result = ingest_json( @@ -1074,18 +1061,18 @@ fn ingest_csv_merge_basic() { let opts_replace = IngestOptions { table: "users_csv".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_csv(&mut te.engine, initial, &opts_replace).unwrap(); let opts_merge = IngestOptions { table: "users_csv".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: None, }; let merge_result = ingest_csv(&mut te.engine, updates, &opts_merge).unwrap(); @@ -1104,3 +1091,62 @@ fn ingest_csv_merge_basic() { assert_eq!(rows[2]["name"], "Carol Updated"); // updated assert_eq!(rows[3]["name"], "Dave"); // inserted } + +// --------------------------------------------------------------------- +// IngestMode::parse — an unrecognised mode must never become a replace. +// --------------------------------------------------------------------- + +/// Only the exact lowercase names parse; everything else is +/// `InvalidArgument` (with a did-you-mean hint for case/whitespace slips). +#[test] +fn ingest_mode_parse_is_strict() { + use hyperdb_mcp::error::ErrorCode; + use hyperdb_mcp::ingest::IngestMode; + let all = ["replace", "append", "merge"]; + let rw = ["replace", "append"]; + + assert_eq!( + IngestMode::parse(None, None, &rw, "load_data").unwrap(), + IngestMode::Replace + ); + assert_eq!( + IngestMode::parse(Some("append"), None, &rw, "load_data").unwrap(), + IngestMode::Append + ); + assert_eq!( + IngestMode::parse(Some("merge"), Some(vec!["id".into()]), &all, "load_file").unwrap(), + IngestMode::Merge { + keys: vec!["id".into()] + } + ); + + for bad in ["Append", "APPEND", " append", "appendd", ""] { + let err = IngestMode::parse(Some(bad), None, &all, "load_file").expect_err(bad); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{bad:?}"); + assert!(err.message.contains("load_file"), "{}", err.message); + assert!( + err.message.contains("replace, append, merge"), + "{}", + err.message + ); + } + let hint = IngestMode::parse(Some("Append"), None, &all, "load_file").unwrap_err(); + assert!( + hint.message.contains("did you mean 'append'"), + "{}", + hint.message + ); + let typo = IngestMode::parse(Some("appendd"), None, &all, "load_file").unwrap_err(); + assert!(!typo.message.contains("did you mean"), "{}", typo.message); + + // merge where the tool does not support it + let err = IngestMode::parse(Some("merge"), Some(vec!["id".into()]), &rw, "load_data") + .expect_err("merge unsupported"); + assert_eq!(err.code, ErrorCode::InvalidArgument); + assert!(err.message.contains("load_data"), "{}", err.message); + + // merge_key rules + assert!(IngestMode::parse(Some("merge"), None, &all, "load_file").is_err()); + assert!(IngestMode::parse(Some("merge"), Some(vec![]), &all, "load_file").is_err()); + assert!(IngestMode::parse(Some("append"), Some(vec!["id".into()]), &all, "load_file").is_err()); +} diff --git a/hyperdb-mcp/tests/integration_tests.rs b/hyperdb-mcp/tests/integration_tests.rs index b7379369..f55e6bbc 100644 --- a/hyperdb-mcp/tests/integration_tests.rs +++ b/hyperdb-mcp/tests/integration_tests.rs @@ -23,9 +23,8 @@ fn full_pipeline_json_to_query() { ]"#; let opts = IngestOptions { table: "orders".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, orders, &opts).unwrap(); @@ -36,9 +35,8 @@ fn full_pipeline_json_to_query() { ]"#; let opts = IngestOptions { table: "customers".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_json(&mut te.engine, customers, &opts).unwrap(); @@ -60,9 +58,8 @@ fn full_pipeline_csv_ingest_and_export() { let csv_data = "product,quantity,price\nWidget,100,9.99\nGadget,50,19.99\n"; let opts = IngestOptions { table: "products".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; ingest_csv(&mut te.engine, csv_data, &opts).unwrap(); @@ -143,16 +140,14 @@ fn append_mode_accumulates_data() { let opts_replace = IngestOptions { table: "acc".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let opts_append = IngestOptions { table: "acc".into(), - mode: "append".into(), + mode: hyperdb_mcp::ingest::IngestMode::Append, schema_override: None, - merge_key: None, target_db: None, }; diff --git a/hyperdb-mcp/tests/lakehouse_tests.rs b/hyperdb-mcp/tests/lakehouse_tests.rs index be62113a..a9a37bc8 100644 --- a/hyperdb-mcp/tests/lakehouse_tests.rs +++ b/hyperdb-mcp/tests/lakehouse_tests.rs @@ -25,7 +25,7 @@ fn ingest_iceberg_missing_path_errors_cleanly() { let mut te = TestEngine::new_ephemeral(); let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::lakehouse::IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; @@ -51,7 +51,7 @@ fn ingest_iceberg_file_instead_of_directory_errors_cleanly() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::lakehouse::IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; @@ -76,7 +76,7 @@ fn ingest_iceberg_empty_directory_surfaces_hyperd_error() { let opts = IcebergIngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::lakehouse::IcebergMode::Replace, metadata_filename: None, version_as_of: None, }; @@ -104,22 +104,19 @@ fn ingest_iceberg_empty_directory_surfaces_hyperd_error() { ); } -/// Mode validation: any string other than `"append"` is treated as a -/// replace. Verify by giving a junk mode against a missing path — the -/// error path exercises `is_replace = true` and should therefore have -/// attempted a DROP before the CTAS. The DROP is silent for a -/// non-existent table, so we just check the outer error is path-related. +/// Mode validation: an unknown `mode` is rejected at the parse boundary +/// instead of silently becoming a destructive replace. #[test] -fn ingest_iceberg_unknown_mode_treated_as_replace() { - let mut te = TestEngine::new_ephemeral(); - let opts = IcebergIngestOptions { - table: "t".into(), - mode: "blahblah".into(), - metadata_filename: None, - version_as_of: None, - }; - let Err(err) = ingest_iceberg_table(&mut te.engine, "/nonexistent/x", &opts) else { - panic!("should fail on path") - }; - assert!(err.to_string().to_lowercase().contains("not exist")); +fn iceberg_mode_rejects_unknown_values() { + use hyperdb_mcp::lakehouse::IcebergMode; + for bad in ["blahblah", "merge", "Append", "", " append"] { + let err = IcebergMode::parse(Some(bad)).expect_err(bad); + assert_eq!(err.code, hyperdb_mcp::error::ErrorCode::InvalidArgument); + assert!(err.message.contains("load_iceberg"), "{}", err.message); + } + assert_eq!(IcebergMode::parse(None).unwrap(), IcebergMode::Replace); + assert_eq!( + IcebergMode::parse(Some("append")).unwrap(), + IcebergMode::Append + ); } diff --git a/hyperdb-mcp/tests/per_tool_database_tests.rs b/hyperdb-mcp/tests/per_tool_database_tests.rs index 018a002d..26c38636 100644 --- a/hyperdb-mcp/tests/per_tool_database_tests.rs +++ b/hyperdb-mcp/tests/per_tool_database_tests.rs @@ -26,9 +26,8 @@ use tempfile::TempDir; fn qualified_table_unqualified_when_target_db_none() { let opts = IngestOptions { table: "widgets".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; assert_eq!(qualified_table(&opts), "\"widgets\""); @@ -40,9 +39,8 @@ fn qualified_table_unqualified_when_target_db_none() { fn qualified_table_qualifies_when_target_db_set() { let opts = IngestOptions { table: "widgets".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; assert_eq!( @@ -57,9 +55,8 @@ fn qualified_table_qualifies_when_target_db_set() { fn qualified_table_escapes_embedded_quotes() { let opts = IngestOptions { table: "weird\"name".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("db\"alias".into()), }; assert_eq!( @@ -120,9 +117,8 @@ fn ingest_json_with_persistent_target_lands_in_persistent() { let mut te = TestEngine::new_ephemeral(); let opts = IngestOptions { table: "persisted_data".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; let data = r#"[{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]"#; @@ -156,9 +152,8 @@ fn ingest_csv_with_persistent_target_lands_in_persistent() { let mut te = TestEngine::new_ephemeral(); let opts = IngestOptions { table: "csv_target".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; let data = "id,name\n1,Alice\n2,Bob\n"; @@ -184,9 +179,8 @@ fn ingest_to_persistent_survives_engine_recreate() { let mut engine = Engine::new_no_daemon(Some(path_str.clone())).unwrap(); let opts = IngestOptions { table: "library".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; let data = r#"[{"id": 1, "title": "Dune"}]"#; @@ -209,9 +203,8 @@ fn ingest_with_no_target_db_lands_in_primary() { let mut te = TestEngine::new_ephemeral(); let opts = IngestOptions { table: "scratch".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let data = r#"[{"x": 1}]"#; @@ -298,9 +291,8 @@ fn sample_table_in_persistent_returns_rows() { let mut te = TestEngine::new_ephemeral(); let opts = IngestOptions { table: "samples".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; ingest_json( @@ -514,9 +506,10 @@ fn merge_into_persistent_creates_table_when_missing() { let mut te = TestEngine::new_ephemeral(); let opts = IngestOptions { table: "merged_persist".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: Some("persistent".into()), }; let data = r#"[{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]"#; @@ -545,9 +538,8 @@ fn merge_into_persistent_replaces_matching_and_appends_new() { // Seed target with two rows. let seed_opts = IngestOptions { table: "merge_target".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; ingest_json( @@ -560,9 +552,10 @@ fn merge_into_persistent_replaces_matching_and_appends_new() { // Merge: id=1 replaces, id=3 appends, id=2 stays. let merge_opts = IngestOptions { table: "merge_target".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: Some("persistent".into()), }; let result = ingest_json( @@ -593,9 +586,8 @@ fn merge_into_persistent_alters_when_incoming_has_new_column() { let mut te = TestEngine::new_ephemeral(); let seed_opts = IngestOptions { table: "widens".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: Some("persistent".into()), }; ingest_json( @@ -607,9 +599,10 @@ fn merge_into_persistent_alters_when_incoming_has_new_column() { let merge_opts = IngestOptions { table: "widens".into(), - mode: "merge".into(), + mode: hyperdb_mcp::ingest::IngestMode::Merge { + keys: vec!["id".into()], + }, schema_override: None, - merge_key: Some(vec!["id".into()]), target_db: Some("persistent".into()), }; let result = ingest_json( diff --git a/hyperdb-mcp/tests/recovery_tests.rs b/hyperdb-mcp/tests/recovery_tests.rs index 009f6793..1fb0b021 100644 --- a/hyperdb-mcp/tests/recovery_tests.rs +++ b/hyperdb-mcp/tests/recovery_tests.rs @@ -349,15 +349,26 @@ fn start_reported_hyper_process(state_dir: &Path, hyper_pid_path: &Path) -> (Hyp let log_dir = state_dir.join("hyper-logs"); std::fs::create_dir_all(&log_dir).expect("create isolated Hyper log directory"); let mut parameters = Parameters::new(); - parameters.set_transport_mode(TransportMode::Tcp); + // IPC, like the real daemon: the engine only dials a `hyperd` endpoint of + // the shape the daemon publishes (`/sockets/` on Unix, a + // local named pipe on Windows) and refuses TCP. + parameters.set_transport_mode(TransportMode::Ipc); parameters.set("log_dir", log_dir.to_string_lossy().as_ref()); - let hyper = HyperProcess::new(None, Some(¶meters)).expect("start real TCP HyperProcess"); - assert_eq!(hyper.transport_mode(), TransportMode::Tcp); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let socket_dir = state_dir.join("sockets"); + std::fs::create_dir_all(&socket_dir).expect("create isolated Hyper socket directory"); + std::fs::set_permissions(&socket_dir, std::fs::Permissions::from_mode(0o700)) + .expect("restrict isolated Hyper socket directory"); + parameters.set_domain_socket_directory(socket_dir); + } + let hyper = HyperProcess::new(None, Some(¶meters)).expect("start real IPC HyperProcess"); + assert_eq!(hyper.transport_mode(), TransportMode::Ipc); let hyper_pid = hyper.pid().expect("HyperProcess must own a child PID"); let endpoint = hyper - .require_endpoint() - .expect("TCP HyperProcess must publish an endpoint") - .to_string(); + .connection_endpoint_string() + .expect("IPC HyperProcess must publish a connection endpoint"); std::fs::write(hyper_pid_path, hyper_pid.to_string()).expect("report exact Hyper PID"); (hyper, endpoint) } diff --git a/hyperdb-mcp/tests/schema_tests.rs b/hyperdb-mcp/tests/schema_tests.rs index 20a0265a..bce8d9f8 100644 --- a/hyperdb-mcp/tests/schema_tests.rs +++ b/hyperdb-mcp/tests/schema_tests.rs @@ -137,14 +137,14 @@ fn infer_csv_no_header() { /// including parameterized types like NUMERIC(12,2). #[test] fn map_hyper_type_names() { - assert!(map_hyper_type("INT").is_some()); - assert!(map_hyper_type("BIGINT").is_some()); - assert!(map_hyper_type("TEXT").is_some()); - assert!(map_hyper_type("DOUBLE PRECISION").is_some()); - assert!(map_hyper_type("BOOL").is_some()); - assert!(map_hyper_type("DATE").is_some()); - assert!(map_hyper_type("TIMESTAMP").is_some()); - assert!(map_hyper_type("NUMERIC(12,2)").is_some()); + assert!(map_hyper_type("INT").is_ok()); + assert!(map_hyper_type("BIGINT").is_ok()); + assert!(map_hyper_type("TEXT").is_ok()); + assert!(map_hyper_type("DOUBLE PRECISION").is_ok()); + assert!(map_hyper_type("BOOL").is_ok()); + assert!(map_hyper_type("DATE").is_ok()); + assert!(map_hyper_type("TIMESTAMP").is_ok()); + assert!(map_hyper_type("NUMERIC(12,2)").is_ok()); } /// Verify that a column where every row is null collapses to TEXT (the safe @@ -225,6 +225,61 @@ fn apply_override_empty_is_noop() { assert_eq!(result[1].hyper_type, "TEXT"); } +/// Run `apply_schema_override` for a single column and return its result, +/// converting a panic into a test failure message (the old code asserted). +fn override_one(ty: &str) -> Result, hyperdb_mcp::error::McpError> { + let inferred = cols(&[("v", "INT")]); + let override_map = json!({ "v": ty }).as_object().cloned().unwrap(); + apply_schema_override(inferred, &override_map) +} + +/// Out-of-range NUMERIC overrides must be a clean `InvalidArgument`, not a +/// panic inside `SqlType::numeric` (which would poison the engine mutex). +#[test] +fn override_rejects_out_of_range_numeric() { + for ty in [ + "NUMERIC(40,2)", + "NUMERIC(0)", + "NUMERIC(5,10)", + "DECIMAL(39)", + ] { + let err = override_one(ty).expect_err(ty); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{ty}"); + assert!( + err.message + .contains("NUMERIC precision must be 1..=38 and scale 0..=precision"), + "{ty}: {}", + err.message + ); + } +} + +/// Trailing junk after a type keyword must not be accepted (it used to match +/// `starts_with("NUMERIC")` and then be spliced raw into DDL). +#[test] +fn override_rejects_numeric_with_trailing_garbage() { + for ty in [ + "NUMERICX", + "NUMERIC, x INT", + "NUMERIC(10,2) NOT NULL", + "NUMERIC(10,2", + ] { + assert!(override_one(ty).is_err(), "{ty} must be rejected"); + } +} + +/// Accepted overrides are stored in canonical form so the DDL splice never +/// sees user-controlled text. +#[test] +fn override_stores_canonical_type_text() { + let r = override_one(" numeric( 12 , 2 ) ").unwrap(); + assert_eq!(r[0].hyper_type, "NUMERIC(12, 2)"); + let r = override_one("numeric").unwrap(); + assert_eq!(r[0].hyper_type, "NUMERIC(38, 0)"); + let r = override_one("varchar").unwrap(); + assert_eq!(r[0].hyper_type, "TEXT"); +} + // --- Full-file numeric widening (B) ------------------------------------- /// Regression test for the bug that triggered this whole workstream: a CSV diff --git a/hyperdb-mcp/tests/transaction_tests.rs b/hyperdb-mcp/tests/transaction_tests.rs index c4cd1904..4cfc8b65 100644 --- a/hyperdb-mcp/tests/transaction_tests.rs +++ b/hyperdb-mcp/tests/transaction_tests.rs @@ -219,9 +219,8 @@ fn failed_json_ingest_rolls_back_partial_inserts() { let opts = IngestOptions { table: "t".into(), - mode: "append".into(), + mode: hyperdb_mcp::ingest::IngestMode::Append, schema_override: None, - merge_key: None, target_db: None, }; @@ -258,9 +257,8 @@ fn successful_replace_commits_atomically() { ]"#; let opts = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: None, - merge_key: None, target_db: None, }; let result = ingest_json(&mut te.engine, data, &opts).unwrap(); @@ -411,9 +409,8 @@ fn failed_replace_leaves_empty_table_not_partial() { override_map.insert("id".into(), serde_json::Value::String("INT".into())); let opts = IngestOptions { table: "t".into(), - mode: "replace".into(), + mode: hyperdb_mcp::ingest::IngestMode::Replace, schema_override: Some(override_map), - merge_key: None, target_db: None, }; @@ -430,3 +427,210 @@ fn failed_replace_leaves_empty_table_not_partial() { "expected partial INSERTs to be rolled back, leaving 0 rows" ); } + +// --------------------------------------------------------------------- +// load_file merge atomicity (D1): the DELETE of matched rows and the +// INSERT of the incoming rows run in one transaction, and a failed merge +// leaves the target's rows *and columns* untouched. +// --------------------------------------------------------------------- + +mod merge_atomicity { + use super::{TestEngine, query_resilient}; + use hyperdb_mcp::error::ErrorCode; + use hyperdb_mcp::ingest::{IngestMode, IngestOptions, ingest_csv, ingest_json}; + + fn opts(table: &str, mode: IngestMode, target_db: Option<&str>) -> IngestOptions { + IngestOptions { + table: table.into(), + mode, + schema_override: None, + target_db: target_db.map(str::to_string), + } + } + + fn merge_on_id() -> IngestMode { + IngestMode::Merge { + keys: vec!["id".into()], + } + } + + /// Seed `t` with (1,a),(2,b); `name` is NOT NULL by JSON inference. + fn seed(te: &mut TestEngine, target_db: Option<&str>) { + ingest_json( + &mut te.engine, + r#"[{"id":1,"name":"a"},{"id":2,"name":"b"}]"#, + &opts("t", IngestMode::Replace, target_db), + ) + .unwrap(); + } + + fn qualified(target_db: Option<&str>) -> String { + match target_db { + Some(db) => format!("\"{db}\".\"public\".\"t\""), + None => "t".into(), + } + } + + fn rows_of(te: &TestEngine, target_db: Option<&str>) -> Vec<(i64, String)> { + query_resilient( + &te.engine, + &format!("SELECT id, name FROM {} ORDER BY id", qualified(target_db)), + ) + .iter() + .map(|r| { + ( + r["id"].as_i64().unwrap(), + r["name"].as_str().unwrap().to_string(), + ) + }) + .collect() + } + + fn original() -> Vec<(i64, String)> { + vec![(1, "a".into()), (2, "b".into())] + } + + fn column_names(te: &TestEngine, target_db: Option<&str>) -> Vec { + te.engine + .column_metadata_in(target_db, "t") + .unwrap() + .into_iter() + .map(|c| c.name) + .collect() + } + + fn scratch_tables(te: &TestEngine) -> Vec { + te.engine + .execute_query_to_json( + "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = 'public'", + ) + .unwrap() + .iter() + .filter_map(|v| v["tablename"].as_str().map(str::to_string)) + .filter(|n| n.starts_with("__hyperdb_")) + .collect() + } + + #[test] + fn merge_null_into_not_null_keeps_matched_rows() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let res = ingest_json( + &mut te.engine, + r#"[{"id":1,"name":null},{"id":3,"name":"c"}]"#, + &opts("t", merge_on_id(), None), + ); + assert!(res.is_err(), "NULL into NOT NULL must fail"); + assert_eq!(rows_of(&te, None), original(), "matched row 1 was lost"); + } + + #[test] + fn merge_csv_null_into_not_null_keeps_matched_rows() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let res = ingest_csv( + &mut te.engine, + "id,name\n1,\n3,c\n", + &opts("t", merge_on_id(), None), + ); + assert!(res.is_err(), "blank CSV name into NOT NULL must fail"); + assert_eq!(rows_of(&te, None), original(), "matched row 1 was lost"); + } + + #[test] + fn failed_merge_rolls_back_added_column() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let res = ingest_json( + &mut te.engine, + r#"[{"id":1,"name":null,"extra":5}]"#, + &opts("t", merge_on_id(), None), + ); + assert!(res.is_err()); + assert_eq!(rows_of(&te, None), original()); + assert_eq!(column_names(&te, None), vec!["id", "name"]); + } + + #[test] + fn merge_missing_not_null_column_is_rejected_up_front() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let err = ingest_json( + &mut te.engine, + r#"[{"id":1,"extra":5}]"#, + &opts("t", merge_on_id(), None), + ) + .unwrap_err(); + assert_eq!(err.code, ErrorCode::InvalidArgument, "{}", err.message); + assert!(err.message.contains("name"), "{}", err.message); + assert_eq!(rows_of(&te, None), original()); + assert_eq!(column_names(&te, None), vec!["id", "name"]); + } + + #[test] + fn merge_failure_in_persistent_target_keeps_rows() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, Some("persistent")); + let res = ingest_json( + &mut te.engine, + r#"[{"id":1,"name":null,"extra":5},{"id":3,"name":"c"}]"#, + &opts("t", merge_on_id(), Some("persistent")), + ); + assert!(res.is_err()); + assert_eq!(rows_of(&te, Some("persistent")), original()); + assert_eq!(column_names(&te, Some("persistent")), vec!["id", "name"]); + } + + #[test] + fn failed_merge_leaves_no_staging_table() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let _ = ingest_json( + &mut te.engine, + r#"[{"id":1,"name":null}]"#, + &opts("t", merge_on_id(), None), + ); + assert_eq!(scratch_tables(&te), Vec::::new()); + } + + /// The staging load itself fails (a non-numeric value in an INT column): + /// Hyper has already committed the staging DDL, so the guard must be armed + /// before the load or the `__hyperdb_merge_*` table leaks. + #[test] + fn failed_staging_load_leaves_no_staging_table() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let mut schema = serde_json::Map::new(); + schema.insert("id".into(), serde_json::json!("INT")); + let res = ingest_csv( + &mut te.engine, + "id,name\nbogus,x\n", + &IngestOptions { + table: "t".into(), + mode: merge_on_id(), + schema_override: Some(schema), + target_db: None, + }, + ); + assert!(res.is_err(), "non-numeric id in an INT column must fail"); + assert_eq!(rows_of(&te, None), original()); + assert_eq!(scratch_tables(&te), Vec::::new()); + } + + #[test] + fn successful_merge_still_upserts() { + let mut te = TestEngine::new_ephemeral(); + seed(&mut te, None); + let r = ingest_json( + &mut te.engine, + r#"[{"id":2,"name":"B"},{"id":3,"name":"c"}]"#, + &opts("t", merge_on_id(), None), + ) + .unwrap(); + assert_eq!(r.rows, 2); + assert_eq!( + rows_of(&te, None), + vec![(1, "a".into()), (2, "B".into()), (3, "c".into())] + ); + } +} From 9d4d0dc6444633c534e493f43d331df1e9880987 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 20:31:40 -0700 Subject: [PATCH 46/61] fix(compile-check): skip registration errors under the rust-analyzer proc-macro server Its shared, lazily-expanding registry can be non-empty yet lack the struct under the cursor, producing false 'table not registered' diagnostics. --- hyperdb-compile-check/CHANGELOG.md | 5 +++++ hyperdb-compile-check/src/registry.rs | 24 ++++++++++++++++++++++++ hyperdb-compile-check/src/validate.rs | 14 +++++++------- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/hyperdb-compile-check/CHANGELOG.md b/hyperdb-compile-check/CHANGELOG.md index 755e7b8c..480b909c 100644 --- a/hyperdb-compile-check/CHANGELOG.md +++ b/hyperdb-compile-check/CHANGELOG.md @@ -18,6 +18,11 @@ surface, and consumers should depend on `hyperdb-api-derive` instead. the statement with nothing bound, so Hyper rejected every parameterized query with "expected to have 0 parameter(s)". Such queries are now prepared with one unspecified-type parameter per placeholder. +- **No more false "not registered" errors in rust-analyzer.** Its shared + proc-macro server can hold a non-empty registry that lacks the struct under + the cursor, so the empty-registry skip was not enough. Validation now also + skips unregistered-struct/table diagnostics when expanding inside + `rust-analyzer-proc-macro-srv`; `cargo` builds are unchanged. ## [1.0.0-rc.3] - 2026-09-07 diff --git a/hyperdb-compile-check/src/registry.rs b/hyperdb-compile-check/src/registry.rs index 10b6940d..18a0faf4 100644 --- a/hyperdb-compile-check/src/registry.rs +++ b/hyperdb-compile-check/src/registry.rs @@ -111,6 +111,30 @@ pub fn is_empty() -> bool { registry().lock().is_empty() } +/// Returns true when "not in the registry" is not evidence of a missing +/// `derive(Table)`, so callers must skip validation instead of erroring. +/// +/// That holds when the registry is empty (see [`is_empty`]) and also whenever +/// the macro is expanding inside rust-analyzer's `proc-macro-srv`. That +/// process is shared across the whole workspace and expands lazily and out of +/// order, so its registry is routinely *non-empty yet missing the struct under +/// the cursor* (some other crate's derive registered a table, this crate's +/// did not run in this process). `cargo`/rustc never has this problem, so real +/// builds keep full validation. +pub fn registration_unknowable() -> bool { + is_empty() || running_under_rust_analyzer() +} + +fn running_under_rust_analyzer() -> bool { + static UNDER_RA: std::sync::OnceLock = std::sync::OnceLock::new(); + *UNDER_RA.get_or_init(|| { + std::env::current_exe() + .ok() + .and_then(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned())) + .is_some_and(|stem| stem.starts_with("rust-analyzer")) + }) +} + /// The public `Registry` type — a thin newtype that provides the seeding /// interface against a live `CompileTimeDb`. Created from a lock guard by /// `validate_query_as`. diff --git a/hyperdb-compile-check/src/validate.rs b/hyperdb-compile-check/src/validate.rs index 51360ce2..2b18925f 100644 --- a/hyperdb-compile-check/src/validate.rs +++ b/hyperdb-compile-check/src/validate.rs @@ -44,10 +44,10 @@ use crate::registry::{self, Registry}; pub fn validate_query_as(struct_name: &str, sql: &str) -> Result<(), ValidationError> { // Step 1: look up by struct ident (not SQL table name — they differ). let Some((_table_name, entry)) = registry::get_by_struct(struct_name) else { - // An empty registry means no `derive(Table)` ran in this process, so we - // have no basis to claim the struct is unregistered. See - // `registry::is_empty`. - if registry::is_empty() { + // An empty registry, or a rust-analyzer proc-macro server with a partial + // one, gives us no basis to claim the struct is unregistered. See + // `registry::registration_unknowable`. + if registry::registration_unknowable() { return Ok(()); } return Err(ValidationError::StructNotRegistered { @@ -116,8 +116,8 @@ pub fn validate_scalar_sql(sql: &str) -> Result<(), ValidationError> { /// /// Stops early on syntax errors, missing-column errors, or unregistered tables. /// -/// Returns `Ok(None)` when validation cannot be performed because nothing is -/// registered in this process — see [`registry::is_empty`]. Callers skip +/// Returns `Ok(None)` when validation cannot be performed because registration +/// cannot be known in this process — see [`registry::registration_unknowable`]. Callers skip /// validation in that case instead of reporting a false diagnostic. fn run_dry_run_with_seed( sql: &str, @@ -134,7 +134,7 @@ fn run_dry_run_with_seed( ErrorClass::MissingTable(t) => match Registry::seed_if_known(&t, db) { Ok(true) => {} // seeded successfully; loop iterates to retry the dry-run Ok(false) => { - if registry::is_empty() { + if registry::registration_unknowable() { return Ok(None); } return Err(ValidationError::TablesNotRegistered { tables: vec![t] }); From 473289aac86732bd855592381aae7c56b9fdf5b0 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 21:13:58 -0700 Subject: [PATCH 47/61] fix(mcp): cap the advertised MCP protocol at 2025-11-25 and harden catalog bootstrap Clients that predate the stateless 2026-07-28 revision rejected the server. Also make the catalog-ready flag an AtomicBool so a panic cannot disable bootstrap, return an error instead of panicking in IngestMode::parse, and correct the CHANGELOG. --- hyperdb-mcp/CHANGELOG.md | 26 +++++++-- hyperdb-mcp/src/ingest.rs | 5 +- hyperdb-mcp/src/server.rs | 59 +++++++++++++------ hyperdb-mcp/tests/end_to_end_mcp_tests.rs | 70 +++++++++++++++++++++++ 4 files changed, 134 insertions(+), 26 deletions(-) diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 4815e2a5..c0cacb37 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed +- **BREAKING (Rust API): `schema::map_hyper_type` now returns `Result`** instead + of panicking on an out-of-range `NUMERIC`; callers must handle the error. - **`get_readme` is now written in a compact schema notation.** Tool signatures read `tool(required, optional?=default) → result`, with a legend at the top. The document is about 10% smaller yet carries more: per-tool @@ -154,14 +156,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `load_file`, `load_files`, `load_iceberg` and `copy_query` used to treat any value other than the exact lowercase names (`Append`, `appendd`, `""`) as `replace`. Each tool now parses `mode` once at the boundary and returns - `INVALID_ARGUMENT` with a did-you-mean hint; `load_files` rejects a bad entry + `INVALID_ARGUMENT` (with a did-you-mean hint for case and whitespace slips such as `Append` or ` append `); `load_files` rejects a bad entry before ingesting anything. - **A failed `load_file` merge no longer loses the matched rows.** The DELETE and INSERT now run in one transaction, so an INSERT failure (for example a NULL into a NOT NULL column) restores the deleted rows. A NOT NULL target column missing from the incoming data is rejected up front, columns added for the merge are dropped again on failure, and the staging table - (`__hyperdb_merge_*`, hidden from listings) is always cleaned up. + (`__hyperdb_merge_*`, hidden from listings) is cleaned up. - **`copy_query` `mode=replace` no longer drops the target before the SELECT succeeds.** The result is built in a hidden staging table (`__hyperdb_copy_*`) and swapped in with two renames, so a failing SELECT @@ -174,10 +176,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/). column, now returns `INVALID_ARGUMENT` (precision must be 1..=38, scale 0..=precision). Type strings are parsed strictly (`NUMERICX` and `NUMERIC, x INT` are rejected) and stored in canonical form, so user text is - never spliced into DDL. A panic in any tool body is now caught and reported as - an internal error instead of poisoning the engine mutex; the engine is - discarded and the next call reconnects (in local mode this drops the whole - ephemeral workspace, so tables loaded earlier in the session are lost). `schema::map_hyper_type` now returns `Result`. + never spliced into DDL. A panic in a tool body that runs under the engine lock + is now caught and reported as an internal error instead of poisoning the + engine mutex; the engine is discarded and the next call reconnects (in local + mode this drops the whole ephemeral workspace, so tables loaded earlier in + the session are lost). `status` contains only a panic in its catalog + bootstrap (it then reports the degraded response), and panics in + `load_files` worker tasks are not covered by this guard. +- **The MCP server no longer advertises protocol revision `2026-07-28`.** + Claude Code rejected the server with "Server's protocol version is not + supported: 2026-07-28". The server now supports revisions up to `2025-11-25` + (rmcp's `LATEST_WITH_INITIALIZE`) and negotiates a client asking for newer + down to that. +- **The catalog-bootstrap flag can no longer be poisoned.** A panic could + previously leave `_table_catalog` bootstrap silently disabled. +- **`IngestMode::parse` no longer panics** on an unexpected mode name; it + returns an `INTERNAL_ERROR`. - **The daemon's `hyperd` endpoint is no longer trusted from `daemon.json` alone.** The client confirms it with the authenticated daemon's `STATUS` reply and accepts only a daemon-owned IPC channel (`/sockets/`, or a diff --git a/hyperdb-mcp/src/ingest.rs b/hyperdb-mcp/src/ingest.rs index c37fe84d..0990250c 100644 --- a/hyperdb-mcp/src/ingest.rs +++ b/hyperdb-mcp/src/ingest.rs @@ -241,7 +241,10 @@ impl IngestMode { Self::Replace }) } - other => unreachable!("parse_mode_name returned a mode outside the known set: {other}"), + other => Err(McpError::new( + ErrorCode::InternalError, + format!("{tool}: internal error: unexpected mode '{other}' from mode parser"), + )), } } diff --git a/hyperdb-mcp/src/server.rs b/hyperdb-mcp/src/server.rs index 4a78864e..800cba11 100644 --- a/hyperdb-mcp/src/server.rs +++ b/hyperdb-mcp/src/server.rs @@ -41,7 +41,7 @@ use rmcp::handler::server::wrapper::Parameters; use rmcp::model::{ CallToolResult, ContentBlock, Implementation, InitializeRequestParams, InitializeResult, ListResourceTemplatesResult, ListResourcesResult, PaginatedRequestParams, PromptMessage, - ReadResourceRequestParams, ReadResourceResponse, ReadResourceResult, Resource, + ProtocolVersion, ReadResourceRequestParams, ReadResourceResponse, ReadResourceResult, Resource, ResourceContents, ResourceTemplate, Role, ServerCapabilities, ServerConfig, SubscribeRequestParams, UnsubscribeRequestParams, }; @@ -56,6 +56,7 @@ use serde_json::{Value, json}; use sqlformat::{FormatOptions, Indent, QueryParams as SqlQueryParams}; use std::fmt::Write as _; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; /// Number of rows returned by the `hyper://tables/{name}/sample` JSON @@ -1020,7 +1021,7 @@ pub struct HyperMcpServer { /// `_table_catalog` once per process. Reset to `false` if the /// underlying engine is torn down (e.g. connection lost) so the next /// call re-bootstraps. - catalog_ready: Arc>, + catalog_ready: Arc, watchers: Arc, saved_queries: Arc, subscriptions: Arc, @@ -1127,7 +1128,7 @@ impl HyperMcpServer { Self { engine: Arc::new(Mutex::new(None)), engine_initialization: Mutex::new(()), - catalog_ready: Arc::new(Mutex::new(false)), + catalog_ready: Arc::new(AtomicBool::new(false)), watchers: Arc::new(crate::watcher::WatcherRegistry::new()), saved_queries, subscriptions: Arc::new(SubscriptionRegistry::new()), @@ -1483,9 +1484,7 @@ impl HyperMcpServer { // New engine → catalog may need to be created/reconciled even if we // already did it against a prior (now-dead) engine. Preserve the // existing engine-then-catalog lock order. - if let Ok(mut ready) = self.catalog_ready.lock() { - *ready = false; - } + self.catalog_ready.store(false, Ordering::Release); drop(initialization); Ok(guard) } @@ -1525,10 +1524,10 @@ impl HyperMcpServer { if self.read_only { return; } - let Ok(mut ready) = self.catalog_ready.lock() else { - return; - }; - if *ready { + // An atomic (not a Mutex) so a panic can never poison the flag and + // silently disable catalog bootstrap. Callers hold the engine lock, + // so bootstrap is already serialized. + if self.catalog_ready.load(Ordering::Acquire) { return; } if let Err(e) = crate::table_catalog::ensure_exists(engine) { @@ -1537,7 +1536,7 @@ impl HyperMcpServer { if let Err(e) = crate::table_catalog::reconcile(engine) { tracing::warn!(err = %e.message, "failed to reconcile _table_catalog on startup"); } - *ready = true; + self.catalog_ready.store(true, Ordering::Release); } /// Best-effort catalog upsert after a successful ingest. Logs and @@ -1656,9 +1655,7 @@ impl HyperMcpServer { .unwrap_or_else(|| "non-string panic payload".to_string()); tracing::error!(panic = %detail, "tool call panicked; discarding engine"); *guard = None; - if let Ok(mut ready) = self.catalog_ready.lock() { - *ready = false; - } + self.catalog_ready.store(false, Ordering::Release); Err(McpError::new( ErrorCode::InternalError, format!( @@ -1685,9 +1682,7 @@ impl HyperMcpServer { *guard = None; // Reset so the next call re-bootstraps the catalog // against the fresh engine. - if let Ok(mut ready) = self.catalog_ready.lock() { - *ready = false; - } + self.catalog_ready.store(false, Ordering::Release); } } drop(guard); @@ -3954,7 +3949,21 @@ impl HyperMcpServer { Err(e) => Self::err_content(e), }; }; - self.ensure_catalog_ready(engine); + // Same panic containment as `with_engine`: a panic in the catalog + // bootstrap must not unwind through the engine mutex guard. + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.ensure_catalog_ready(engine); + })) + .is_err() + { + tracing::error!("catalog bootstrap panicked in status; reporting degraded status"); + self.catalog_ready.store(false, Ordering::Release); + drop(guard); + return match self.status_degraded() { + Ok(status) => Self::ok_content(status), + Err(e) => Self::err_content(e), + }; + } let result = engine.status(); match result { @@ -4907,6 +4916,17 @@ impl HyperMcpServer { reason = "ServerHandler trait methods are async by trait contract; several handlers here (and the tool_handler/prompt_handler macro expansions) have no await points but keeping them as async fn matches the trait signature callers expect" )] impl ServerHandler for HyperMcpServer { + /// Newest MCP revision this server speaks. rmcp's default also advertises + /// `2026-07-28`, the stateless-lifecycle revision; clients that do not + /// implement it reject the server with "Server's protocol version is not + /// supported: 2026-07-28". Capped at rmcp's `LATEST_WITH_INITIALIZE` (2025-11-25), the newest + /// revision that still uses the `initialize` handshake (`LATEST` is `2026-07-28`). + fn supported_protocol_versions(&self) -> std::borrow::Cow<'static, [ProtocolVersion]> { + std::borrow::Cow::Borrowed(ProtocolVersion::known_up_to( + &ProtocolVersion::LATEST_WITH_INITIALIZE, + )) + } + fn get_info(&self) -> ServerConfig { let sql_dialect = "\n\ \n\ @@ -5740,7 +5760,8 @@ fn replace_via_staging( swap_err.code, format!( "copy_query replace failed ({swap_err}) and the original table could not be restored ({restore_err}); \ - your data is preserved in hidden table '{backup_name}'" + your data may still be in hidden table '{backup_name}' (for a persistent or attached \ + target; an ephemeral/local target's hidden table is discarded when the session ends)" ), )), }; diff --git a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs index 537b64a4..8af86c34 100644 --- a/hyperdb-mcp/tests/end_to_end_mcp_tests.rs +++ b/hyperdb-mcp/tests/end_to_end_mcp_tests.rs @@ -4776,3 +4776,73 @@ async fn tool_load_file_arrow_decimal256_out_of_range_is_clean_error() -> TestRe ); h.shutdown().await } + +/// Claude Code rejects a server that answers `initialize` with, or lists, +/// the stateless-lifecycle revision `2026-07-28`. The server must cap its +/// protocol versions at rmcp's `LATEST_WITH_INITIALIZE` revision. +#[tokio::test] +async fn initialize_never_negotiates_protocol_2026_07_28() -> TestResult { + use rmcp::ServerHandler; + use rmcp::model::ProtocolVersion; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + let server = HyperMcpServer::with_no_daemon(None, true, true); + let advertised: Vec = server + .supported_protocol_versions() + .iter() + .map(|v| v.as_str().to_string()) + .collect(); + assert!( + !advertised.iter().any(|v| v == "2026-07-28"), + "advertised versions must not include 2026-07-28: {advertised:?}" + ); + assert!( + advertised + .iter() + .any(|v| v == ProtocolVersion::LATEST_WITH_INITIALIZE.as_str()), + "LATEST_WITH_INITIALIZE must stay supported: {advertised:?}" + ); + + // Raw JSON-RPC handshake: a client asking for 2026-07-28 must be answered + // with a version at or below LATEST (2025-11-25; ISO dates sort lexically). + let (server_io, client_io) = tokio::io::duplex(64 * 1024); + let server_task = tokio::spawn(async move { + if let Ok(running) = server.serve(server_io).await { + let _ = running.waiting().await; + } + }); + let (read_half, mut write_half) = tokio::io::split(client_io); + let init = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2026-07-28", + "capabilities": {}, + "clientInfo": {"name": "version-cap-test", "version": "0"} + } + }); + write_half.write_all(format!("{init}\n").as_bytes()).await?; + write_half.flush().await?; + + let mut line = String::new(); + tokio::time::timeout( + Duration::from_secs(20), + BufReader::new(read_half).read_line(&mut line), + ) + .await??; + let response: serde_json::Value = serde_json::from_str(line.trim())?; + let negotiated = response["result"]["protocolVersion"] + .as_str() + .ok_or_else(|| format!("initialize response had no protocolVersion: {response}"))?; + assert!( + negotiated <= ProtocolVersion::LATEST_WITH_INITIALIZE.as_str(), + "negotiated {negotiated}, expected <= {}", + ProtocolVersion::LATEST_WITH_INITIALIZE.as_str() + ); + assert_ne!(negotiated, "2026-07-28"); + + drop(write_half); + server_task.abort(); + Ok(()) +} From d12f819f1071de11fbcbee6a38728cdfca7d85f6 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 22:01:48 -0700 Subject: [PATCH 48/61] fix(mcp): answer the server/discover probe with method-not-found so Claude Code falls back to initialize --- hyperdb-mcp/CHANGELOG.md | 1 + hyperdb-mcp/src/main.rs | 58 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index c0cacb37..0cd74e16 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -91,6 +91,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- The stdio server now answers Claude Code's opening `server/discover` probe with `-32601 Method not found`, so the client falls back to `initialize` instead of failing with "Server's protocol version is not supported: 2026-07-28". Previously rmcp either served the probe, locking the session into the stateless `2026-07-28` lifecycle, or rejected it with `-32022`. - **Version takeover starts the new daemon reliably.** A newer client used to spawn the replacement as soon as the old daemon's health channel went quiet, while the old `hyperd` still held its socket; the new daemon then died diff --git a/hyperdb-mcp/src/main.rs b/hyperdb-mcp/src/main.rs index 122b11dc..0f06527b 100644 --- a/hyperdb-mcp/src/main.rs +++ b/hyperdb-mcp/src/main.rs @@ -287,7 +287,12 @@ async fn run_mcp_mode(cli: Cli) -> Result<(), Box> { server }) .await?; - let service = server.serve(rmcp::transport::io::stdio()).await?; + let transport = NoDiscover(rmcp::transport::IntoTransport::< + rmcp::service::RoleServer, + _, + _, + >::into_transport(rmcp::transport::io::stdio())); + let service = server.serve(transport).await?; service.waiting().await?; Ok(()) @@ -415,3 +420,54 @@ fn daemon_status() { println!(" Started: {}", info.started_at); println!(" Version: {}", info.version); } + +/// Stdio transport that answers `server/discover` itself with `-32601`. +/// +/// Claude Code opens with a `server/discover` probe naming protocol +/// `2026-07-28`. rmcp would either serve it, which locks the session into the +/// stateless lifecycle and makes `initialize` answer `2026-07-28`, or reject it +/// with `-32022`; both end in the client's "Server's protocol version is not +/// supported". A server that predates `server/discover` answers `Method not +/// found`, which makes the client fall back to `initialize`, so rmcp never sees +/// the probe. +struct NoDiscover(T); + +impl> + rmcp::transport::Transport for NoDiscover +{ + type Error = T::Error; + + fn send( + &mut self, + item: rmcp::model::ServerJsonRpcMessage, + ) -> impl std::future::Future> + Send + 'static { + self.0.send(item) + } + + async fn receive(&mut self) -> Option { + loop { + let message = self.0.receive().await?; + if let rmcp::model::ClientJsonRpcMessage::Request(request) = &message + && matches!( + request.request, + rmcp::model::ClientRequest::DiscoverRequest(_) + ) + { + let error = rmcp::model::ErrorData::method_not_found::< + rmcp::model::DiscoverRequestMethod, + >(); + let reply = + rmcp::model::ServerJsonRpcMessage::error(error, Some(request.id.clone())); + if self.0.send(reply).await.is_err() { + return None; + } + continue; + } + return Some(message); + } + } + + fn close(&mut self) -> impl std::future::Future> + Send { + self.0.close() + } +} From dfc0218b72d16143dc9694968b2fe80c41f5b260 Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 22:07:39 -0700 Subject: [PATCH 49/61] fix(node): cap the pool under concurrent acquire and correct the hasData, write-type and API-summary docs --- docs/NODEJS_API_SUMMARY.md | 17 +++++++++-------- hyperdb-api-node/CHANGELOG.md | 7 +++++++ hyperdb-api-node/README.md | 15 +++++++++------ hyperdb-api-node/__test__/smoke.mjs | 25 +++++++++++++++++++++++++ hyperdb-api-node/index.d.ts | 2 +- hyperdb-api-node/pool.mjs | 15 +++++++++++---- hyperdb-api-node/src/arrow_inserter.rs | 2 +- 7 files changed, 63 insertions(+), 20 deletions(-) diff --git a/docs/NODEJS_API_SUMMARY.md b/docs/NODEJS_API_SUMMARY.md index 5e2edd25..5a8de285 100644 --- a/docs/NODEJS_API_SUMMARY.md +++ b/docs/NODEJS_API_SUMMARY.md @@ -63,12 +63,14 @@ graph TB ## Core API - **`HyperProcess`** — manages local `hyperd` server; properties: `endpoint`, `isOpen`, `logPath`; `connectToDatabase()` convenience method -- **`Connection`** — async database connection; static factories: `connect()`, `connectWithAuth()`, `withoutDatabase()` -- **`ConnectionBuilder`** — fluent builder: `database()`, `createMode()`, `user()`, `password()`, `loginTimeout()`, `build()` +- **`Connection`** — async database connection; static factories: `connect()`, `connectWithAuth()`, `withoutDatabase()`; `isAlive`, `isTls`, `database`, `close()` +- **`ConnectionBuilder`** — fluent builder: `database()`, `createMode()`, `user()`, `password()`, `loginTimeout()`, `tls()`, `build()` - **`Catalog`** — DDL operations: `createSchema/Table`, `dropSchema/Table`, `getTableNames`, `getSchemaNames`, `hasTable`, `hasSchema`, `createTableIfNotExists`, `dropTableIfExists` - **`TableDefinition`** — schema definition: `addColumn(name, type, nullable)`, `withSchema()`, `toCreateSql()`, `getColumns()` - **`SqlType`** — 18 factory methods: `bool`, `smallInt`, `int`, `bigInt`, `float`, `double`, `numeric(p,s)`, `text`, `varchar`, `char`, `bytes`, `date`, `time`, `timestamp`, `timestampTz`, `interval`, `json`, `geography` -- **`Inserter`** — bulk loader via COPY protocol: `addRow()`, `addRows()`, `addColumnar()`, `execute()` +- **`RowInserter`** — bulk loader via COPY protocol: `new RowInserter(conn, tableDef)`, `addRow()`, `addRows()`, `addColumnar()`, `bufferedRowCount`, `execute()` +- **`ArrowInserter`** — Arrow IPC bulk loader: `ArrowInserter.create(conn, tableDef)`, `insertRaw(buf)`, `execute()`, `cancel()`, `hasData()`, `totalBytes()`, `chunkCount()` +- **`PreparedStatement`** — from `conn.prepare(sql)` / `conn.prepareTyped(sql, oids)`: `query()`, `execute()`, `fetchOne()`, `fetchOptional()`, `getSchema()`, `close()`; properties `paramCount`, `sql` - **`CreateMode`** — `DoNotCreate`, `Create`, `CreateIfNotExists`, `CreateAndReplace` ## Query APIs (by performance tier) @@ -82,16 +84,15 @@ graph TB **Result types:** -- **`RowData`** — typed accessors: `getInt32`, `getInt64`, `getBigInt`, `getFloat64`, `getString`, `getBool`, `getBytes`, `getDateMs`, `getTimestampMs`, `getJson`, `isNull`; `toJSON()` with `setColumnNames()` for schema-aware serialization +- **`RowData`** — typed accessors: `getInt32`, `getInt64`, `getBigInt`, `getFloat64`, `getString`, `getBool`, `getBytes`, `getDateMs`, `getTimestampMs`, `getJson`, `isNull`; `columnCount` - **`QueryStream`** — `nextChunk()` + `for await...of` async iteration -- **`ColumnarStream`** / **`ColumnarChunk`** — typed arrays per column: `getInt32Column`, `getFloat64Column`, `getStringColumn`, `getNulls` +- **`ColumnarStream`** / **`ColumnarChunk`** — typed arrays per column: `getInt32Column`, `getFloat64Column`, `getStringColumn`, `getInt64Column`, `getNulls`; `rowCount`, `columnCount` ## Key Features - **Tagged template literals** — `` conn.sql`SELECT * FROM t WHERE id = ${id}` `` / `` conn.command`...` `` — SQL-injection-safe, no `$1` boilerplate -- **Parameterized queries** — `executeQueryParams(sql, params)` / `executeCommandParams(sql, params)` with `$1`/`$2` syntax -- **`ConnectionPool`** (from `pool.mjs`) — `min`/`max` connections, idle timeout, acquire timeout; shortcuts: `pool.query()`, `pool.queryParams()`, `pool.use(fn)`; stats: `size`, `idle`, `active`, `pending` -- **Query event hooks** — `conn.on('query', ({sql, durationMs, rowCount, type}) => ...)` for logging/metrics +- **Parameterized queries** — `conn.prepare(sql)` returns a server-side `PreparedStatement` with `$1`/`$2` placeholders (`query(params)` / `execute(params)`); `conn.prepareTyped(sql, paramOids)` pins parameter types +- **`ConnectionPool`** (from `pool.mjs`) — `min`/`max` connections, idle timeout, acquire timeout; shortcuts: `pool.query()`, `pool.queryParams()`, `pool.command()`, `pool.use(fn)`; manual `acquire()` / `release()`; stats: `size`, `idle`, `active`, `pending` - **Query statistics** — `conn.enableQueryStats(logPath)` / `conn.lastQueryStats()` for Hyper engine metrics (parse, compile, execute times, memory, storage I/O) - **Resource management** — `Symbol.asyncDispose` / `Symbol.dispose` for `await using` (Node 22+) - **Tableau compatible** — `createExtractTable(conn, name, columns)` helper for the Extract schema convention diff --git a/hyperdb-api-node/CHANGELOG.md b/hyperdb-api-node/CHANGELOG.md index dfa5c73b..582b383a 100644 --- a/hyperdb-api-node/CHANGELOG.md +++ b/hyperdb-api-node/CHANGELOG.md @@ -64,6 +64,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +* `ConnectionPool.acquire()` no longer exceeds `max` under concurrent calls: + in-flight connection attempts now reserve a slot (released on failure), where + before only established connections were counted. +* Corrected the `ArrowInserter.hasData()` doc comment, which said "true until + any data has been sent"; it returns true once data has been sent. The README + write-type table now matches `RowInserter` behavior (only `Buffer` is sent as + bytes; `Date` and plain objects are string-coerced, not ISO/JSON-converted). * `NUMERIC` columns are now decoded correctly. Previously the bindings read numerics with `getF64`, which reinterpreted the raw unscaled-integer bytes as an IEEE-754 double and returned garbage values or `NaN`. Numerics are now diff --git a/hyperdb-api-node/README.md b/hyperdb-api-node/README.md index 53a4d25d..f200ecee 100644 --- a/hyperdb-api-node/README.md +++ b/hyperdb-api-node/README.md @@ -356,17 +356,20 @@ for await (const row of conn.executeQueryStream('SELECT * FROM big_table')) { | JSON | `getJson(i)` -> `JSON.parse()` | `string` | | Any type | `getString(i)` | `string` (fallback) | -**Writing (JS -> Hyper) — via RowInserter or tagged templates:** +**Writing (JS -> Hyper) via `RowInserter`:** -| JS Type | SQL Type | +`RowInserter.addRow()` accepts `number`, `string`, `boolean`, `null`/`undefined`, and `Buffer`. Values are encoded as follows (see `js_value_to_insert_value` in `src/inserter.rs`): + +| JS Type | Encoded as | |---|---| -| `number` | INT / BIGINT / DOUBLE (auto-detected) | +| `number` | Integer-valued and within `i32` range: INT; integer-valued within `i64` range: BIGINT; otherwise DOUBLE. Converted to the destination column's numeric type (SMALLINT / INT / BIGINT / FLOAT / DOUBLE); an integer that does not fit the column is rejected | | `string` | TEXT | | `boolean` | BOOLEAN | | `null` / `undefined` | NULL | -| `Buffer` | BYTEA (`RowInserter` only) | -| `Date` | TIMESTAMP (via `toISOString()`) | -| `{}` (plain object) | JSON (via `JSON.stringify()`) | +| `Buffer` | BYTEA (the only value sent as raw bytes) | +| Anything else (`Date`, plain objects, arrays, ...) | Coerced with JS `String(value)` and sent as TEXT. A `Date` is **not** converted with `toISOString()` and a plain object is **not** serialized as JSON, so `String(date)` / `"[object Object]"` is sent. Convert explicitly first, e.g. `date.toISOString()` or `JSON.stringify(obj)`, and use a TEXT column (or cast after loading) | + +**Writing via tagged templates / prepared statements:** `` conn.sql`...` `` and `` conn.command`...` `` accept `number`, `bigint`, `string`, `boolean`, `null`, and `Date`. A `Date` is sent as its UTC `toISOString()` value with `T` replaced by a space, so it works for both TIMESTAMP and TIMESTAMPTZ. `Buffer` parameters are rejected (stage BYTEA data via `RowInserter`/COPY). ### Tagged Template Literals diff --git a/hyperdb-api-node/__test__/smoke.mjs b/hyperdb-api-node/__test__/smoke.mjs index 500b9616..560da403 100644 --- a/hyperdb-api-node/__test__/smoke.mjs +++ b/hyperdb-api-node/__test__/smoke.mjs @@ -230,6 +230,31 @@ async function main() { console.log(` Concurrent queries: ${concurrent.length} completed`); console.log(` Pool stats: size=${pool.size}, idle=${pool.idle}, active=${pool.active}`); + // Burst of concurrent acquires on a cold pool must never exceed max + // (in-flight connects have to count against the limit). + const burstPool = new ConnectionPool(hyper.endpoint, join(TEST_DIR, 'smoke_test.hyper'), { + min: 0, + max: 3, + idleTimeoutMs: 0, + createMode: CreateMode.DoNotCreate, + }); + let peakActive = 0; + let peakSize = 0; + await Promise.all( + Array.from({ length: 12 }, () => + burstPool.use(async (c) => { + peakActive = Math.max(peakActive, burstPool.active); + peakSize = Math.max(peakSize, burstPool.size); + await c.executeQuery('SELECT 1'); + }) + ) + ); + assert(peakActive <= 3, `pool active ${peakActive} exceeded max 3`); + assert(peakSize <= 3, `pool size ${peakSize} exceeded max 3`); + assert(burstPool.size <= 3, `pool size ${burstPool.size} exceeded max 3 after burst`); + await burstPool.close(); + console.log(` Concurrent acquire burst respects max (peak active=${peakActive}) ✓`); + await pool.close(); console.log(' Pool closed ✓'); diff --git a/hyperdb-api-node/index.d.ts b/hyperdb-api-node/index.d.ts index caaa36aa..0973ea30 100644 --- a/hyperdb-api-node/index.d.ts +++ b/hyperdb-api-node/index.d.ts @@ -175,7 +175,7 @@ export declare class ArrowInserter { execute(): Promise /** Cancels the COPY stream without committing any rows. */ cancel(): Promise - /** True until any data has been sent; useful for conditional flushes. */ + /** True once any data (the schema or a record batch) has been sent; useful for conditional flushes. */ hasData(): Promise /** Total bytes sent to the server so far. */ totalBytes(): Promise diff --git a/hyperdb-api-node/pool.mjs b/hyperdb-api-node/pool.mjs index f7345e05..514d5f42 100644 --- a/hyperdb-api-node/pool.mjs +++ b/hyperdb-api-node/pool.mjs @@ -49,6 +49,7 @@ export class ConnectionPool { #idle = []; // { conn, lastUsed } #active = new Set(); #waiting = []; // { resolve, reject, timer } waiting for a connection + #connecting = 0; // in-flight #connect() calls holding a reserved slot #closed = false; #idleTimer = null; #acquireTimeoutMs; @@ -144,10 +145,16 @@ export class ConnectionPool { } // Create a new connection if below max - if (this.size < this.#max) { - const conn = await this.#connect(); - this.#active.add(conn); - return conn; + // Reserve the slot before awaiting so concurrent acquires cannot overshoot max. + if (this.size + this.#connecting < this.#max) { + this.#connecting++; + try { + const conn = await this.#connect(); + this.#active.add(conn); + return conn; + } finally { + this.#connecting--; + } } // At max capacity — wait for a release (with timeout) diff --git a/hyperdb-api-node/src/arrow_inserter.rs b/hyperdb-api-node/src/arrow_inserter.rs index c367332f..3ffda2c7 100644 --- a/hyperdb-api-node/src/arrow_inserter.rs +++ b/hyperdb-api-node/src/arrow_inserter.rs @@ -101,7 +101,7 @@ impl ArrowInserter { Ok(()) } - /// True until any data has been sent; useful for conditional flushes. + /// True once any data (the schema or a record batch) has been sent; useful for conditional flushes. #[napi] pub async fn has_data(&self) -> bool { let guard = self.inner.lock().await; From 1dce10f874611ba37f14565ea4250553b643936f Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 22:12:04 -0700 Subject: [PATCH 50/61] fix(api): copy_database uses CREATE DATABASE FROM and close() skips DETACH on gRPC --- hyperdb-api/CHANGELOG.md | 1 + hyperdb-api/src/async_connection.rs | 22 ++++-- hyperdb-api/src/connection.rs | 29 +++++--- hyperdb-api/tests/copy_database_tests.rs | 71 +++++++++++++++++++ hyperdb-api/tests/grpc_close_tests.rs | 58 +++++++++++++++ hyperdb-api/tests/remaining_features_tests.rs | 38 +--------- 6 files changed, 166 insertions(+), 53 deletions(-) create mode 100644 hyperdb-api/tests/copy_database_tests.rs create mode 100644 hyperdb-api/tests/grpc_close_tests.rs diff --git a/hyperdb-api/CHANGELOG.md b/hyperdb-api/CHANGELOG.md index 8fac310a..da8075a6 100644 --- a/hyperdb-api/CHANGELOG.md +++ b/hyperdb-api/CHANGELOG.md @@ -124,6 +124,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- `Connection::copy_database` and `AsyncConnection::copy_database` now emit `CREATE DATABASE FROM `; the `COPY DATABASE ... TO ...` they sent before is not Hyper grammar and always failed. The source must not be attached read-write. `close()` on a gRPC connection that has a database no longer tries `DETACH DATABASE` (rejected as read-only), so it no longer returns a spurious error. - Cancelling or dropping an `Inserter` after an auto-flush no longer leaves the connection mid-COPY: `cancel` now sends `CopyFail` and returns the connection to `ReadyForQuery`. `import_text` also cancels the COPY when reading the source fails. - `MappedInserter` no longer deadlocks on `cancel` after an auto-flush, and its staging table is dropped on every path (execute, cancel, error, drop). Staging tables are named `_hyper_staging_{pid}_{n}`, so two mapped inserters on one connection no longer collide. - On the sync path, `fetch_one`, `fetch_optional`, `fetch_scalar`, `fetch_optional_scalar`, `query_count`, `first_row` and the `QueryAs`/`QueryScalar` single-row fetchers used to leave the unread remainder of a multi-chunk (more than 65,536 rows) result on the wire, so the next request failed with "connection is desynchronized". They now read only the first chunk and drop the stream, whose `Drop` cancels the query and discards the backlog. diff --git a/hyperdb-api/src/async_connection.rs b/hyperdb-api/src/async_connection.rs index f966e7a1..b9515140 100644 --- a/hyperdb-api/src/async_connection.rs +++ b/hyperdb-api/src/async_connection.rs @@ -861,18 +861,22 @@ impl AsyncConnection { Ok(()) } - /// Copies a database file to a new path (async). + /// Copies a database file to a new path (async) using + /// `CREATE DATABASE FROM `. + /// + /// The source must not be attached read-write (see + /// [`Connection::copy_database`](crate::Connection::copy_database)). /// /// # Errors /// /// Returns [`Error::Server`] if the server rejects the - /// `COPY DATABASE` statement — e.g. the source is not attached or the - /// destination path is not writable. + /// `CREATE DATABASE ... FROM ...` statement — e.g. the source is still + /// attached read-write or the destination path is not writable. pub async fn copy_database(&self, source: &str, destination: &str) -> Result<()> { let sql = format!( - "COPY DATABASE {} TO {}", - escape_sql_path(source), - escape_sql_path(destination) + "CREATE DATABASE {} FROM {}", + escape_sql_path(destination), + escape_sql_path(source) ); self.execute_command(&sql).await?; Ok(()) @@ -1106,7 +1110,11 @@ impl AsyncConnection { /// attached database could not be detached but the transport close /// itself succeeded. pub async fn close(self) -> Result<()> { - let detach_err = if let Some(ref db_path) = self.database { + // gRPC is read-only and stateless: nothing was attached by this + // connection, and `DETACH DATABASE` would be rejected, so skip it. + let detach_err = if let (Some(db_path), true) = + (self.database.as_ref(), self.transport.supports_writes()) + { let db_alias = std::path::Path::new(db_path) .file_stem() .and_then(|s| s.to_str()) diff --git a/hyperdb-api/src/connection.rs b/hyperdb-api/src/connection.rs index e3d76fb9..ce6fbd6a 100644 --- a/hyperdb-api/src/connection.rs +++ b/hyperdb-api/src/connection.rs @@ -1672,9 +1672,13 @@ impl Connection { crate::ServerVersion::parse(&version_str) } - /// Copies a database file to a new path. + /// Copies a database file to a new path using + /// `CREATE DATABASE FROM `. /// - /// The source database must be attached to this connection. + /// The source database file must **not** be attached read-write: `hyperd` + /// rejects the copy while the source is loaded in `readwrite` mode, so + /// detach it (or copy it from a connection that never attached it) first. + /// The destination is created but not attached. /// /// # Example /// @@ -1683,7 +1687,8 @@ impl Connection { /// /// fn main() -> Result<()> { /// let hyper = HyperProcess::new(None, None)?; - /// let conn = Connection::new(&hyper, "source.hyper", CreateMode::DoNotCreate)?; + /// // A connection to a scratch database; `source.hyper` is not attached. + /// let conn = Connection::new(&hyper, "scratch.hyper", CreateMode::CreateIfNotExists)?; /// conn.copy_database("source.hyper", "backup.hyper")?; /// Ok(()) /// } @@ -1692,13 +1697,14 @@ impl Connection { /// # Errors /// /// Returns [`Error::Server`] if the server rejects the - /// `COPY DATABASE` statement — e.g. the source is not attached, the - /// destination path is not writable, or it already exists. + /// `CREATE DATABASE ... FROM ...` statement — e.g. the source is still + /// attached read-write, the destination path is not writable, or it + /// already exists. pub fn copy_database(&self, source: &str, destination: &str) -> Result<()> { let sql = format!( - "COPY DATABASE {} TO {}", - escape_sql_path(source), - escape_sql_path(destination) + "CREATE DATABASE {} FROM {}", + escape_sql_path(destination), + escape_sql_path(source) ); self.execute_command(&sql)?; Ok(()) @@ -2011,7 +2017,12 @@ impl Connection { pub fn close(self) -> Result<()> { // Detach the attached database to ensure files are flushed and released. // Always attempt close, even if detach fails. - let detach_err = if let Some(ref db_path) = self.database { + // gRPC is read-only and stateless: nothing was attached by this + // connection, and `DETACH DATABASE` would be rejected, so skip it. + let detach_err = if let (Some(db_path), false) = ( + self.database.as_ref(), + matches!(self.transport, Transport::Grpc(_)), + ) { let db_alias = std::path::Path::new(db_path) .file_stem() .and_then(|s| s.to_str()) diff --git a/hyperdb-api/tests/copy_database_tests.rs b/hyperdb-api/tests/copy_database_tests.rs new file mode 100644 index 00000000..1b9e6c59 --- /dev/null +++ b/hyperdb-api/tests/copy_database_tests.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! `copy_database` (sync + async) emits `CREATE DATABASE FROM `. +//! Hyper has no `COPY DATABASE` statement. + +mod common; + +use common::test_result_path; +use hyperdb_api::{AsyncConnection, Connection, CreateMode, HyperProcess, Result}; + +/// Creates a populated source database (closed afterwards) and returns +/// `(source, destination)` paths with the destination absent. +fn make_source(hyper: &HyperProcess, name: &str) -> Result<(String, String)> { + let src = test_result_path(&format!("{name}_src"), "hyper")? + .to_string_lossy() + .to_string(); + let dst = test_result_path(&format!("{name}_dst"), "hyper")? + .to_string_lossy() + .to_string(); + let _ = std::fs::remove_file(&dst); + let conn = Connection::new(hyper, &src, CreateMode::CreateAndReplace)?; + conn.execute_command("CREATE TABLE t (id INT NOT NULL)")?; + conn.execute_command("INSERT INTO t VALUES (1), (2), (3)")?; + conn.close()?; + Ok((src, dst)) +} + +fn count_in_copy(hyper: &HyperProcess, dst: &str) -> Result { + let conn = Connection::new(hyper, dst, CreateMode::DoNotCreate)?; + let n: i64 = conn.fetch_scalar("SELECT count(*) FROM t")?; + conn.close()?; + Ok(n) +} + +#[test] +fn copy_database_sync_copies_data() -> Result<()> { + let hyper = HyperProcess::new(None, None)?; + let (src, dst) = make_source(&hyper, "copy_db_sync")?; + let scratch = test_result_path("copy_db_sync_scratch", "hyper")? + .to_string_lossy() + .to_string(); + let conn = Connection::new(&hyper, &scratch, CreateMode::CreateAndReplace)?; + conn.copy_database(&src, &dst)?; + assert!(std::path::Path::new(&dst).exists()); + assert_eq!(count_in_copy(&hyper, &dst)?, 3); + conn.close()?; + for f in [&src, &dst, &scratch] { + let _ = std::fs::remove_file(f); + } + Ok(()) +} + +#[tokio::test] +async fn copy_database_async_copies_data() -> Result<()> { + let hyper = HyperProcess::new(None, None)?; + let (src, dst) = make_source(&hyper, "copy_db_async")?; + let scratch = test_result_path("copy_db_async_scratch", "hyper")? + .to_string_lossy() + .to_string(); + let endpoint = hyper.require_endpoint()?.to_string(); + let conn = AsyncConnection::connect(&endpoint, &scratch, CreateMode::CreateAndReplace).await?; + conn.copy_database(&src, &dst).await?; + assert!(std::path::Path::new(&dst).exists()); + assert_eq!(count_in_copy(&hyper, &dst)?, 3); + conn.close().await?; + for f in [&src, &dst, &scratch] { + let _ = std::fs::remove_file(f); + } + Ok(()) +} diff --git a/hyperdb-api/tests/grpc_close_tests.rs b/hyperdb-api/tests/grpc_close_tests.rs new file mode 100644 index 00000000..a5b38fd2 --- /dev/null +++ b/hyperdb-api/tests/grpc_close_tests.rs @@ -0,0 +1,58 @@ +// Copyright (c) 2026, Salesforce, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! `close()` on a gRPC connection that has a database must not try to +//! `DETACH DATABASE` (gRPC is read-only, so the server would reject it and +//! `close()` would return an error). + +mod common; + +use common::test_result_path; +use hyperdb_api::{ + AsyncConnection, Connection, CreateMode, HyperProcess, ListenMode, Parameters, Result, +}; + +/// Creates a database file via TCP, then starts a gRPC-only `hyperd`. +fn grpc_hyperd_with_db(name: &str) -> Result<(HyperProcess, String, String)> { + let db = test_result_path(name, "hyper")? + .to_string_lossy() + .to_string(); + { + let tcp = HyperProcess::new(None, None)?; + let conn = Connection::new(&tcp, &db, CreateMode::CreateAndReplace)?; + conn.execute_command("CREATE TABLE t (id INT NOT NULL)")?; + conn.close()?; + } + let mut params = Parameters::new(); + params.set("log_dir", "test_results"); + params.set_listen_mode(ListenMode::Grpc { port: 0 }); + let hyper = HyperProcess::new(None, Some(¶ms))?; + let url = hyper.grpc_url().expect("gRPC URL"); + Ok((hyper, url, db)) +} + +#[test] +fn grpc_close_with_database_succeeds_sync() -> Result<()> { + let (_hyper, url, db) = grpc_hyperd_with_db("grpc_close_sync")?; + let conn = Connection::builder(&url) + .database(&db) + .create_mode(CreateMode::DoNotCreate) + .build()?; + assert_eq!(conn.transport_type(), "gRPC"); + conn.close()?; + let _ = std::fs::remove_file(&db); + Ok(()) +} + +#[tokio::test] +async fn grpc_close_with_database_succeeds_async() -> Result<()> { + let (_hyper, url, db) = grpc_hyperd_with_db("grpc_close_async")?; + let conn = AsyncConnection::builder(&url) + .database(&db) + .create_mode(CreateMode::DoNotCreate) + .build() + .await?; + conn.close().await?; + let _ = std::fs::remove_file(&db); + Ok(()) +} diff --git a/hyperdb-api/tests/remaining_features_tests.rs b/hyperdb-api/tests/remaining_features_tests.rs index 1cdeeb4a..4bc629d7 100644 --- a/hyperdb-api/tests/remaining_features_tests.rs +++ b/hyperdb-api/tests/remaining_features_tests.rs @@ -5,7 +5,7 @@ //! - #7: Batch statement execution (`execute_batch`) //! - #12: Catalog metadata helpers (`get_row_count`, `get_column_names`, `get_database_names`) //! - #15: Server version struct (`ServerVersion`) -//! - #14: Database copy (`copy_database`) +//! - #14: Database copy (`copy_database`) — see `copy_database_tests.rs` //! - #13: EXPLAIN (explain, `explain_analyze`) //! - #11: Connection timeouts (`query_timeout`, `application_name` on `ConnectionBuilder`) //! - #17: `FromRow` trait (`fetch_one_as`, `fetch_all_as`) @@ -167,42 +167,6 @@ fn test_server_version_from_connection() { } } -// ============================================================================= -// #14: Database Copy -// ============================================================================= - -#[test] -fn test_copy_database() { - let test = TestConnection::new().expect("Failed to create test connection"); - - // Create some data in the source database - test.execute_command("CREATE TABLE copy_src (id INT NOT NULL)") - .expect("create"); - test.execute_command("INSERT INTO copy_src VALUES (1), (2), (3)") - .expect("insert"); - - // Get the source database path and construct a destination path - let src_path = test.database_path.to_string_lossy().to_string(); - let dst_path = src_path.replace(".hyper", "_backup.hyper"); - - // Remove destination if it exists from a previous run - let _ = std::fs::remove_file(&dst_path); - - // Copy the database - let result = test.connection.copy_database(&src_path, &dst_path); - // COPY DATABASE may not be supported in all Hyper versions - if let Ok(()) = result { - // Verify the backup exists - assert!( - std::path::Path::new(&dst_path).exists(), - "Backup file should exist" - ); - // Clean up - let _ = std::fs::remove_file(&dst_path); - } - // If it fails, that's OK — COPY DATABASE may not be available -} - // ============================================================================= // #13: EXPLAIN // ============================================================================= From 9c5e5919adeac3388723999b084e678eb808f23f Mon Sep 17 00:00:00 2001 From: Stefan Steiner Date: Sat, 10 Oct 2026 22:12:04 -0700 Subject: [PATCH 51/61] fix(mcp): render BYTES, TIME and INTERVAL values in query results --- hyperdb-mcp/CHANGELOG.md | 1 + hyperdb-mcp/src/engine.rs | 33 ++++++++++++++++++++++++++++--- hyperdb-mcp/tests/engine_tests.rs | 23 +++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/hyperdb-mcp/CHANGELOG.md b/hyperdb-mcp/CHANGELOG.md index 0cd74e16..3b7428fb 100644 --- a/hyperdb-mcp/CHANGELOG.md +++ b/hyperdb-mcp/CHANGELOG.md @@ -91,6 +91,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- `query` results now render `BYTES` as a `\x`-prefixed lowercase hex string and `TIME`/`INTERVAL` as their text form instead of null/garbage from the string fallback. - The stdio server now answers Claude Code's opening `server/discover` probe with `-32601 Method not found`, so the client falls back to `initialize` instead of failing with "Server's protocol version is not supported: 2026-07-28". Previously rmcp either served the probe, locking the session into the stateless `2026-07-28` lifecycle, or rejected it with `-32022`. - **Version takeover starts the new daemon reliably.** A newer client used to spawn the replacement as soon as the old daemon's health channel went quiet, diff --git a/hyperdb-mcp/src/engine.rs b/hyperdb-mcp/src/engine.rs index c933dece..f2507e88 100644 --- a/hyperdb-mcp/src/engine.rs +++ b/hyperdb-mcp/src/engine.rs @@ -1876,11 +1876,14 @@ impl Engine { /// | `NUMERIC` | number when representable as a finite `f64`, else string | /// | `DATE` | ISO 8601 date string (`YYYY-MM-DD`) | /// | `TIMESTAMP` / `TIMESTAMP_TZ` | ISO 8601 timestamp string | +/// | `TIME` | `HH:MM:SS[.ffffff]` string (the type's `Display` form) | +/// | `INTERVAL` | interval text, e.g. `1 mon 2 days 03:04:05` (the type's `Display` form) | +/// | `BYTE_A` (`BYTES`) | PostgreSQL-style hex string: `\x` followed by lowercase hex digits | /// | `TEXT` / `VARCHAR` | string | -/// | anything else | string (fallback; may be garbage for binary types) | +/// | anything else | string (fallback; may be garbage for other binary types) | fn row_value_to_json(row: &hyperdb_api::Row, idx: usize, sql_type: &SqlType) -> Value { use hyperdb_api::oids; - use hyperdb_api::{Date, Numeric, OffsetTimestamp, Timestamp}; + use hyperdb_api::{Date, Interval, Numeric, OffsetTimestamp, Time, Timestamp}; if row.is_null(idx) { return Value::Null; @@ -1955,11 +1958,35 @@ fn row_value_to_json(row: &hyperdb_api::Row, idx: usize, sql_type: &SqlType) -> .get::(idx) .map_or(Value::Null, |t| Value::String(t.to_string())); } + if oid_val == oids::TIME.0 { + return row + .get::