From 8db58ec7ddbfbe29d295404cb37c84c4b2b458d0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:58:05 -0800 Subject: [PATCH 01/50] fix(retention): verify and expose transactional GC progress Report candidate discharge and replay SQL frontiers for each scanned page. Reject incomplete candidate discharge or nonadvancing replay pages. Add >48-row behavioral regressions for zero-reclamation backlogs. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 28 ++++++++--- .../observation/canonical_body_gc/tests.rs | 49 +++++++++++++++++++ .../src/retention/replay_metadata.rs | 13 +++++ .../src/retention/replay_metadata/tests.rs | 34 +++++++++++++ crates/tracedecay/src/daemon/maintenance.rs | 18 +++++++ 5 files changed, 135 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 6c5af50fea3..7b5c0b056f4 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -25,6 +25,8 @@ pub const MAX_CANONICAL_BODY_GC_ROWS: usize = 48; pub struct CanonicalBodyGcReport { pub roots_scanned: u64, pub bodies_scanned: u64, + /// Candidates discharged in the committed page, even if their body stays live. + pub candidates_discharged: u64, pub bodies_deleted: u64, /// Stored BLOB bytes deleted, including the effect of their existing /// compression. This is not database-file or filesystem space reclaimed. @@ -323,13 +325,25 @@ async fn collect_candidates( drop(deleted); // Referenced candidates are discharged too. The final owner removal // records them again; idle ticks never revisit every retained body. - conn.execute( - "DELETE FROM canonical_body_gc_candidates - WHERE content_hash IN (SELECT value FROM json_each(?1))", - params![hashes], - ) - .await - .map_err(db_error)?; + // Deleting an unreferenced body cascades to its candidate row. The + // remaining explicit removals discharge referenced bodies that stay live. + let explicitly_discharged = conn + .execute( + "DELETE FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1))", + params![hashes], + ) + .await + .map_err(db_error)?; + let discharged = explicitly_discharged + .checked_add(report.bodies_deleted) + .ok_or_else(|| integrity_error("canonical body candidate discharge overflow"))?; + if discharged != report.bodies_scanned { + return Err(integrity_error( + "canonical body candidate page was not fully discharged", + )); + } + report.candidates_discharged = discharged; Ok(()) } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 9da4f654323..9b2accfd09e 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -506,3 +506,52 @@ async fn nonpositive_legacy_owner_ids_cannot_be_skipped_by_bootstrap() { ); } } + +#[asupersync::test] +async fn zero_delete_pages_discharge_distinct_candidates_and_quiesce() { + let store = Store::open().await; + for ordinal in 0..52 { + let body = + StoredCanonicalBody::pack(format!("still referenced {ordinal}").as_bytes()).unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + + let mut discharged_pages = Vec::new(); + for _ in 0..16 { + let report = store.tick(48).await; + assert_eq!(report.bodies_deleted, 0, "referenced bodies stay live"); + if report.candidates_discharged > 0 { + assert_eq!(report.candidates_discharged, report.bodies_scanned); + discharged_pages.push(report.candidates_discharged); + } + if !report.has_more { + break; + } + } + assert_eq!(discharged_pages, vec![48, 4]); + assert!(!has_candidates(store.conn()).await.unwrap()); + assert_eq!( + store.tick(48).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); + let mut rows = store + .conn() + .query("SELECT COUNT(*) FROM session_canonical_bodies", ()) + .await + .unwrap(); + let count: i64 = rows.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(count, 52); +} diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index b7be38726d9..72910e4798a 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -25,6 +25,9 @@ const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ReplayMetadataCompactionReport { pub rows_scanned: u64, + /// Transactional store-id frontier consumed by this page, absent on idle reads. + pub frontier_before: Option, + pub frontier_after: Option, pub rows_compacted: u64, /// Eligible rows retained because their representation, sanitizer result, /// or byte bound could not prove this narrow lossless transformation. @@ -150,6 +153,7 @@ async fn step( max_rows: usize, ) -> Result { let mut after = frontier(conn).await?; + let before = after; let candidates = candidates(conn, after, max_rows).await?; let mut report = ReplayMetadataCompactionReport::default(); let mut remaining = MAX_BATCH_BYTES; @@ -199,6 +203,15 @@ async fn step( if guard_suspended { summary_convergence::restore_raw_update_guard(conn).await?; } + if report.rows_scanned > 0 { + if after <= before { + return Err(db_error( + "replay metadata page did not advance its frontier", + )); + } + report.frontier_before = Some(before); + report.frontier_after = Some(after); + } schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; report.has_more = has_work(conn, after).await?; Ok(report) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 1bc50de1ce3..38a405796cb 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -369,3 +369,37 @@ async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_ let idle = store.tick(48).await; assert_eq!(idle, ReplayMetadataCompactionReport::default()); } + +#[asupersync::test] +async fn zero_compaction_pages_advance_distinct_durable_frontiers() { + let store = Store::open().await; + for ordinal in 1..=52 { + store + .insert( + ordinal, + "canonical source", + Value::String("different replay".to_owned()), + false, + ) + .await; + } + let first = store.tick(48).await; + assert_eq!(first.rows_scanned, 48); + assert_eq!(first.rows_compacted, 0); + assert_eq!(first.frontier_before, Some(0)); + assert_eq!(first.frontier_after, Some(48)); + assert!(first.has_more); + + let second = store.tick(48).await; + assert_eq!(second.rows_scanned, 4); + assert_eq!(second.rows_compacted, 0); + assert_eq!(second.frontier_before, Some(48)); + assert_eq!(second.frontier_after, Some(52)); + assert!(!second.has_more); + assert_eq!(frontier(&store.conn).await.unwrap(), 52); + + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); +} diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index ea2070195f9..9e293532be4 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -386,6 +386,10 @@ fn record_canonical_body_gc_outcome( &[ ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), + ( + "candidates_discharged", + report.candidates_discharged.to_string(), + ), ("bodies_deleted", report.bodies_deleted.to_string()), ( "stored_bytes_deleted", @@ -428,6 +432,20 @@ fn record_replay_metadata_outcome( "retention_replay_metadata", &[ ("rows_scanned", report.rows_scanned.to_string()), + ( + "frontier_before", + report.frontier_before.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), + ( + "frontier_after", + report.frontier_after.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), ("rows_compacted", report.rows_compacted.to_string()), ("rows_skipped", report.rows_skipped.to_string()), ( From 001b1abe47f636b05ed95c964e3cbd6f6ab5cdee Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:36:01 -0800 Subject: [PATCH 02/50] fix(retention): bound replay eligibility parsing before scanning Admit a size-bounded metadata prefix before running SQL JSON eligibility checks, including rows that cannot be compacted. Resolve admitted IDs in one query under the existing writer transaction. Preserve oversized metadata while advancing past it without parsing or blocking later rows. Expose examined metadata bytes through the page report and tracing span. Add database-backed regressions for large ineligible backlogs and an oversized stored document followed by valid compactable work. Refs #4787 --- .../src/retention/replay_metadata.rs | 95 ++++++++++--- .../src/retention/replay_metadata/tests.rs | 129 ++++++++++++++++++ 2 files changed, 208 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 72910e4798a..da43f273f78 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -5,6 +5,8 @@ //! representations keep their own content. A durable rowid frontier and a byte //! budget keep this repair off the foreground ingest path. +use std::collections::BTreeSet; + use serde::{Deserialize, Serialize}; use serde_json::Value; use tracedecay_runtime_core::db::Database; @@ -19,7 +21,8 @@ const FRONTIER_KEY: &str = "replay_metadata_compaction_store_id_v1"; const OPERATION: &str = "compact duplicate LCM replay metadata"; pub const MAX_REPLAY_METADATA_COMPACTION_ROWS: usize = 48; // Bound both receipt verification and metadata sanitization while a writer is -// held. The raw metadata envelope itself is capped at 1 MiB by ingestion. +// held. Eligibility parsing has its own equal byte budget: ineligible rows +// still cost JSON work. Ingestion caps each metadata envelope at 1 MiB. const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -29,12 +32,15 @@ pub struct ReplayMetadataCompactionReport { pub frontier_before: Option, pub frontier_after: Option, pub rows_compacted: u64, - /// Eligible rows retained because their representation, sanitizer result, - /// or byte bound could not prove this narrow lossless transformation. + /// Rows retained because their representation, sanitizer result, or byte + /// bound could not prove this narrow lossless transformation. pub rows_skipped: u64, /// Subset of skipped rows whose metadata plus canonical/stored body /// representations exceed this pass's 4 MiB read and verification budget. pub rows_skipped_oversized: u64, + /// Metadata bytes admitted to SQL eligibility parsing, including ineligible + /// rows and any prefetched rows deferred by the separate body byte budget. + pub metadata_bytes_examined: u64, /// UTF-8 metadata bytes removed. This does not measure file shrinkage. pub metadata_bytes_removed: u64, pub has_more: bool, @@ -96,22 +102,20 @@ pub async fn compact_replay_metadata( struct Candidate { store_id: i64, eligible: bool, + metadata_oversized: bool, read_bytes: Option, } -/// Read only sizes and identities until each candidate earns the byte budget. +/// Admit a size-bounded prefix before parsing any metadata. Eligibility must +/// not parse all 48 documents before the body's separate byte budget is applied. async fn candidates( conn: &(impl QueryExecutor + ?Sized), after: i64, limit: usize, -) -> Result, LcmError> { +) -> Result<(Vec, u64), LcmError> { let mut rows = conn .query( - "SELECT r.store_id, - CASE WHEN r.storage_kind = 'inline' AND json_valid(r.metadata_json) - THEN json_type(r.metadata_json, '$.lcm_active_replay') = 'true' - AND json_type(r.metadata_json, '$.active_replay.content') = 'text' - ELSE 0 END, + "SELECT r.store_id, r.storage_kind = 'inline', LENGTH(CAST(r.metadata_json AS BLOB)), COALESCE(LENGTH(CAST(r.content AS BLOB)), b.uncompressed_bytes), COALESCE(LENGTH(b.body), 0) @@ -125,10 +129,29 @@ async fn candidates( ) .await?; let mut result = Vec::new(); + let mut eligibility_ids = Vec::new(); + let mut remaining = MAX_BATCH_BYTES; while let Some(row) = rows.next().await? { + let store_id: i64 = row.get(0)?; + let inline = row.get::(1)? != 0; let metadata: Option = row.get(2)?; let body: Option = row.get(3)?; let stored: i64 = row.get(4)?; + let metadata_bytes = if inline { + u64::try_from(metadata.unwrap_or(0)).map_err(db_error)? + } else { + 0 + }; + let metadata_oversized = metadata_bytes > MAX_BATCH_BYTES; + if !metadata_oversized { + if metadata_bytes > remaining { + break; + } + remaining -= metadata_bytes; + if inline && metadata.is_some() { + eligibility_ids.push(store_id); + } + } let read_bytes = metadata .zip(body) .and_then(|(metadata, body)| { @@ -140,26 +163,66 @@ async fn candidates( }) .and_then(|(metadata, body, stored)| metadata.checked_add(body)?.checked_add(stored)); result.push(Candidate { - store_id: row.get(0)?, - eligible: row.get::>(1)?.unwrap_or(0) != 0, + store_id, + eligible: false, + metadata_oversized, read_bytes, }); } - Ok(result) + drop(rows); + // Resolve the admitted prefix in one query, not one query per raw row. + // Both reads share the writer transaction, so sizes and identities cannot + // change between admission and parsing. Oversized documents stay untouched. + if !eligibility_ids.is_empty() { + let ids = serde_json::to_string(&eligibility_ids).map_err(db_error)?; + let mut rows = conn + .query( + "SELECT store_id FROM lcm_raw_messages + WHERE store_id IN (SELECT value FROM json_each(?1)) + AND CASE WHEN storage_kind = 'inline' AND json_valid(metadata_json) + THEN json_type(metadata_json, '$.lcm_active_replay') = 'true' + AND json_type(metadata_json, '$.active_replay.content') = 'text' + ELSE 0 END", + params![ids], + ) + .await?; + let mut eligible = BTreeSet::new(); + while let Some(row) = rows.next().await? { + eligible.insert(row.get::(0)?); + } + for candidate in &mut result { + candidate.eligible = eligible.contains(&candidate.store_id); + } + } + Ok((result, MAX_BATCH_BYTES - remaining)) } +#[tracing::instrument( + name = "lcm.retention.replay_metadata.page", + level = "trace", + skip_all, + fields(metadata_bytes_examined = tracing::field::Empty), + err +)] async fn step( conn: &(impl Executor + ?Sized), max_rows: usize, ) -> Result { let mut after = frontier(conn).await?; let before = after; - let candidates = candidates(conn, after, max_rows).await?; - let mut report = ReplayMetadataCompactionReport::default(); + let (candidates, metadata_bytes_examined) = candidates(conn, after, max_rows).await?; + tracing::Span::current().record("metadata_bytes_examined", metadata_bytes_examined); + let mut report = ReplayMetadataCompactionReport { + metadata_bytes_examined, + ..ReplayMetadataCompactionReport::default() + }; let mut remaining = MAX_BATCH_BYTES; let mut guard_suspended = false; for candidate in candidates { - if candidate.eligible { + if candidate.metadata_oversized { + report.rows_skipped += 1; + report.rows_skipped_oversized += 1; + } else if candidate.eligible { let bytes = candidate.read_bytes.ok_or_else(|| { db_error(format!( "raw row {} has no bounded canonical body", diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 38a405796cb..a9b39a4c727 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -403,3 +403,132 @@ async fn zero_compaction_pages_advance_distinct_durable_frontiers() { ReplayMetadataCompactionReport::default() ); } + +#[asupersync::test] +async fn large_ineligible_metadata_is_bounded_before_eligibility_checks() { + let store = Store::open().await; + let structured = serde_json::json!([{ + "type": "text", + "text": "retained structured replay ".repeat(20_000) + }]); + let mut originals = Vec::new(); + for ordinal in 1..=9 { + let id = store + .insert(ordinal, "small verified body", structured.clone(), false) + .await; + originals.push(( + id, + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + )); + } + let total_metadata_bytes: u64 = originals + .iter() + .map(|(_, message)| message.metadata_json.as_ref().unwrap().len() as u64) + .sum(); + assert!(total_metadata_bytes > MAX_BATCH_BYTES); + assert!(originals.iter().all(|(_, message)| { + message.metadata_json.as_ref().unwrap().len() < 1024 * 1024 + })); + + let mut after = 0; + let mut examined = 0; + let mut scanned = 0; + let mut ticks = 0; + for _ in 0..originals.len() { + let report = store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await; + ticks += 1; + assert!(report.rows_scanned > 0); + assert_eq!(report.rows_compacted, 0); + assert_eq!(report.rows_skipped, 0); + assert_eq!(report.metadata_bytes_removed, 0); + assert!(report.metadata_bytes_examined > 0); + assert!(report.metadata_bytes_examined <= MAX_BATCH_BYTES); + assert_eq!(report.frontier_before, Some(after)); + let next = report.frontier_after.unwrap(); + assert!(next > after); + assert_eq!(frontier(&store.conn).await.unwrap(), next); + after = next; + examined += report.metadata_bytes_examined; + scanned += report.rows_scanned; + if !report.has_more { + break; + } + } + assert!(ticks > 1, "ineligible JSON must obey the byte budget too"); + assert_eq!(scanned, originals.len() as u64); + assert_eq!(examined, total_metadata_bytes, "no document is revisited"); + assert_eq!(after, originals.last().unwrap().0); + assert_eq!( + store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await, + ReplayMetadataCompactionReport::default() + ); + for (id, original) in originals { + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + original, + "bounded scanning must preserve the body, replay and receipts" + ); + } +} + +#[asupersync::test] +async fn oversized_metadata_is_preserved_without_blocking_later_compaction() { + let store = Store::open().await; + let text = "verified duplicate content ".repeat(80); + let oversized_id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let next_id = store + .insert(2, &text, Value::String(text.clone()), false) + .await; + // Bypass ingress only to exercise a pre-existing oversized stored document. + // The repair must not parse or rewrite it, nor pin the durable frontier. + let oversized = serde_json::json!({ + "lcm_active_replay": true, + "active_replay": {"content": text}, + "padding": "x".repeat(MAX_BATCH_BYTES as usize) + }) + .to_string(); + store + .conn + .execute( + "UPDATE lcm_raw_messages SET metadata_json = ?1 WHERE store_id = ?2", + params![oversized.as_str(), oversized_id], + ) + .await + .unwrap(); + let report = store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await; + assert_eq!(report.rows_scanned, 2); + assert_eq!(report.rows_skipped, 1); + assert_eq!(report.rows_skipped_oversized, 1); + assert_eq!(report.rows_compacted, 1); + assert!(report.metadata_bytes_examined > 0); + assert!(report.metadata_bytes_examined < MAX_BATCH_BYTES); + assert_eq!(report.frontier_after, Some(next_id)); + assert_eq!(frontier(&store.conn).await.unwrap(), next_id); + assert!(!report.has_more); + let mut rows = store + .conn + .query( + "SELECT metadata_json FROM lcm_raw_messages WHERE store_id = ?1", + params![oversized_id], + ) + .await + .unwrap(); + assert_eq!( + rows.next().await.unwrap().unwrap().get::(0).unwrap(), + oversized + ); + drop(rows); + assert_eq!( + store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await, + ReplayMetadataCompactionReport::default() + ); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .unwrap(); +} From 2a42006eef255c577137620543f9fa25cdbe96c0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:18:36 -0800 Subject: [PATCH 03/50] style(retention): satisfy Bazel rustfmt for replay regressions Apply only the two exact Rustfmt changes identified by the blocked repository formatting gate on the last PR head. Refs #4787 --- .../src/retention/replay_metadata/tests.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index a9b39a4c727..204b56de215 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -428,9 +428,11 @@ async fn large_ineligible_metadata_is_bounded_before_eligibility_checks() { .map(|(_, message)| message.metadata_json.as_ref().unwrap().len() as u64) .sum(); assert!(total_metadata_bytes > MAX_BATCH_BYTES); - assert!(originals.iter().all(|(_, message)| { - message.metadata_json.as_ref().unwrap().len() < 1024 * 1024 - })); + assert!( + originals + .iter() + .all(|(_, message)| { message.metadata_json.as_ref().unwrap().len() < 1024 * 1024 }) + ); let mut after = 0; let mut examined = 0; @@ -520,7 +522,12 @@ async fn oversized_metadata_is_preserved_without_blocking_later_compaction() { .await .unwrap(); assert_eq!( - rows.next().await.unwrap().unwrap().get::(0).unwrap(), + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), oversized ); drop(rows); From 91ffd680d7ec128dbf5cd14d9bcc721b4e457a21 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:20:15 -0800 Subject: [PATCH 04/50] fix(retention): verify stored replay checkpoints before commit Read back the replay frontier in the page's writer transaction before reporting progress. Refuse ignored or rewritten checkpoint writes so the existing transaction owner rolls back metadata changes rather than committing a page that will be visited again. Idle pages do not write. Add real database regressions for an ignored initial checkpoint, a rewritten existing checkpoint, rollback and successful recovery, and write-free idle pages. Preserve existing parsing and body budgets. Refs #4787 --- .../src/retention/replay_metadata.rs | 13 +- .../src/retention/replay_metadata/tests.rs | 143 ++++++++++++++++++ 2 files changed, 154 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index da43f273f78..6beaefecd52 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -272,10 +272,19 @@ async fn step( "replay metadata page did not advance its frontier", )); } + schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; + // An advancing local key is not proof that its checkpoint was stored. + // Verify inside the same writer so a refused or rewritten checkpoint + // rolls back this page instead of committing a false progress report. + let persisted = frontier(conn).await?; + if persisted != after { + return Err(db_error(format!( + "replay metadata frontier was not persisted: expected {after}, found {persisted}" + ))); + } report.frontier_before = Some(before); - report.frontier_after = Some(after); + report.frontier_after = Some(persisted); } - schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; report.has_more = has_work(conn, after).await?; Ok(report) } diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 204b56de215..23a0b3962af 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -539,3 +539,146 @@ async fn oversized_metadata_is_preserved_without_blocking_later_compaction() { .await .unwrap(); } + +async fn assert_checkpoint_refusal_is_atomic(store: &Store, id: i64, trigger_sql: &str) { + let checkpoint = schema::get_gc_meta(&store.conn, FRONTIER_KEY) + .await + .unwrap(); + let before = frontier(&store.conn).await.unwrap(); + let original = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + assert!(compacted_metadata(&original).unwrap().is_some()); + let revision = scalar( + &store.conn, + "SELECT raw_revision_generation FROM lcm_summary_convergence_queue", + ) + .await; + store.conn.execute_batch(trigger_sql).await.unwrap(); + let transaction = store + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("frontier was not persisted")); + let staged = raw::load_raw_message_by_store_id(&transaction, id) + .await + .unwrap(); + assert_ne!(staged.metadata_json, original.metadata_json); + assert!(metadata(&staged)["active_replay"].get("content").is_none()); + transaction.rollback().await.unwrap(); + + assert_eq!( + schema::get_gc_meta(&store.conn, FRONTIER_KEY) + .await + .unwrap(), + checkpoint, + "rollback preserves even an absent checkpoint" + ); + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + original, + "a refused checkpoint must restore the complete message and receipts" + ); + assert_eq!( + scalar( + &store.conn, + "SELECT raw_revision_generation FROM lcm_summary_convergence_queue" + ) + .await, + revision + ); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .unwrap(); + + store + .conn + .execute_batch("DROP TRIGGER refuse_replay_checkpoint") + .await + .unwrap(); + let recovered = store.tick(48).await; + assert_eq!(recovered.rows_scanned, 1); + assert_eq!(recovered.rows_compacted, 1); + assert!(recovered.metadata_bytes_removed > 0); + assert_eq!(recovered.frontier_before, Some(before)); + assert_eq!(recovered.frontier_after, Some(id)); + assert_eq!(frontier(&store.conn).await.unwrap(), id); + assert!(!recovered.has_more); + let repaired = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + assert_eq!(repaired.content, original.content); + assert_eq!(repaired.content_hash, original.content_hash); + assert_eq!( + crate::replay_transactions::raw_replay_message(&repaired), + crate::replay_transactions::raw_replay_message(&original) + ); + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); +} + +#[asupersync::test] +async fn ignored_initial_checkpoint_rolls_back_compaction() { + let store = Store::open().await; + let text = "verified replay checkpoint content ".repeat(80); + let id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let trigger = format!( + "CREATE TRIGGER refuse_replay_checkpoint BEFORE INSERT ON lcm_gc_meta + WHEN NEW.key = '{FRONTIER_KEY}' BEGIN SELECT RAISE(IGNORE); END" + ); + assert_checkpoint_refusal_is_atomic(&store, id, &trigger).await; +} + +#[asupersync::test] +async fn rewritten_checkpoint_cannot_claim_durable_progress() { + let store = Store::open().await; + let text = "verified replay checkpoint content ".repeat(80); + let previous_id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let first = store.tick(48).await; + assert_eq!(first.rows_compacted, 1); + assert_eq!(first.frontier_after, Some(previous_id)); + let id = store + .insert(2, &text, Value::String(text.clone()), false) + .await; + // A successful INSERT can still leave a different checkpoint behind. + // Counting affected rows alone cannot detect this failure. + let trigger = format!( + "CREATE TRIGGER refuse_replay_checkpoint AFTER INSERT ON lcm_gc_meta + WHEN NEW.key = '{FRONTIER_KEY}' BEGIN + UPDATE lcm_gc_meta SET value = '{previous_id}' WHERE key = NEW.key; + END" + ); + assert_checkpoint_refusal_is_atomic(&store, id, &trigger).await; +} + +#[asupersync::test] +async fn idle_page_does_not_initialize_a_checkpoint() { + let store = Store::open().await; + let trigger = format!( + "CREATE TRIGGER refuse_replay_checkpoint BEFORE INSERT ON lcm_gc_meta + WHEN NEW.key = '{FRONTIER_KEY}' BEGIN + SELECT RAISE(ABORT, 'idle page attempted a checkpoint write'); + END" + ); + store.conn.execute_batch(&trigger).await.unwrap(); + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); + assert_eq!( + schema::get_gc_meta(&store.conn, FRONTIER_KEY) + .await + .unwrap(), + None + ); +} From ad3729b860d9331d5913c41edc1a479bf3a18f08 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:44:16 -0800 Subject: [PATCH 05/50] fix(retention): verify canonical GC bootstrap checkpoints Refuse an unchanged bootstrap page or a checkpoint that did not persist, even when its SQL UPDATE reports one affected row. Roll back enrolled refs and seed writes through the existing writer transaction on refusal. Add a database-backed 52-owner regression proving repeated 48-row pages fail closed when a trigger rewrites their frontier, then resume through 48+4 pages and quiesce without collecting the live shared body. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 15 ++++ .../observation/canonical_body_gc/tests.rs | 79 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 7b5c0b056f4..819fa15afcf 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -36,6 +36,7 @@ pub struct CanonicalBodyGcReport { pub has_more: bool, } +#[derive(Clone, Debug, PartialEq, Eq)] struct Progress { raw_through: i64, raw_until: i64, @@ -120,6 +121,14 @@ async fn save_progress(conn: &(impl Executor + ?Sized), progress: &Progress) -> if changed != 1 { return Err(integrity_error("canonical body ownership frontier changed")); } + // The UPDATE can report one affected row even if a database trigger + // rewrites the checkpoint. Verify the durable value before committing the + // references or candidate seed that depended on this bootstrap page. + if read_progress(conn).await? != *progress { + return Err(integrity_error( + "canonical body ownership bootstrap frontier was not persisted", + )); + } Ok(()) } @@ -146,6 +155,7 @@ async fn step(conn: &(impl Executor + ?Sized), max_rows: usize) -> Result Result 0 BEGIN + UPDATE canonical_body_gc_progress SET raw_through = 0 WHERE singleton = 1; + END", + ) + .await + .unwrap(); + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("frontier was not persisted")); + let mut staged = transaction + .query("SELECT COUNT(*) FROM canonical_body_gc_refs", ()) + .await + .unwrap(); + let staged_count: i64 = staged.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(staged_count, 48, "the page was enrolled before checkpoint refusal"); + drop(staged); + transaction.rollback().await.unwrap(); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 0); + let mut durable = store + .conn() + .query("SELECT COUNT(*) FROM canonical_body_gc_refs", ()) + .await + .unwrap(); + let durable_count: i64 = durable.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(durable_count, 0, "the refused page must roll back its refs"); + drop(durable); + assert!(body_exists(store.conn(), &shared.content_hash).await); + + store + .conn() + .execute_batch("DROP TRIGGER corrupt_gc_bootstrap") + .await + .unwrap(); + let first = store.tick(48).await; + assert_eq!(first.roots_scanned, 48); + assert!(first.has_more); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 48); + let second = store.tick(48).await; + assert_eq!(second.roots_scanned, 4); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 52); + let (deleted, bytes) = drain(&store, 48).await; + assert_eq!((deleted, bytes), (0, 0), "live bodies remain referenced"); + assert!(body_exists(store.conn(), &shared.content_hash).await); + assert!(!has_candidates(store.conn()).await.unwrap()); + let idle = store.tick(48).await; + assert!(idle.bootstrap_complete); + assert_eq!(idle.roots_scanned, 0); + assert_eq!(idle.bodies_scanned, 0); + assert!(!idle.has_more); +} From 21032e3de2e90d2860d8d73b8e10e3c8faf6ffd7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:24:55 -0800 Subject: [PATCH 06/50] style(retention): satisfy Bazel rustfmt for GC regression Apply the exact formatting required by the failing Bazel rustfmt gate in the new canonical-body frontier rollback regression. Refs #4787 --- .../src/observation/canonical_body_gc/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 9c405001fa8..a81eb82aeef 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -598,7 +598,10 @@ async fn rewritten_bootstrap_checkpoint_refuses_commit_and_recovers_across_48_ro .await .unwrap(); let staged_count: i64 = staged.next().await.unwrap().unwrap().get(0).unwrap(); - assert_eq!(staged_count, 48, "the page was enrolled before checkpoint refusal"); + assert_eq!( + staged_count, 48, + "the page was enrolled before checkpoint refusal" + ); drop(staged); transaction.rollback().await.unwrap(); assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 0); From 8e044d468ac8cf755752792ca9f7ef7212f77578 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:30:02 -0800 Subject: [PATCH 07/50] perf(retention): avoid repeated replay eligibility parsing Bound replay eligibility admission by the conservative body verification budget before parsing metadata. Deferred rows no longer incur repeated JSON validation on successive ticks. Assert exactly-once metadata admission across database-backed multi-page body-budget regressions. Refs #4787 --- .../src/retention/replay_metadata.rs | 35 ++++++++++++------- .../src/retention/replay_metadata/tests.rs | 21 ++++++++++- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 6beaefecd52..6e435e32ce5 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -39,7 +39,7 @@ pub struct ReplayMetadataCompactionReport { /// representations exceed this pass's 4 MiB read and verification budget. pub rows_skipped_oversized: u64, /// Metadata bytes admitted to SQL eligibility parsing, including ineligible - /// rows and any prefetched rows deferred by the separate body byte budget. + /// rows. Work deferred by the body verification budget is not parsed. pub metadata_bytes_examined: u64, /// UTF-8 metadata bytes removed. This does not measure file shrinkage. pub metadata_bytes_removed: u64, @@ -106,8 +106,8 @@ struct Candidate { read_bytes: Option, } -/// Admit a size-bounded prefix before parsing any metadata. Eligibility must -/// not parse all 48 documents before the body's separate byte budget is applied. +/// Admit a prefix bounded by both metadata parsing and body-read estimates +/// before testing eligibility. Deferred rows must not be JSON-parsed again. async fn candidates( conn: &(impl QueryExecutor + ?Sized), after: i64, @@ -131,6 +131,7 @@ async fn candidates( let mut result = Vec::new(); let mut eligibility_ids = Vec::new(); let mut remaining = MAX_BATCH_BYTES; + let mut read_remaining = MAX_BATCH_BYTES; while let Some(row) = rows.next().await? { let store_id: i64 = row.get(0)?; let inline = row.get::(1)? != 0; @@ -143,15 +144,6 @@ async fn candidates( 0 }; let metadata_oversized = metadata_bytes > MAX_BATCH_BYTES; - if !metadata_oversized { - if metadata_bytes > remaining { - break; - } - remaining -= metadata_bytes; - if inline && metadata.is_some() { - eligibility_ids.push(store_id); - } - } let read_bytes = metadata .zip(body) .and_then(|(metadata, body)| { @@ -162,6 +154,25 @@ async fn candidates( )) }) .and_then(|(metadata, body, stored)| metadata.checked_add(body)?.checked_add(stored)); + if !metadata_oversized { + if metadata_bytes > remaining { + break; + } + // Eligibility is not yet known. Charge every inline row against + // the verification budget so deferred rows are not JSON-parsed + // again on the next tick. Individually oversized rows still + // advance without loading their bodies. + if inline && let Some(bytes) = read_bytes && bytes <= MAX_BATCH_BYTES { + if bytes > read_remaining { + break; + } + read_remaining -= bytes; + } + remaining -= metadata_bytes; + if inline && metadata.is_some() { + eligibility_ids.push(store_id); + } + } result.push(Candidate { store_id, eligible: false, diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 23a0b3962af..5c794d2ed99 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -348,14 +348,20 @@ async fn stale_metadata_cannot_be_overwritten_and_missing_guard_refuses_repair() async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_noops() { let store = Store::open().await; let text = "bounded replay text ".repeat(35_000); + let mut metadata_bytes = Vec::new(); for ordinal in 1..=4 { - store + let id = store .insert(ordinal, &text, Value::String(text.clone()), true) .await; + let message = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + metadata_bytes.push(message.metadata_json.unwrap().len() as u64); } let first = store.tick(1).await; assert_eq!(first.rows_scanned, 1); assert_eq!(first.rows_compacted, 1); + assert_eq!(first.metadata_bytes_examined, metadata_bytes[0]); assert!(first.has_more); assert_eq!(frontier(&store.conn).await.unwrap(), 1); let second = store.tick(48).await; @@ -364,7 +370,20 @@ async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_ "byte cap stops before the third large row" ); assert!(second.has_more); + assert_eq!( + second.metadata_bytes_examined, + metadata_bytes[1..1 + second.rows_scanned as usize] + .iter() + .sum::(), + "the body budget must not pre-parse replay metadata for deferred rows" + ); let last = store.tick(48).await; + assert_eq!( + last.metadata_bytes_examined, + metadata_bytes[1 + second.rows_scanned as usize..] + .iter() + .sum::() + ); assert!(!last.has_more); let idle = store.tick(48).await; assert_eq!(idle, ReplayMetadataCompactionReport::default()); From a1b2bcf0572bd0b654ae10983329469b776c0238 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:08:31 -0800 Subject: [PATCH 08/50] feat(retention): record committed page latency and replay parsing work Measure each canonical-body GC and replay-metadata pass from admission through successful writer commit. Include elapsed milliseconds and admitted metadata bytes in the existing daemon progress events so an affected host can distinguish stalled page retries from costly but advancing backlogs. Preserve the transactional progress guards. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 8 +++++++- .../src/retention/replay_metadata.rs | 7 ++++++- crates/tracedecay/src/daemon/maintenance.rs | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 819fa15afcf..34dbdf30ad2 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -6,6 +6,8 @@ //! bootstrap frontiers are complete. A candidate and its indexed references are //! checked under one guarded writer, so another owner cannot race collection. +use std::time::Instant; + use serde::{Deserialize, Serialize}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::db::Database; @@ -25,6 +27,8 @@ pub const MAX_CANONICAL_BODY_GC_ROWS: usize = 48; pub struct CanonicalBodyGcReport { pub roots_scanned: u64, pub bodies_scanned: u64, + /// Wall-clock duration from writer admission through commit, not CPU time. + pub elapsed_ms: Option, /// Candidates discharged in the committed page, even if their body stays live. pub candidates_discharged: u64, pub bodies_deleted: u64, @@ -134,10 +138,12 @@ async fn save_progress(conn: &(impl Executor + ?Sized), progress: &Progress) -> #[tracing::instrument(name = "global_db.canonical_bodies.gc", level = "trace", skip_all)] pub(crate) async fn run(database: &Database, max_rows: usize) -> Result { + let started = Instant::now(); let transaction = database.begin_write_transaction(OPERATION).await?; match step(&transaction, max_rows).await { - Ok(report) => { + Ok(mut report) => { transaction.commit().await?; + report.elapsed_ms = Some(started.elapsed().as_millis()); Ok(report) } Err(error) => match transaction.rollback().await { diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 6e435e32ce5..48b7128b81c 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -6,6 +6,7 @@ //! budget keep this repair off the foreground ingest path. use std::collections::BTreeSet; +use std::time::Instant; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -28,6 +29,8 @@ const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ReplayMetadataCompactionReport { pub rows_scanned: u64, + /// Wall-clock duration from initial frontier read through commit, not CPU time. + pub elapsed_ms: Option, /// Transactional store-id frontier consumed by this page, absent on idle reads. pub frontier_before: Option, pub frontier_after: Option, @@ -79,6 +82,7 @@ pub async fn compact_replay_metadata( database: &Database, max_rows: usize, ) -> Result { + let started = Instant::now(); let read = database.read_connection(); if !has_work(&read, frontier(&read).await?).await? { return Ok(ReplayMetadataCompactionReport::default()); @@ -88,8 +92,9 @@ pub async fn compact_replay_metadata( .await .map_err(db_error)?; match step(&transaction, max_rows).await { - Ok(report) => { + Ok(mut report) => { transaction.commit().await.map_err(db_error)?; + report.elapsed_ms = Some(started.elapsed().as_millis()); Ok(report) } Err(error) => match transaction.rollback().await { diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 9e293532be4..967bf06dbba 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -386,6 +386,13 @@ fn record_canonical_body_gc_outcome( &[ ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), + ( + "elapsed_ms", + report.elapsed_ms.map_or_else( + || "unmeasured".to_owned(), + |value| value.to_string(), + ), + ), ( "candidates_discharged", report.candidates_discharged.to_string(), @@ -432,6 +439,17 @@ fn record_replay_metadata_outcome( "retention_replay_metadata", &[ ("rows_scanned", report.rows_scanned.to_string()), + ( + "elapsed_ms", + report.elapsed_ms.map_or_else( + || "unmeasured".to_owned(), + |value| value.to_string(), + ), + ), + ( + "metadata_bytes_examined", + report.metadata_bytes_examined.to_string(), + ), ( "frontier_before", report.frontier_before.map_or_else( From 3924b36324044b9770ebb81fc201b40ae0e55a10 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:22:21 -0800 Subject: [PATCH 09/50] style(retention): fix Bazel rustfmt for per-tick measurements Apply the exact three rustfmt transformations in the latest failed repository gate without changing retention behavior. Refs #4787 --- .../src/retention/replay_metadata.rs | 5 ++++- crates/tracedecay/src/daemon/maintenance.rs | 14 ++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 48b7128b81c..50d9e115c24 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -167,7 +167,10 @@ async fn candidates( // the verification budget so deferred rows are not JSON-parsed // again on the next tick. Individually oversized rows still // advance without loading their bodies. - if inline && let Some(bytes) = read_bytes && bytes <= MAX_BATCH_BYTES { + if inline + && let Some(bytes) = read_bytes + && bytes <= MAX_BATCH_BYTES + { if bytes > read_remaining { break; } diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 967bf06dbba..8916ab52886 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -388,10 +388,9 @@ fn record_canonical_body_gc_outcome( ("bodies_scanned", report.bodies_scanned.to_string()), ( "elapsed_ms", - report.elapsed_ms.map_or_else( - || "unmeasured".to_owned(), - |value| value.to_string(), - ), + report + .elapsed_ms + .map_or_else(|| "unmeasured".to_owned(), |value| value.to_string()), ), ( "candidates_discharged", @@ -441,10 +440,9 @@ fn record_replay_metadata_outcome( ("rows_scanned", report.rows_scanned.to_string()), ( "elapsed_ms", - report.elapsed_ms.map_or_else( - || "unmeasured".to_owned(), - |value| value.to_string(), - ), + report + .elapsed_ms + .map_or_else(|| "unmeasured".to_owned(), |value| value.to_string()), ), ( "metadata_bytes_examined", From a945a8b85b4a1f3f146872ff226d3319014b9b21 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:28 -0800 Subject: [PATCH 10/50] fix(retention): reject oversized payloads before sizing reads Use SQLite record byte lengths without materializing raw text. Defer storage-kind reads and unnecessary canonical lookups until admission, and do not read canonical size scalars beyond an oversized BLOB. Skip eligibility parsing when the stored representations already exceed the page budget, while preserving the rows and advancing the frontier. Add an isolated Linux database regression measuring actual read syscall bytes for oversized inline, metadata, and canonical payloads. Verify subsequent valid compaction, complete byte preservation, and quiescence. Refs #4787 --- .../src/retention/replay_metadata.rs | 58 +++--- .../retention/replay_metadata/read_cost.rs | 195 ++++++++++++++++++ 2 files changed, 224 insertions(+), 29 deletions(-) create mode 100644 crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 50d9e115c24..dc3dd3b0d0e 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -15,6 +15,8 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use crate::{LcmError, LcmRawMessage, gc, payload, raw, schema, security, summary_convergence}; +#[cfg(all(test, target_os = "linux"))] +mod read_cost; #[cfg(test)] mod tests; @@ -41,7 +43,7 @@ pub struct ReplayMetadataCompactionReport { /// Subset of skipped rows whose metadata plus canonical/stored body /// representations exceed this pass's 4 MiB read and verification budget. pub rows_skipped_oversized: u64, - /// Metadata bytes admitted to SQL eligibility parsing, including ineligible + /// Metadata bytes admitted to SQL eligibility checks, including ineligible /// rows. Work deferred by the body verification budget is not parsed. pub metadata_bytes_examined: u64, /// UTF-8 metadata bytes removed. This does not measure file shrinkage. @@ -107,7 +109,7 @@ pub async fn compact_replay_metadata( struct Candidate { store_id: i64, eligible: bool, - metadata_oversized: bool, + oversized: bool, read_bytes: Option, } @@ -118,18 +120,24 @@ async fn candidates( after: i64, limit: usize, ) -> Result<(Vec, u64), LcmError> { + // octet_length reads record metadata, unlike length(CAST(text AS BLOB)). + // Even a small scalar after a large body can load its overflow chain: + // defer storage_kind to eligibility and avoid unnecessary canonical joins. let mut rows = conn .query( - "SELECT r.store_id, r.storage_kind = 'inline', - LENGTH(CAST(r.metadata_json AS BLOB)), - COALESCE(LENGTH(CAST(r.content AS BLOB)), b.uncompressed_bytes), + "SELECT r.store_id, octet_length(r.metadata_json), + COALESCE(octet_length(r.content), + CASE WHEN LENGTH(b.body) <= ?3 THEN b.uncompressed_bytes END), COALESCE(LENGTH(b.body), 0) FROM lcm_raw_messages r - LEFT JOIN session_canonical_bodies b ON b.content_hash = r.content_hash + LEFT JOIN session_canonical_bodies b ON b.content_hash = CASE + WHEN r.content IS NULL AND octet_length(r.metadata_json) <= ?3 + THEN r.content_hash END WHERE r.store_id > ?1 ORDER BY r.store_id LIMIT ?2", params![ after, - limit.clamp(1, MAX_REPLAY_METADATA_COMPACTION_ROWS) as i64 + limit.clamp(1, MAX_REPLAY_METADATA_COMPACTION_ROWS) as i64, + MAX_BATCH_BYTES as i64 ], ) .await?; @@ -139,16 +147,10 @@ async fn candidates( let mut read_remaining = MAX_BATCH_BYTES; while let Some(row) = rows.next().await? { let store_id: i64 = row.get(0)?; - let inline = row.get::(1)? != 0; - let metadata: Option = row.get(2)?; - let body: Option = row.get(3)?; - let stored: i64 = row.get(4)?; - let metadata_bytes = if inline { - u64::try_from(metadata.unwrap_or(0)).map_err(db_error)? - } else { - 0 - }; - let metadata_oversized = metadata_bytes > MAX_BATCH_BYTES; + let metadata: Option = row.get(1)?; + let body: Option = row.get(2)?; + let stored: i64 = row.get(3)?; + let metadata_bytes = u64::try_from(metadata.unwrap_or(0)).map_err(db_error)?; let read_bytes = metadata .zip(body) .and_then(|(metadata, body)| { @@ -159,32 +161,30 @@ async fn candidates( )) }) .and_then(|(metadata, body, stored)| metadata.checked_add(body)?.checked_add(stored)); - if !metadata_oversized { + let oversized = metadata_bytes > MAX_BATCH_BYTES + || stored > MAX_BATCH_BYTES as i64 + || read_bytes.is_some_and(|bytes| bytes > MAX_BATCH_BYTES); + if !oversized { if metadata_bytes > remaining { break; } - // Eligibility is not yet known. Charge every inline row against - // the verification budget so deferred rows are not JSON-parsed - // again on the next tick. Individually oversized rows still - // advance without loading their bodies. - if inline - && let Some(bytes) = read_bytes - && bytes <= MAX_BATCH_BYTES - { + // Eligibility is not yet known. Charge each potential body read + // before parsing so deferred rows are not revisited for JSON work. + if let Some(bytes) = read_bytes { if bytes > read_remaining { break; } read_remaining -= bytes; } remaining -= metadata_bytes; - if inline && metadata.is_some() { + if metadata.is_some() { eligibility_ids.push(store_id); } } result.push(Candidate { store_id, eligible: false, - metadata_oversized, + oversized, read_bytes, }); } @@ -238,7 +238,7 @@ async fn step( let mut remaining = MAX_BATCH_BYTES; let mut guard_suspended = false; for candidate in candidates { - if candidate.metadata_oversized { + if candidate.oversized { report.rows_skipped += 1; report.rows_skipped_oversized += 1; } else if candidate.eligible { diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs new file mode 100644 index 00000000000..81ee6bfaefb --- /dev/null +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs @@ -0,0 +1,195 @@ +//! Isolate process-wide read accounting from concurrently running unit tests. + +use std::process::Command; + +use tracedecay_privacy::sanitize_lcm_payload_text; +use tracedecay_runtime_core::db::engine::{ + Executor, QueryExecutor, TestConnection, TransactionBehavior, params, +}; + +use super::{MAX_BATCH_BYTES, ReplayMetadataCompactionReport, frontier, step}; +use crate::{raw, retrieval_content::projected_content_hash, schema, summary_convergence}; + +fn read_chars() -> u64 { + std::fs::read_to_string("/proc/self/io") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("rchar:")) + .unwrap() + .trim() + .parse() + .unwrap() +} + +#[asupersync::test] +async fn oversized_rows_skip_payload_io() { + const CHILD: &str = "TRACEDECAY_REPLAY_READ_COST_CHILD"; + const TEST: &str = "retention::replay_metadata::read_cost::oversized_rows_skip_payload_io"; + if std::env::var_os(CHILD).is_none() { + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST, "--test-threads=1"]) + .env(CHILD, "1") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "isolated read-cost test failed:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(stdout.contains("1 passed; 0 failed"), "{stdout}"); + return; + } + + let directory = tempfile::tempdir().unwrap(); + let conn = TestConnection::open(&directory.path().join("replay-read-cost.db")); + conn.execute_batch( + "CREATE TABLE sessions ( + provider TEXT NOT NULL, session_id TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); INSERT INTO sessions VALUES ('codex', 'session'); + PRAGMA cache_size = 8; + PRAGMA mmap_size = 0;", + ) + .await + .unwrap(); + schema::ensure_lcm_schema(&conn).await.unwrap(); + + let small = "verified replay after oversized rows ".repeat(80); + let sanitized = sanitize_lcm_payload_text(&small).unwrap(); + let metadata = serde_json::json!({ + "lcm_active_replay": true, + "active_replay": {"role": "assistant", "content": small}, + "ingest_protection": {"sanitization_receipt": sanitized.receipt()} + }); + let metadata = + raw::resanitize_stored_provider_metadata_json(&metadata.to_string()).unwrap(); + let large = "x".repeat(2 * MAX_BATCH_BYTES as usize); + let padded = format!("{metadata}{}", " ".repeat(2 * MAX_BATCH_BYTES as usize)); + let large_hash = projected_content_hash(&large); + let small_hash = projected_content_hash(&small); + conn.execute( + "INSERT INTO session_canonical_bodies(content_hash, encoding, body, uncompressed_bytes) + VALUES (?1, 'identity', ?2, ?3)", + params![large_hash.as_str(), large.as_bytes(), large.len() as i64], + ) + .await + .unwrap(); + // Direct fixture writes represent oversized retained data, not new ingress. + // Each overflow location must be rejected before its bytes are loaded: + // inline body, metadata, then a separately stored canonical body. + let fixtures = [ + ( + 1_i64, + Some(large.as_str()), + large_hash.as_str(), + metadata.as_str(), + ), + ( + 2, + Some(small.as_str()), + small_hash.as_str(), + padded.as_str(), + ), + (3, None, large_hash.as_str(), metadata.as_str()), + ( + 4, + Some(small.as_str()), + small_hash.as_str(), + metadata.as_str(), + ), + ]; + for (id, content, hash, document) in fixtures { + conn.execute( + "INSERT INTO lcm_raw_messages(provider, message_id, session_id, role, ordinal, + content, content_hash, storage_kind, metadata_json) + VALUES ('codex', ?1, 'session', 'assistant', ?2, ?3, ?4, 'inline', ?5)", + params![format!("message-{id}"), id, content, hash, document], + ) + .await + .unwrap(); + } + let original = raw::load_raw_message_by_store_id(&conn, 4).await.unwrap(); + + for id in 1..=3 { + conn.execute_batch("PRAGMA shrink_memory;").await.unwrap(); + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let before = read_chars(); + let report = step(&transaction, 1).await.unwrap(); + transaction.commit().await.unwrap(); + let read = read_chars() - before; + assert!( + read < MAX_BATCH_BYTES, + "rejected row {id} read {read} bytes before admission; budget={MAX_BATCH_BYTES}" + ); + assert_eq!(report.rows_scanned, 1); + assert_eq!(report.rows_skipped_oversized, 1); + assert_eq!(report.rows_compacted, 0); + assert_eq!(report.metadata_bytes_examined, 0); + assert_eq!(report.frontier_before, Some(id - 1)); + assert_eq!(report.frontier_after, Some(id)); + assert_eq!(frontier(&conn).await.unwrap(), id); + assert!(report.has_more); + } + + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let report = step(&transaction, 48).await.unwrap(); + transaction.commit().await.unwrap(); + assert_eq!(report.rows_scanned, 1); + assert_eq!(report.rows_compacted, 1); + assert!(report.metadata_bytes_removed > 0); + assert_eq!(report.frontier_after, Some(4)); + assert!(!report.has_more); + let repaired = raw::load_raw_message_by_store_id(&conn, 4).await.unwrap(); + assert_eq!(repaired.content, original.content); + assert_eq!( + crate::replay_transactions::raw_replay_message(&repaired), + crate::replay_transactions::raw_replay_message(&original) + ); + summary_convergence::validate_raw_update_guard(&conn) + .await + .unwrap(); + + // Check complete retained bytes outside the measured interval. + for (id, content, _, document) in &fixtures[..3] { + let mut rows = conn + .query( + "SELECT content, metadata_json FROM lcm_raw_messages WHERE store_id = ?1", + params![*id], + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().unwrap(); + assert_eq!(row.get::>(0).unwrap().as_deref(), *content); + assert_eq!(row.get::(1).unwrap(), *document); + } + let mut rows = conn + .query("SELECT body FROM session_canonical_bodies", ()) + .await + .unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::>(0) + .unwrap(), + large.as_bytes() + ); + drop(rows); + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + assert_eq!( + step(&transaction, 48).await.unwrap(), + ReplayMetadataCompactionReport::default() + ); + transaction.commit().await.unwrap(); +} From ee4dc4cb18947177d78862ed4e1c9f400a7a02d2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:53:29 -0800 Subject: [PATCH 11/50] style(retention): format replay read-cost regression Apply the exact Bazel rustfmt_check_2024 correction at read_cost.rs:62. No runtime or test behavior changes. Refs #4787 --- .../tracedecay-lcm/src/retention/replay_metadata/read_cost.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs index 81ee6bfaefb..e490778a241 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs @@ -62,8 +62,7 @@ async fn oversized_rows_skip_payload_io() { "active_replay": {"role": "assistant", "content": small}, "ingest_protection": {"sanitization_receipt": sanitized.receipt()} }); - let metadata = - raw::resanitize_stored_provider_metadata_json(&metadata.to_string()).unwrap(); + let metadata = raw::resanitize_stored_provider_metadata_json(&metadata.to_string()).unwrap(); let large = "x".repeat(2 * MAX_BATCH_BYTES as usize); let padded = format!("{metadata}{}", " ".repeat(2 * MAX_BATCH_BYTES as usize)); let large_hash = projected_content_hash(&large); From 33988f731c8d8de09b2fbeca18fe4572e5599a8c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:51:23 -0800 Subject: [PATCH 12/50] test(retention): reopen read-cost fixture without denied PRAGMAs Refs #4787 Use a fresh exact-SQL test connection for each measured replay page, so SQLite cannot reuse a warmed per-connection page cache. Remove the cache/mmap/shrink PRAGMAs rejected by the registered SQL authority. Keep the byte-read thresholds, 48-row semantics, frontier assertions, post-repair replay equivalence, and idle checks unchanged. --- .../src/retention/replay_metadata/read_cost.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs index e490778a241..548348e2cd6 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs @@ -42,14 +42,13 @@ async fn oversized_rows_skip_payload_io() { } let directory = tempfile::tempdir().unwrap(); - let conn = TestConnection::open(&directory.path().join("replay-read-cost.db")); + let database_path = directory.path().join("replay-read-cost.db"); + let conn = TestConnection::open(&database_path); conn.execute_batch( "CREATE TABLE sessions ( provider TEXT NOT NULL, session_id TEXT NOT NULL, PRIMARY KEY(provider, session_id) - ); INSERT INTO sessions VALUES ('codex', 'session'); - PRAGMA cache_size = 8; - PRAGMA mmap_size = 0;", + ); INSERT INTO sessions VALUES ('codex', 'session');", ) .await .unwrap(); @@ -109,9 +108,12 @@ async fn oversized_rows_skip_payload_io() { .unwrap(); } let original = raw::load_raw_message_by_store_id(&conn, 4).await.unwrap(); + drop(conn); for id in 1..=3 { - conn.execute_batch("PRAGMA shrink_memory;").await.unwrap(); + // Fresh connections preserve the oversized-read measurement without + // issuing PRAGMAs forbidden by the exact-SQL test authority. + let conn = TestConnection::open(&database_path); let transaction = conn .transaction_with_behavior(TransactionBehavior::Immediate) .await @@ -134,6 +136,7 @@ async fn oversized_rows_skip_payload_io() { assert!(report.has_more); } + let conn = TestConnection::open(&database_path); let transaction = conn .transaction_with_behavior(TransactionBehavior::Immediate) .await From 370e91d088b9139f6ff9070143169d4e23a90e50 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:36:16 -0800 Subject: [PATCH 13/50] fix(retention): verify canonical candidate discharge after triggers Refs #4787 Validate the selected candidate hashes are absent inside the same writer transaction after DELETE. An AFTER DELETE trigger can reinsert all 48 rows while SQLite reports 48 affected deletes, producing false progress and repeat page work. The readback fails closed so the caller rolls back. Add a 52-body database-backed regression proving reinserted pages are refused without losing candidates, then recover as distinct 48+4 pages. --- .../src/observation/canonical_body_gc.rs | 19 +++- .../observation/canonical_body_gc/tests.rs | 93 +++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 34dbdf30ad2..771a4054888 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -352,10 +352,27 @@ async fn collect_candidates( .execute( "DELETE FROM canonical_body_gc_candidates WHERE content_hash IN (SELECT value FROM json_each(?1))", - params![hashes], + params![hashes.as_str()], ) .await .map_err(db_error)?; + // A DELETE row count is not a durable discharge receipt: an AFTER DELETE + // trigger may have re-enqueued the very same candidates. Refuse the page + // under this writer instead of recommitting an identical 48-row batch. + let mut remaining = conn + .query( + "SELECT 1 FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1)) LIMIT 1", + params![hashes.as_str()], + ) + .await + .map_err(db_error)?; + if remaining.next().await.map_err(db_error)?.is_some() { + return Err(integrity_error( + "canonical body candidate discharge was not persisted", + )); + } + drop(remaining); let discharged = explicitly_discharged .checked_add(report.bodies_deleted) .ok_or_else(|| integrity_error("canonical body candidate discharge overflow"))?; diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index a81eb82aeef..15cf06540f0 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -92,6 +92,15 @@ async fn body_exists(conn: &(impl QueryExecutor + ?Sized), hash: &str) -> bool { .is_some() } + +async fn candidate_count(conn: &(impl QueryExecutor + ?Sized)) -> i64 { + let mut rows = conn + .query("SELECT COUNT(*) FROM canonical_body_gc_candidates", ()) + .await + .unwrap(); + rows.next().await.unwrap().unwrap().get(0).unwrap() +} + fn observation(hash: &str) -> String { serde_json::json!({ "payload": {"text": null}, @@ -637,3 +646,87 @@ async fn rewritten_bootstrap_checkpoint_refuses_commit_and_recovers_across_48_ro assert_eq!(idle.bodies_scanned, 0); assert!(!idle.has_more); } + +#[asupersync::test] +async fn reinserted_candidate_page_refuses_false_discharge_and_recovers() { + let store = Store::open().await; + for ordinal in 0..52 { + let body = StoredCanonicalBody::pack(format!("retained {ordinal}").as_bytes()).unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + let mut bootstrap_finished = false; + for _ in 0..16 { + if store.tick(48).await.bootstrap_complete { + bootstrap_finished = true; + break; + } + } + assert!(bootstrap_finished, "candidate collection must become eligible"); + assert_eq!(candidate_count(store.conn()).await, 52); + + // The explicit DELETE returns 48 affected rows even when an AFTER DELETE + // trigger restores all of them. Only a same-transaction readback catches + // the false progress that otherwise repeats this exact 48-row page. + store + .conn() + .execute_batch( + "CREATE TRIGGER keep_candidates_after_delete + AFTER DELETE ON canonical_body_gc_candidates BEGIN + INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) + VALUES (OLD.content_hash); + END", + ) + .await + .unwrap(); + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error.to_string().contains("candidate discharge was not persisted"), + "an apparent 48-row DELETE must not count as committed progress: {error}" + ); + assert_eq!( + candidate_count(&transaction).await, + 52, + "the trigger left every candidate in the still-open transaction" + ); + transaction.rollback().await.unwrap(); + assert_eq!( + candidate_count(store.conn()).await, + 52, + "the refused page must leave all candidate identities durable" + ); + store + .conn() + .execute_batch("DROP TRIGGER keep_candidates_after_delete") + .await + .unwrap(); + + let first = store.tick(48).await; + assert_eq!(first.bodies_scanned, 48); + assert_eq!(first.candidates_discharged, 48); + assert_eq!(first.bodies_deleted, 0); + assert!(first.has_more); + let second = store.tick(48).await; + assert_eq!(second.bodies_scanned, 4); + assert_eq!(second.candidates_discharged, 4); + assert_eq!(second.bodies_deleted, 0); + assert!(!second.has_more); + assert_eq!(candidate_count(store.conn()).await, 0); + let idle = store.tick(48).await; + assert_eq!(idle.bodies_scanned, 0); + assert!(!idle.has_more); +} From fd2ac3dd2489905a7426fc1e3a1f1c82383b97d0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:37:05 -0800 Subject: [PATCH 14/50] style(retention): keep canonical GC test helpers rustfmt-clean Refs #4787 Remove an extra blank line introduced with the candidate-count helper. --- .../src/observation/canonical_body_gc/tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 15cf06540f0..bd71002ad1a 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -92,7 +92,6 @@ async fn body_exists(conn: &(impl QueryExecutor + ?Sized), hash: &str) -> bool { .is_some() } - async fn candidate_count(conn: &(impl QueryExecutor + ?Sized)) -> i64 { let mut rows = conn .query("SELECT COUNT(*) FROM canonical_body_gc_candidates", ()) From d21084185df37bd8230545e25cd18524b625980b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:59:10 -0800 Subject: [PATCH 15/50] style(retention): format candidate discharge assertions --- .../src/observation/canonical_body_gc/tests.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index bd71002ad1a..2327c0a9488 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -670,7 +670,10 @@ async fn reinserted_candidate_page_refuses_false_discharge_and_recovers() { break; } } - assert!(bootstrap_finished, "candidate collection must become eligible"); + assert!( + bootstrap_finished, + "candidate collection must become eligible" + ); assert_eq!(candidate_count(store.conn()).await, 52); // The explicit DELETE returns 48 affected rows even when an AFTER DELETE @@ -694,7 +697,9 @@ async fn reinserted_candidate_page_refuses_false_discharge_and_recovers() { .unwrap(); let error = step(&transaction, 48).await.unwrap_err(); assert!( - error.to_string().contains("candidate discharge was not persisted"), + error + .to_string() + .contains("candidate discharge was not persisted"), "an apparent 48-row DELETE must not count as committed progress: {error}" ); assert_eq!( From 57bae69e909ec818e9238f26cbf1bfe41dcba69f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 16/50] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 4984a54fc4cf6cd71d2191d60a80639b1f0b183f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:31:24 -0800 Subject: [PATCH 17/50] test(daemon): verify rollback at the store mount boundary --- .../drop_shutdown_journey_test.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs index 7b159cbcf6a..51efc83f65c 100644 --- a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs @@ -185,7 +185,7 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() .expect("refused store root"); std::fs::write(&refused_graph, b"not a project graph database").expect("refused graph"); - let error = match ProductionProjectCompositionHarnessV1::open( + let error = match ProductionProjectCompositionHarnessV1::open_for_session_retrieval( isolation.path(), [first.clone(), refused.clone()], ) @@ -207,7 +207,7 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() let stores = [ profile.join("global.db"), profile.join("user-sessions.db"), - first_layout.data_root.join("sessions.db"), + first_layout.sessions_db_path, first_layout.graph_db_path, ]; for store in &stores { @@ -229,14 +229,23 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() } std::fs::remove_file(&refused_graph).expect("remove the fixture refusal"); - let harness = ProductionProjectCompositionHarnessV1::open( + let harness = ProductionProjectCompositionHarnessV1::open_for_session_retrieval( isolation.path(), [first.clone(), refused.clone()], ) .await .expect("both exact projects reopen after failed-open rollback"); - assert!(harness.server(&first).is_ok()); - assert!(harness.server(&refused).is_ok()); + for project in [&first, &refused] { + let server = harness.server(project).expect("reopened project server"); + let store = server + .project_session_db() + .expect("mounted project sessions"); + let layout = default_profile_sharded_layout(project, &profile).unwrap(); + assert_eq!( + store.db_path().canonicalize().unwrap(), + layout.sessions_db_path.canonicalize().unwrap() + ); + } harness.shutdown().await; } From ee44f8c16b736c36075eaadfe4bee6551194951a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:26:19 -0800 Subject: [PATCH 18/50] test(daemon): isolate failed-mount cleanup from graph readiness Preserve the failed-mount fixture repair from 4984a54 byte-for-byte in its own reviewable change above the shared scheduler repair #4821. The rollback journey uses the existing session-retrieval opener, checks the authoritative session path, and verifies both reopened projects bind their own session stores. No deadline or assertion is removed, and the malformed graph must still refuse admission. This separates CI fixture work from the retention-specific #4798 review without reverting fixes, rewriting history, or editing #4813. It is not evidence that the affected-host retention CPU incident is resolved. Refs #4787. --- .../drop_shutdown_journey_test.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs index 7b159cbcf6a..51efc83f65c 100644 --- a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs @@ -185,7 +185,7 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() .expect("refused store root"); std::fs::write(&refused_graph, b"not a project graph database").expect("refused graph"); - let error = match ProductionProjectCompositionHarnessV1::open( + let error = match ProductionProjectCompositionHarnessV1::open_for_session_retrieval( isolation.path(), [first.clone(), refused.clone()], ) @@ -207,7 +207,7 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() let stores = [ profile.join("global.db"), profile.join("user-sessions.db"), - first_layout.data_root.join("sessions.db"), + first_layout.sessions_db_path, first_layout.graph_db_path, ]; for store in &stores { @@ -229,14 +229,23 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() } std::fs::remove_file(&refused_graph).expect("remove the fixture refusal"); - let harness = ProductionProjectCompositionHarnessV1::open( + let harness = ProductionProjectCompositionHarnessV1::open_for_session_retrieval( isolation.path(), [first.clone(), refused.clone()], ) .await .expect("both exact projects reopen after failed-open rollback"); - assert!(harness.server(&first).is_ok()); - assert!(harness.server(&refused).is_ok()); + for project in [&first, &refused] { + let server = harness.server(project).expect("reopened project server"); + let store = server + .project_session_db() + .expect("mounted project sessions"); + let layout = default_profile_sharded_layout(project, &profile).unwrap(); + assert_eq!( + store.db_path().canonicalize().unwrap(), + layout.sessions_db_path.canonicalize().unwrap() + ); + } harness.shutdown().await; } From ce112eff1754af4a2fc966acda79014157a599d3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:08:35 -0800 Subject: [PATCH 19/50] fix(retention): report committed canonical bootstrap frontiers Expose the phase-qualified raw, observation and canonical-body SQL frontiers before and after each successful bootstrap page. Populate these only after the same-transaction checkpoint readback succeeds; collection and idle pages cannot masquerade as bootstrap progress. Log the frontiers with the already measured page cost and discharge counts. A real SQLite-backed regression checks three distinct phases, the durable cursor after each tick, retained live content, collection and quiescence. The existing 52-owner rollback/48+4 regression now asserts its committed 0->48->52 frontier values. Refs #4787; not incident acceptance. No CI, dependency, release, or concurrent writer-admission change. --- .../src/observation/canonical_body_gc.rs | 23 +++++- .../observation/canonical_body_gc/tests.rs | 71 +++++++++++++++++++ crates/tracedecay/src/daemon/maintenance.rs | 16 +++++ 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 771a4054888..3b2725a92ad 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -31,6 +31,11 @@ pub struct CanonicalBodyGcReport { pub elapsed_ms: Option, /// Candidates discharged in the committed page, even if their body stays live. pub candidates_discharged: u64, + /// Phase-qualified SQL bootstrap cursor before the committed page. + /// The value is absent during candidate collection and idle ticks. + pub bootstrap_frontier_before: Option, + /// The corresponding persisted cursor, verified in the same writer. + pub bootstrap_frontier_after: Option, pub bodies_deleted: u64, /// Stored BLOB bytes deleted, including the effect of their existing /// compression. This is not database-file or filesystem space reclaimed. @@ -162,19 +167,33 @@ async fn step(conn: &(impl Executor + ?Sized), max_rows: usize) -> Result format!("raw:{}", persisted.raw_through), + 1 => format!("observations:{}", persisted.observations_through), + _ => format!("bodies:{}", persisted.bodies_after), + }; + assert_eq!(stored_after, after, "reported cursor must be durable"); + assert_eq!(report.roots_scanned + report.bodies_scanned, 1); + assert_eq!(report.bodies_deleted, 0); + assert_eq!(report.candidates_discharged, 0); + assert!(report.has_more); + } + let discharged = store.tick(1).await; + assert_eq!(discharged.candidates_discharged, 1); + assert_eq!(discharged.bodies_deleted, 0); + assert!(discharged.bootstrap_frontier_before.is_none()); + assert!(discharged.bootstrap_frontier_after.is_none()); + assert!(!discharged.has_more); + assert!(body_exists(store.conn(), &body.content_hash).await); + assert_eq!( + store.tick(1).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); +} + #[asupersync::test] async fn rewritten_bootstrap_checkpoint_refuses_commit_and_recovers_across_48_row_pages() { let store = Store::open().await; @@ -630,10 +697,14 @@ async fn rewritten_bootstrap_checkpoint_refuses_commit_and_recovers_across_48_ro .unwrap(); let first = store.tick(48).await; assert_eq!(first.roots_scanned, 48); + assert_eq!(first.bootstrap_frontier_before.as_deref(), Some("raw:0")); + assert_eq!(first.bootstrap_frontier_after.as_deref(), Some("raw:48")); assert!(first.has_more); assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 48); let second = store.tick(48).await; assert_eq!(second.roots_scanned, 4); + assert_eq!(second.bootstrap_frontier_before.as_deref(), Some("raw:48")); + assert_eq!(second.bootstrap_frontier_after.as_deref(), Some("raw:52")); assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 52); let (deleted, bytes) = drain(&store, 48).await; assert_eq!((deleted, bytes), (0, 0), "live bodies remain referenced"); diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 8916ab52886..7c70079ab69 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -386,6 +386,22 @@ fn record_canonical_body_gc_outcome( &[ ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), + ( + "bootstrap_frontier_before", + report + .bootstrap_frontier_before + .as_deref() + .unwrap_or("not_scanned") + .to_owned(), + ), + ( + "bootstrap_frontier_after", + report + .bootstrap_frontier_after + .as_deref() + .unwrap_or("not_scanned") + .to_owned(), + ), ( "elapsed_ms", report From 1f0446809f5d95ecf0d86c022ec886941c2c9e83 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:36:37 -0800 Subject: [PATCH 20/50] fix(retention): report committed empty bootstrap pages --- .../src/observation/canonical_body_gc.rs | 9 +++++++ .../observation/canonical_body_gc/tests.rs | 27 ++++++++++++++++++- crates/tracedecay/src/daemon/maintenance.rs | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 3b2725a92ad..c69ef98cd98 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -45,6 +45,15 @@ pub struct CanonicalBodyGcReport { pub has_more: bool, } +impl CanonicalBodyGcReport { + /// A bootstrap page can commit its remaining frontier after every owner in it was deleted. + pub fn has_committed_work(&self) -> bool { + self.roots_scanned != 0 + || self.bodies_scanned != 0 + || self.bootstrap_frontier_after.is_some() + } +} + #[derive(Clone, Debug, PartialEq, Eq)] struct Progress { raw_through: i64, diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index e66c7b7cb10..149f523b8ef 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -591,7 +591,10 @@ async fn bootstrap_reports_each_committed_sql_frontier_without_claiming_collecti let frontiers = [ ("raw:0".to_owned(), "raw:1".to_owned()), ("observations:0".to_owned(), "observations:1".to_owned()), - ("bodies:".to_owned(), format!("bodies:{}", body.content_hash)), + ( + "bodies:".to_owned(), + format!("bodies:{}", body.content_hash), + ), ]; for (phase, (before, after)) in frontiers.into_iter().enumerate() { let report = store.tick(1).await; @@ -631,6 +634,28 @@ async fn bootstrap_reports_each_committed_sql_frontier_without_claiming_collecti ); } +#[asupersync::test] +async fn empty_bootstrap_page_reports_its_committed_frontier_once() { + let store = Store::open().await; + store.conn().execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) VALUES ('inline', 'text', 'fixture')", () + ).await.unwrap(); + store.install().await; + store + .conn() + .execute("DELETE FROM lcm_raw_messages", ()) + .await + .unwrap(); + + let page = store.tick(48).await; + assert_eq!((page.roots_scanned, page.bodies_scanned), (0, 0)); + assert_eq!(page.bootstrap_frontier_before.as_deref(), Some("raw:0")); + assert_eq!(page.bootstrap_frontier_after.as_deref(), Some("raw:1")); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 1); + assert!(page.has_committed_work()); + assert!(!store.tick(48).await.has_committed_work()); +} + #[asupersync::test] async fn rewritten_bootstrap_checkpoint_refuses_commit_and_recovers_across_48_row_pages() { let store = Store::open().await; diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 7c70079ab69..97278339b41 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -380,7 +380,7 @@ fn record_canonical_body_gc_outcome( ) -> MaintenanceTickOutcome { match outcome { Ok(report) => { - if report.roots_scanned > 0 || report.bodies_scanned > 0 { + if report.has_committed_work() { log_daemon_event( "retention_canonical_bodies", &[ From dfd0c6a10600ecdcd93f7e0fa5ed94611e36de44 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:59:32 -0800 Subject: [PATCH 21/50] simplify(retention): admit replay body bytes once per page --- .../src/retention/replay_metadata.rs | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index dc3dd3b0d0e..79849cb70ca 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -110,7 +110,7 @@ struct Candidate { store_id: i64, eligible: bool, oversized: bool, - read_bytes: Option, + body_size_known: bool, } /// Admit a prefix bounded by both metadata parsing and body-read estimates @@ -185,7 +185,7 @@ async fn candidates( store_id, eligible: false, oversized, - read_bytes, + body_size_known: read_bytes.is_some(), }); } drop(rows); @@ -235,48 +235,38 @@ async fn step( metadata_bytes_examined, ..ReplayMetadataCompactionReport::default() }; - let mut remaining = MAX_BATCH_BYTES; let mut guard_suspended = false; for candidate in candidates { if candidate.oversized { report.rows_skipped += 1; report.rows_skipped_oversized += 1; } else if candidate.eligible { - let bytes = candidate.read_bytes.ok_or_else(|| { - db_error(format!( + if !candidate.body_size_known { + return Err(db_error(format!( "raw row {} has no bounded canonical body", candidate.store_id - )) - })?; - if bytes > MAX_BATCH_BYTES { - report.rows_skipped += 1; - report.rows_skipped_oversized += 1; - } else if bytes > remaining { - break; - } else { - remaining -= bytes; - let message = raw::load_raw_message_by_store_id(conn, candidate.store_id).await?; - if let Some(metadata) = compacted_metadata(&message)? { - if !guard_suspended { - summary_convergence::suspend_raw_update_guard(conn).await?; - guard_suspended = true; - } - let changed = - apply_metadata(conn, candidate.store_id, &message, &metadata).await?; - if changed { - report.rows_compacted += 1; - report.metadata_bytes_removed += message - .metadata_json - .as_deref() - .map_or(0, str::len) - .saturating_sub(metadata.len()) - as u64; - } else { - report.rows_skipped += 1; - } + ))); + } + let message = raw::load_raw_message_by_store_id(conn, candidate.store_id).await?; + if let Some(metadata) = compacted_metadata(&message)? { + if !guard_suspended { + summary_convergence::suspend_raw_update_guard(conn).await?; + guard_suspended = true; + } + let changed = apply_metadata(conn, candidate.store_id, &message, &metadata).await?; + if changed { + report.rows_compacted += 1; + report.metadata_bytes_removed += message + .metadata_json + .as_deref() + .map_or(0, str::len) + .saturating_sub(metadata.len()) + as u64; } else { report.rows_skipped += 1; } + } else { + report.rows_skipped += 1; } } report.rows_scanned += 1; From 9a1c8a27d08bba857319950ffb3e4f0678199443 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:13:20 -0800 Subject: [PATCH 22/50] fix(retention): account for ineligible replay pages Rows skipped by replay eligibility were not counted as skipped, so the reported 48 scanned / 0 compacted / 0 skipped contradicted what the page actually examined. Include those intact rows in the skip count; keep oversized skips as a subset and leave the durable frontier, eligibility, body reads and scheduling unchanged. Assert the real SQLite-backed 48+4 ineligible pages, five preserved representation kinds and multi-tick metadata-budget pages report true skipped counts while preserving content and quiescence. Refs #4787 --- crates/tracedecay-lcm/src/retention/replay_metadata.rs | 6 +++++- .../src/retention/replay_metadata/tests.rs | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index 79849cb70ca..a948c9dfd1b 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -38,7 +38,8 @@ pub struct ReplayMetadataCompactionReport { pub frontier_after: Option, pub rows_compacted: u64, /// Rows retained because their representation, sanitizer result, or byte - /// bound could not prove this narrow lossless transformation. + /// bound could not prove this narrow lossless transformation, including + /// rows rejected by the eligibility check before body verification. pub rows_skipped: u64, /// Subset of skipped rows whose metadata plus canonical/stored body /// representations exceed this pass's 4 MiB read and verification budget. @@ -268,6 +269,9 @@ async fn step( } else { report.rows_skipped += 1; } + } else { + // Eligibility refusal retains the original representation too. + report.rows_skipped += 1; } report.rows_scanned += 1; after = candidate.store_id; diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 5c794d2ed99..fae7245359a 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -250,6 +250,8 @@ async fn structured_different_media_and_external_representations_are_retained() let report = store.tick(48).await; assert_eq!(report.rows_scanned, 5); assert_eq!(report.rows_compacted, 0); + assert_eq!(report.rows_skipped, 5); + assert_eq!(report.rows_skipped_oversized, 0); assert_eq!(report.metadata_bytes_removed, 0); // All five provider representations still carry their original content. assert_eq!(scalar(&store.conn, "SELECT COUNT(*) FROM lcm_raw_messages WHERE json_type(metadata_json, '$.active_replay.content') IS NOT NULL").await, 5); @@ -405,6 +407,8 @@ async fn zero_compaction_pages_advance_distinct_durable_frontiers() { let first = store.tick(48).await; assert_eq!(first.rows_scanned, 48); assert_eq!(first.rows_compacted, 0); + assert_eq!(first.rows_skipped, 48); + assert_eq!(first.rows_skipped_oversized, 0); assert_eq!(first.frontier_before, Some(0)); assert_eq!(first.frontier_after, Some(48)); assert!(first.has_more); @@ -412,6 +416,8 @@ async fn zero_compaction_pages_advance_distinct_durable_frontiers() { let second = store.tick(48).await; assert_eq!(second.rows_scanned, 4); assert_eq!(second.rows_compacted, 0); + assert_eq!(second.rows_skipped, 4); + assert_eq!(second.rows_skipped_oversized, 0); assert_eq!(second.frontier_before, Some(48)); assert_eq!(second.frontier_after, Some(52)); assert!(!second.has_more); @@ -462,7 +468,8 @@ async fn large_ineligible_metadata_is_bounded_before_eligibility_checks() { ticks += 1; assert!(report.rows_scanned > 0); assert_eq!(report.rows_compacted, 0); - assert_eq!(report.rows_skipped, 0); + assert_eq!(report.rows_skipped, report.rows_scanned); + assert_eq!(report.rows_skipped_oversized, 0); assert_eq!(report.metadata_bytes_removed, 0); assert!(report.metadata_bytes_examined > 0); assert!(report.metadata_bytes_examined <= MAX_BATCH_BYTES); From bfba4837c1e75340af3370d6ec69c7815d4ba55a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:39:50 -0800 Subject: [PATCH 23/50] fix(retention): scope SQL frontier logs to each store Mounted session stores can commit identical 48-row pages in the same maintenance cycle. Previously their canonical and replay operator lines had no store identity, so interleaved advancing SQL frontiers appeared to be the same stalled page. Include the exact mounted database path in success and refusal events without changing retention or scheduling. A subprocess regression exercises the actual operator stderr with two stores both reporting 0->48, requires four uniquely attributable log lines, and rejects vacuous zero-test success. Refs #4787. --- crates/tracedecay/src/daemon/maintenance.rs | 88 ++++++++++++++++++++- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 97278339b41..4eeef0fa4b6 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -267,8 +267,10 @@ async fn run_registered_store_retention( Ok(_) => {} } log_payload_gc_outcome(&report.payload_gc); - let canonical_body_outcome = record_canonical_body_gc_outcome(&report.canonical_body_gc); - let replay_metadata_outcome = record_replay_metadata_outcome(&report.replay_metadata); + let canonical_body_outcome = + record_canonical_body_gc_outcome(database.db_path(), &report.canonical_body_gc); + let replay_metadata_outcome = + record_replay_metadata_outcome(database.db_path(), &report.replay_metadata); let outcome = if report.succeeded() { session_storage_outcome .combine(canonical_body_outcome) @@ -368,11 +370,12 @@ async fn run_registered_storage_continuation( ) .await; record_session_lcm_outcome(&session_lcm) - .combine(record_canonical_body_gc_outcome(&canonical_body_gc)) - .combine(record_replay_metadata_outcome(&replay_metadata)) + .combine(record_canonical_body_gc_outcome(database.db_path(), &canonical_body_gc)) + .combine(record_replay_metadata_outcome(database.db_path(), &replay_metadata)) } fn record_canonical_body_gc_outcome( + store: &Path, outcome: &Result< tracedecay_global_db::observation::CanonicalBodyGcReport, tracedecay_maintenance::retention::registered_store::RegisteredStoreRetentionErrorV1, @@ -384,6 +387,7 @@ fn record_canonical_body_gc_outcome( log_daemon_event( "retention_canonical_bodies", &[ + ("store", store.display().to_string()), ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), ( @@ -433,6 +437,7 @@ fn record_canonical_body_gc_outcome( "retention_degraded", &[ ("pass", "canonical_body_gc".to_owned()), + ("store", store.display().to_string()), ("failure", error.diagnostic().to_owned()), ], ); @@ -442,6 +447,7 @@ fn record_canonical_body_gc_outcome( } fn record_replay_metadata_outcome( + store: &Path, outcome: &Result< tracedecay_lcm::retention::ReplayMetadataCompactionReport, tracedecay_maintenance::retention::registered_store::RegisteredStoreRetentionErrorV1, @@ -453,6 +459,7 @@ fn record_replay_metadata_outcome( log_daemon_event( "retention_replay_metadata", &[ + ("store", store.display().to_string()), ("rows_scanned", report.rows_scanned.to_string()), ( "elapsed_ms", @@ -503,6 +510,7 @@ fn record_replay_metadata_outcome( "retention_degraded", &[ ("pass", "replay_metadata_compaction".to_owned()), + ("store", store.display().to_string()), ("failure", error.diagnostic().to_owned()), ], ); @@ -2069,4 +2077,76 @@ mod tests { .expect("native runtime"); assert_eq!(crate::test_support::native_runtime_hazards(), hazards); } + + /// Two mounted databases may commit identical 48-row pages in one tick. + /// Operator logs must not conflate their independent SQL frontiers. + #[test] + fn retention_progress_logs_distinguish_mounted_stores() { + const CHILD: &str = "TRACEDECAY_RETENTION_STORE_LOG_CHILD"; + const TEST: &str = + "daemon::maintenance::tests::retention_progress_logs_distinguish_mounted_stores"; + let stores = ["/project-a/sessions.db", "/project-b/sessions.db"]; + + if std::env::var_os(CHILD).is_some() { + for store in stores { + let canonical = tracedecay_global_db::observation::CanonicalBodyGcReport { + roots_scanned: 48, + bootstrap_frontier_before: Some("raw:0".to_owned()), + bootstrap_frontier_after: Some("raw:48".to_owned()), + has_more: true, + ..Default::default() + }; + let _ = super::record_canonical_body_gc_outcome(Path::new(store), &Ok(canonical)); + let replay = tracedecay_lcm::retention::ReplayMetadataCompactionReport { + rows_scanned: 48, + rows_skipped: 48, + frontier_before: Some(0), + frontier_after: Some(48), + has_more: true, + ..Default::default() + }; + let _ = super::record_replay_metadata_outcome(Path::new(store), &Ok(replay)); + } + return; + } + + // Capture the actual operator stderr lines in a child process. The + // normal daemon logger deliberately bypasses the test output capture. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST, "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output() + .expect("spawn isolated operator log test"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success() && stdout.contains("1 passed; 0 failed"), + "isolated test did not execute:\n{stdout}\n{stderr}" + ); + + let lines = stderr + .lines() + .filter(|line| { + line.contains("event=retention_canonical_bodies ") + || line.contains("event=retention_replay_metadata ") + }) + .collect::>(); + assert_eq!(lines.len(), 4, "expected two events per store: {stderr}"); + for store in stores { + for (event, frontier) in [ + ("retention_canonical_bodies", "bootstrap_frontier_after=raw:48"), + ("retention_replay_metadata", "frontier_after=48"), + ] { + let count = lines + .iter() + .filter(|line| { + line.contains(&format!("event={event} ")) + && line.contains(&format!("store={store} ")) + && line.contains(frontier) + }) + .count(); + assert_eq!(count, 1, "missing distinct {event} page for {store}: {stderr}"); + } + } + } } From ddabf4a972f3a89d745204d65af29023e0b4be28 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:11:43 -0800 Subject: [PATCH 24/50] style(retention): format store-scoped frontier diagnostics --- crates/tracedecay/src/daemon/maintenance.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 4eeef0fa4b6..a7c1dce45d1 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -370,8 +370,14 @@ async fn run_registered_storage_continuation( ) .await; record_session_lcm_outcome(&session_lcm) - .combine(record_canonical_body_gc_outcome(database.db_path(), &canonical_body_gc)) - .combine(record_replay_metadata_outcome(database.db_path(), &replay_metadata)) + .combine(record_canonical_body_gc_outcome( + database.db_path(), + &canonical_body_gc, + )) + .combine(record_replay_metadata_outcome( + database.db_path(), + &replay_metadata, + )) } fn record_canonical_body_gc_outcome( @@ -2134,7 +2140,10 @@ mod tests { assert_eq!(lines.len(), 4, "expected two events per store: {stderr}"); for store in stores { for (event, frontier) in [ - ("retention_canonical_bodies", "bootstrap_frontier_after=raw:48"), + ( + "retention_canonical_bodies", + "bootstrap_frontier_after=raw:48", + ), ("retention_replay_metadata", "frontier_after=48"), ] { let count = lines @@ -2145,7 +2154,10 @@ mod tests { && line.contains(frontier) }) .count(); - assert_eq!(count, 1, "missing distinct {event} page for {store}: {stderr}"); + assert_eq!( + count, 1, + "missing distinct {event} page for {store}: {stderr}" + ); } } } From c3f2d94d2bcc4b7c70e074e06e3b7b294c2452aa Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:35:47 -0800 Subject: [PATCH 25/50] fix(retention): refuse orphaned canonical bodies on vetoed delete Keep selected unreferenced body candidates queued if the underlying DELETE was ignored. Verify the body is absent before discharging its candidate; otherwise rollback the page instead of silently leaking unreclaimed bodies. Exercise 52 real candidates with a vetoed DELETE, rollback, recovery, 48+4 committed reclamation, and an idle final tick. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 21 ++++++ .../observation/canonical_body_gc/tests.rs | 74 +++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index c69ef98cd98..e8513b02a21 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -372,6 +372,27 @@ async fn collect_candidates( report.stored_bytes_deleted = report.stored_bytes_deleted.saturating_add(bytes); } drop(deleted); + // An ignored or otherwise suppressed body DELETE must not silently + // discharge an unreferenced candidate. Once that candidate disappears, + // no owner-triggered re-enqueue is guaranteed, and the body can persist + // indefinitely without another collection attempt. + let mut uncollected = conn + .query( + "SELECT 1 FROM session_canonical_bodies b + WHERE b.content_hash IN (SELECT value FROM json_each(?1)) + AND NOT EXISTS (SELECT 1 FROM canonical_body_gc_refs r + WHERE r.content_hash = b.content_hash) + LIMIT 1", + params![hashes.as_str()], + ) + .await + .map_err(db_error)?; + if uncollected.next().await.map_err(db_error)?.is_some() { + return Err(integrity_error( + "canonical body collection left an unreferenced body behind", + )); + } + drop(uncollected); // Referenced candidates are discharged too. The final owner removal // records them again; idle ticks never revisit every retained body. // Deleting an unreferenced body cascades to its candidate row. The diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 149f523b8ef..ba18b93368b 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -515,6 +515,80 @@ async fn nonpositive_legacy_owner_ids_cannot_be_skipped_by_bootstrap() { } } +#[asupersync::test] +async fn ignored_body_delete_rolls_back_candidates_then_reclaims_distinct_pages() { + let store = Store::open().await; + store.install().await; + let mut hashes = Vec::new(); + for ordinal in 0..52 { + let body = + StoredCanonicalBody::pack(format!("unreferenced candidate {ordinal}").as_bytes()) + .unwrap(); + insert_body(store.conn(), &body).await; + hashes.push(body.content_hash); + } + assert_eq!(candidate_count(store.conn()).await, 52); + store + .conn() + .execute_batch( + "CREATE TRIGGER refuse_body_gc_delete + BEFORE DELETE ON session_canonical_bodies BEGIN + SELECT RAISE(IGNORE); + END", + ) + .await + .unwrap(); + + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error + .to_string() + .contains("collection left an unreferenced body behind") + ); + transaction.rollback().await.unwrap(); + assert_eq!( + candidate_count(store.conn()).await, + 52, + "a vetoed DELETE cannot discard the only future collection opportunity" + ); + for hash in &hashes { + assert!(body_exists(store.conn(), hash).await); + } + + store + .conn() + .execute_batch("DROP TRIGGER refuse_body_gc_delete") + .await + .unwrap(); + let first = store.tick(48).await; + assert_eq!(first.bodies_scanned, 48); + assert_eq!(first.bodies_deleted, 48); + assert_eq!(first.candidates_discharged, 48); + assert!(first.has_more); + assert_eq!(candidate_count(store.conn()).await, 4); + let second = store.tick(48).await; + assert_eq!(second.bodies_scanned, 4); + assert_eq!(second.bodies_deleted, 4); + assert_eq!(second.candidates_discharged, 4); + assert!(!second.has_more); + assert_eq!(candidate_count(store.conn()).await, 0); + for hash in &hashes { + assert!(!body_exists(store.conn(), hash).await); + } + assert_eq!( + store.tick(48).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); +} + #[asupersync::test] async fn zero_delete_pages_discharge_distinct_candidates_and_quiesce() { let store = Store::open().await; From eb9bd17c6eea314364739fb9a57c191a52dcc3ae Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:51:36 -0800 Subject: [PATCH 26/50] perf(retention): batch canonical bootstrap writes Replace per-owner and per-body bootstrap INSERT submissions with one indexed INSERT SELECT per bounded page. Keep the selected last identity, sparse-key semantics, eligibility predicate and transactional checkpoint. Add real SQLite executor regressions for bounded write submissions, 52-row sparse frontiers, live-body preservation, eligibility at the page tail, and late statement failure with full rollback and successful retry. Preserve the concurrent vetoed-delete repair without rewriting history. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 50 +-- .../bootstrap_write_tests.rs | 319 ++++++++++++++++++ 2 files changed, 346 insertions(+), 23 deletions(-) create mode 100644 crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index e8513b02a21..8d86d11440e 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -14,6 +14,8 @@ use tracedecay_runtime_core::db::Database; use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use tracedecay_store::collect_body_refs; +#[cfg(test)] +mod bootstrap_write_tests; pub(crate) mod schema; #[cfg(test)] mod tests; @@ -218,33 +220,31 @@ async fn bootstrap_raw( ) -> Result { let mut rows = conn .query( - "SELECT store_id, CASE WHEN storage_kind = 'inline' AND content IS NULL - THEN content_hash END - FROM lcm_raw_messages WHERE store_id > ?1 AND store_id <= ?2 - ORDER BY store_id LIMIT ?3", + "SELECT store_id FROM lcm_raw_messages + WHERE store_id > ?1 AND store_id <= ?2 ORDER BY store_id LIMIT ?3", params![progress.raw_through, progress.raw_until, limit], ) .await .map_err(db_error)?; let mut page = Vec::new(); while let Some(row) = rows.next().await.map_err(db_error)? { - page.push(( - row.get::(0).map_err(db_error)?, - row.get::>(1).map_err(db_error)?, - )); + page.push(row.get::(0).map_err(db_error)?); } drop(rows); - for (owner, hash) in &page { - if let Some(hash) = hash { - conn.execute( - "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) - VALUES ('raw', ?1, ?2)", - params![*owner, hash.as_str()], - ) - .await - .map_err(db_error)?; - } - progress.raw_through = *owner; + if let Some(&last) = page.last() { + // Both reads share the writer transaction. The last selected identity + // bounds this indexed insert to the same page, including sparse IDs, + // without one worker submission for every referenced owner. + conn.execute( + "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + SELECT 'raw', store_id, content_hash FROM lcm_raw_messages + WHERE store_id > ?1 AND store_id <= ?2 + AND storage_kind = 'inline' AND content IS NULL", + params![progress.raw_through, last], + ) + .await + .map_err(db_error)?; + progress.raw_through = last; } if page.len() < limit as usize { progress.raw_through = progress.raw_until; @@ -317,14 +317,18 @@ async fn bootstrap_bodies( page.push(row.get::(0).map_err(db_error)?); } drop(rows); - for hash in &page { + if let Some(last) = page.last() { + // Seed the selected key range in one statement, not one asynchronous + // writer submission per body. No body payload is materialized here. conn.execute( - "INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) VALUES (?1)", - params![hash.as_str()], + "INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) + SELECT content_hash FROM session_canonical_bodies + WHERE content_hash > ?1 AND content_hash <= ?2", + params![progress.bodies_after.as_str(), last.as_str()], ) .await .map_err(db_error)?; - progress.bodies_after.clone_from(hash); + progress.bodies_after.clone_from(last); } if page.len() < limit as usize { progress.bodies_after.clone_from(&progress.bodies_until); diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs new file mode 100644 index 00000000000..5630d1426c7 --- /dev/null +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs @@ -0,0 +1,319 @@ +//! Measure real bootstrap write submissions without replacing the SQLite executor. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use tracedecay_runtime_core::db::engine::{ + Executor, IntoParams, QueryExecutor, Result as EngineResult, Rows, TestConnection, + TransactionBehavior, WriteStatement, params, +}; +use tracedecay_store::{CANONICAL_BODIES_TABLE_SQL, StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL}; + +use super::{CanonicalBodyGcReport, read_progress, schema, step}; + +struct CountWrites<'a, E: ?Sized> { + inner: &'a E, + submissions: AtomicUsize, +} + +impl QueryExecutor for CountWrites<'_, E> { + async fn query

(&self, sql: &str, params: P) -> EngineResult + where + P: IntoParams, + { + self.inner.query(sql, params).await + } +} + +impl Executor for CountWrites<'_, E> { + async fn execute

(&self, sql: &str, params: P) -> EngineResult + where + P: IntoParams, + { + self.submissions.fetch_add(1, Ordering::Relaxed); + self.inner.execute(sql, params).await + } + + async fn execute_statements(&self, statements: Vec) -> EngineResult> { + self.submissions.fetch_add(1, Ordering::Relaxed); + self.inner.execute_statements(statements).await + } + + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { + self.submissions.fetch_add(1, Ordering::Relaxed); + self.inner.execute_batch(sql).await + } +} + +struct Fixture { + conn: TestConnection, + bodies: Vec, + _directory: tempfile::TempDir, +} + +impl Fixture { + async fn open() -> Self { + let directory = tempfile::tempdir().unwrap(); + let conn = TestConnection::open(&directory.path().join("bootstrap-writes.db")); + conn.execute_batch(CANONICAL_BODIES_TABLE_SQL).await.unwrap(); + conn.execute_batch( + "CREATE TABLE lcm_raw_messages ( + store_id INTEGER PRIMARY KEY AUTOINCREMENT, + storage_kind TEXT NOT NULL, content TEXT, content_hash TEXT NOT NULL + ); + CREATE TABLE observations ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + observation_json TEXT NOT NULL + );", + ) + .await + .unwrap(); + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let mut bodies = Vec::new(); + for ordinal in 1..=52_i64 { + let text = format!("retained bootstrap body {ordinal}"); + let body = StoredCanonicalBody::pack(text.as_bytes()).unwrap(); + transaction + .execute( + UPSERT_CANONICAL_BODY_SQL, + params![ + body.content_hash.as_str(), + body.encoding, + body.blob.as_slice(), + body.uncompressed_bytes + ], + ) + .await + .unwrap(); + // Holes in the key space must not turn the page into an ID count. + transaction + .execute( + "INSERT INTO lcm_raw_messages(store_id, storage_kind, content_hash) + VALUES (?1, 'inline', ?2)", + params![ordinal * 2, body.content_hash.as_str()], + ) + .await + .unwrap(); + bodies.push(body); + } + schema::install(&transaction).await.unwrap(); + transaction.commit().await.unwrap(); + Self { + conn, + bodies, + _directory: directory, + } + } + + async fn tick(&self) -> (CanonicalBodyGcReport, usize) { + let transaction = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let (report, submissions) = { + let counted = CountWrites { + inner: &transaction, + submissions: AtomicUsize::new(0), + }; + let report = step(&counted, 48).await.unwrap(); + (report, counted.submissions.load(Ordering::Relaxed)) + }; + transaction.commit().await.unwrap(); + (report, submissions) + } + + async fn assert_bodies_preserved(&self) { + let count = scalar(&self.conn, "SELECT COUNT(*) FROM session_canonical_bodies").await; + assert_eq!(count, self.bodies.len() as i64); + for body in &self.bodies { + let mut rows = self + .conn + .query( + "SELECT body FROM session_canonical_bodies WHERE content_hash = ?1", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + let stored = rows.next().await.unwrap().unwrap(); + assert_eq!(stored.get::>(0).unwrap(), body.blob); + } + } +} + +async fn scalar(conn: &(impl QueryExecutor + ?Sized), sql: &str) -> i64 { + conn.query(sql, ()) + .await + .unwrap() + .next() + .await + .unwrap() + .unwrap() + .get(0) + .unwrap() +} + +async fn raw_owners(conn: &(impl QueryExecutor + ?Sized)) -> Vec { + let mut rows = conn + .query( + "SELECT owner_id FROM canonical_body_gc_refs WHERE owner_kind = 'raw' + ORDER BY owner_id", + (), + ) + .await + .unwrap(); + let mut owners = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + owners.push(row.get(0).unwrap()); + } + owners +} + +#[asupersync::test] +async fn bootstrap_pages_batch_writes_and_preserve_sparse_frontiers() { + let fixture = Fixture::open().await; + for (scanned, through, owners) in [(48_u64, 96_i64, 48_i64), (4, 104, 52)] { + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.conn).await.unwrap(); + assert_eq!(report.roots_scanned, scanned); + assert_eq!(submissions, 2, "one seed submission plus the checkpoint"); + assert_eq!(progress.raw_through, through); + assert_eq!( + raw_owners(&fixture.conn).await, + (1..=owners).map(|ordinal| ordinal * 2).collect::>() + ); + assert_eq!(report.bodies_deleted, 0); + assert!(report.has_more); + } + + let mut hashes = fixture + .bodies + .iter() + .map(|body| body.content_hash.clone()) + .collect::>(); + hashes.sort(); + for (scanned, total) in [(48_u64, 48_usize), (4, 52)] { + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.conn).await.unwrap(); + let candidates = scalar( + &fixture.conn, + "SELECT COUNT(*) FROM canonical_body_gc_candidates", + ) + .await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!(submissions, 2, "one candidate seed plus the checkpoint"); + assert_eq!(progress.bodies_after, hashes[total - 1]); + assert_eq!(candidates, total as i64); + assert_eq!(report.bodies_deleted, 0); + assert!(report.has_more); + } + + for scanned in [48_u64, 4] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!(report.candidates_discharged, scanned); + assert_eq!(report.bodies_deleted, 0); + assert_eq!(report.has_more, scanned == 48); + } + let (idle, submissions) = fixture.tick().await; + assert!(!idle.has_committed_work()); + assert!(!idle.has_more); + assert_eq!(submissions, 0); + fixture.assert_bodies_preserved().await; +} + +#[asupersync::test] +async fn bootstrap_batch_refusal_rolls_back_and_recovers() { + for bodies_phase in [false, true] { + let fixture = Fixture::open().await; + if bodies_phase { + fixture.tick().await; + fixture.tick().await; + } + let before = read_progress(&fixture.conn).await.unwrap(); + let table = if bodies_phase { + "canonical_body_gc_candidates" + } else { + "canonical_body_gc_refs" + }; + // FAIL deliberately leaves the first 47 inserts in the transaction. + // The collector's rollback must discard them with the uncommitted page. + fixture + .conn + .execute_batch(&format!( + "CREATE TRIGGER test_bootstrap_refusal BEFORE INSERT ON {table} + WHEN (SELECT COUNT(*) FROM {table}) = 47 BEGIN + SELECT RAISE(FAIL, 'injected bootstrap batch refusal'); + END" + )) + .await + .unwrap(); + let transaction = fixture + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("injected bootstrap batch refusal")); + transaction.rollback().await.unwrap(); + assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); + let count = scalar(&fixture.conn, &format!("SELECT COUNT(*) FROM {table}")).await; + assert_eq!(count, 0); + fixture.assert_bodies_preserved().await; + fixture + .conn + .execute_batch("DROP TRIGGER test_bootstrap_refusal") + .await + .unwrap(); + for scanned in [48_u64, 4] { + let (report, submissions) = fixture.tick().await; + let actual = if bodies_phase { + report.bodies_scanned + } else { + report.roots_scanned + }; + assert_eq!(submissions, 2); + assert_eq!(actual, scanned); + assert_eq!(report.bodies_deleted, 0); + } + let count = scalar(&fixture.conn, &format!("SELECT COUNT(*) FROM {table}")).await; + assert_eq!(count, 52); + fixture.assert_bodies_preserved().await; + } +} + +#[asupersync::test] +async fn bootstrap_batch_eligibility_does_not_change_the_page_boundary() { + let fixture = Fixture::open().await; + fixture + .conn + .execute_batch( + "UPDATE lcm_raw_messages SET storage_kind = 'external' WHERE store_id = 94; + UPDATE lcm_raw_messages SET content = 'owned inline text' WHERE store_id = 96;", + ) + .await + .unwrap(); + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.conn).await.unwrap(); + assert_eq!(report.roots_scanned, 48); + assert_eq!(submissions, 2); + assert_eq!(progress.raw_through, 96); + assert_eq!( + raw_owners(&fixture.conn).await, + (1..=46_i64).map(|ordinal| ordinal * 2).collect::>() + ); + assert!(report.has_more); + let (tail, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.conn).await.unwrap(); + assert_eq!(tail.roots_scanned, 4); + assert_eq!(submissions, 2); + assert_eq!(progress.raw_through, 104); + let expected = (1..=52_i64) + .filter(|ordinal| ![47, 48].contains(ordinal)) + .map(|ordinal| ordinal * 2) + .collect::>(); + assert_eq!(raw_owners(&fixture.conn).await, expected); + fixture.assert_bodies_preserved().await; +} From 9b50818107afea5486193022122c3a45f9a5b40d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:26:27 -0800 Subject: [PATCH 27/50] perf(retention): batch observation bootstrap writer submissions Group validated observation reference inserts into bounded storage-worker submissions instead of one asynchronous request per referenced hash. Preserve Rust reference parsing, identity order and transactional rollback. Prove 48+4 pages across the 128-statement boundary, durable SQL frontiers, partial-batch refusal/rollback, reference deduplication and quiescence. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 31 +++- .../bootstrap_write_tests.rs | 143 ++++++++++++++++++ 2 files changed, 166 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 8d86d11440e..bce70bf46aa 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -11,7 +11,7 @@ use std::time::Instant; use serde::{Deserialize, Serialize}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::db::Database; -use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, WriteStatement, params}; use tracedecay_store::collect_body_refs; #[cfg(test)] @@ -277,18 +277,33 @@ async fn bootstrap_observations( )); } drop(rows); + // Parsing preserves the canonical Rust reference contract, including + // value deduplication. Submit validated writes in bounded groups so an + // observation page cannot schedule one storage-worker hop per body hash. + // The caller's writer transaction rolls back every group on any failure. + const MAX_PENDING_OBSERVATION_REFS: usize = 128; + let mut pending = Vec::new(); for (owner, json) in &page { for hash in collect_body_refs(json).map_err(db_error)? { - conn.execute( - "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) - VALUES ('observation', ?1, ?2)", - params![*owner, hash], - ) - .await - .map_err(db_error)?; + pending.push( + WriteStatement::new( + "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + VALUES ('observation', ?1, ?2)", + params![*owner, hash], + ) + .map_err(db_error)?, + ); + if pending.len() == MAX_PENDING_OBSERVATION_REFS { + conn.execute_statements(std::mem::take(&mut pending)) + .await + .map_err(db_error)?; + } } progress.observations_through = *owner; } + if !pending.is_empty() { + conn.execute_statements(pending).await.map_err(db_error)?; + } if page.len() < limit as usize { progress.observations_through = progress.observations_until; } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs index 5630d1426c7..56da4607408 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs @@ -52,6 +52,10 @@ struct Fixture { impl Fixture { async fn open() -> Self { + Self::with_observations(false).await + } + + async fn with_observations(include_observations: bool) -> Self { let directory = tempfile::tempdir().unwrap(); let conn = TestConnection::open(&directory.path().join("bootstrap-writes.db")); conn.execute_batch(CANONICAL_BODIES_TABLE_SQL).await.unwrap(); @@ -96,6 +100,35 @@ impl Fixture { ) .await .unwrap(); + if include_observations { + // Three distinct hashes from the third owner onward, so a + // 48-owner page crosses the 128-write submission boundary. + let current = body.content_hash.as_str(); + let previous = bodies + .last() + .map_or(current, |body: &StoredCanonicalBody| body.content_hash.as_str()); + let earlier = bodies + .iter() + .rev() + .nth(1) + .map_or(current, |body| body.content_hash.as_str()); + let json = serde_json::json!({ + "payload": {"a": null, "b": null, "c": null}, + "_tracedecay_canonical_body_refs": { + "/payload/a": current, + "/payload/b": previous, + "/payload/c": earlier + } + }) + .to_string(); + transaction + .execute( + "INSERT INTO observations(sequence, observation_json) VALUES (?1, ?2)", + params![ordinal * 2, json], + ) + .await + .unwrap(); + } bodies.push(body); } schema::install(&transaction).await.unwrap(); @@ -317,3 +350,113 @@ async fn bootstrap_batch_eligibility_does_not_change_the_page_boundary() { assert_eq!(raw_owners(&fixture.conn).await, expected); fixture.assert_bodies_preserved().await; } + +async fn observation_owners(conn: &(impl QueryExecutor + ?Sized)) -> Vec { + let mut rows = conn + .query( + "SELECT DISTINCT owner_id FROM canonical_body_gc_refs + WHERE owner_kind = 'observation' ORDER BY owner_id", + (), + ) + .await + .unwrap(); + let mut owners = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + owners.push(row.get(0).unwrap()); + } + owners +} + +#[asupersync::test] +async fn observation_bootstrap_batches_across_a_bounded_write_submission() { + let fixture = Fixture::with_observations(true).await; + for _ in 0..2 { + fixture.tick().await; + } + for (scanned, frontier, owners, refs, submissions) in [ + (48_u64, 96_i64, 48_i64, 141_i64, 3_usize), + (4, 104, 52, 153, 2), + ] { + let (report, actual_submissions) = fixture.tick().await; + assert_eq!(report.roots_scanned, scanned); + assert_eq!(report.bootstrap_frontier_after, Some(format!("observations:{frontier}"))); + assert_eq!(read_progress(&fixture.conn).await.unwrap().observations_through, frontier); + assert_eq!(actual_submissions, submissions); + assert_eq!( + observation_owners(&fixture.conn).await, + (1..=owners).map(|ordinal| ordinal * 2).collect::>() + ); + assert_eq!( + scalar(&fixture.conn, "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'") + .await, + refs, + "duplicate references within an observation must stay deduplicated" + ); + assert_eq!(report.bodies_deleted, 0); + assert!(report.has_more); + } + let mut finished = false; + for _ in 0..6 { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_deleted, 0); + if !report.has_more { + finished = true; + break; + } + } + assert!(finished, "bounded bootstrap and candidate discharge must terminate"); + fixture.assert_bodies_preserved().await; + let (idle, submissions) = fixture.tick().await; + assert!(!idle.has_more); + assert!(!idle.has_committed_work()); + assert_eq!(submissions, 0); +} + +#[asupersync::test] +async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recovers() { + let fixture = Fixture::with_observations(true).await; + for _ in 0..2 { + fixture.tick().await; + } + let before = read_progress(&fixture.conn).await.unwrap(); + fixture + .conn + .execute_batch( + "CREATE TRIGGER refuse_observation_seed BEFORE INSERT ON canonical_body_gc_refs + WHEN NEW.owner_kind = 'observation' AND NEW.owner_id = 94 BEGIN + SELECT RAISE(FAIL, 'injected observation batch refusal'); + END", + ) + .await + .unwrap(); + let transaction = fixture + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("injected observation batch refusal")); + transaction.rollback().await.unwrap(); + assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); + assert!(observation_owners(&fixture.conn).await.is_empty()); + fixture.assert_bodies_preserved().await; + + fixture + .conn + .execute_batch("DROP TRIGGER refuse_observation_seed") + .await + .unwrap(); + for (frontier, references, submissions) in [(96_i64, 141_i64, 3_usize), (104, 153, 2)] { + let (report, actual_submissions) = fixture.tick().await; + assert_eq!(report.bootstrap_frontier_after, Some(format!("observations:{frontier}"))); + assert_eq!(actual_submissions, submissions); + assert_eq!( + scalar(&fixture.conn, "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'") + .await, + references + ); + } + fixture.assert_bodies_preserved().await; +} From 6edf06957b7bff930d19e6956f6573ac90d22da7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:35:46 -0800 Subject: [PATCH 28/50] fix(retention): verify bootstrap seed integrity before advancing After batched bootstrap writes, verify the selected raw owner references, Rust-validated observation references, and canonical-body candidates exist inside the writer transaction. A suppressed INSERT must not commit a frontier that can drop a live body or permanently abandon an orphan. Exercise RAISE(IGNORE) for all three phases, rollback, 48+4 recovery, and live-data preservation. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 72 ++++++++ .../bootstrap_write_tests.rs | 155 +++++++++++++++++- 2 files changed, 225 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index bce70bf46aa..a95b950ddab 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -244,6 +244,29 @@ async fn bootstrap_raw( ) .await .map_err(db_error)?; + // A suppressed INSERT must not advance bootstrap past an owner whose + // canonical body would subsequently appear unreferenced. + let mut missing = conn + .query( + "SELECT 1 FROM lcm_raw_messages raw + WHERE raw.store_id > ?1 AND raw.store_id <= ?2 + AND raw.storage_kind = 'inline' AND raw.content IS NULL + AND NOT EXISTS ( + SELECT 1 FROM canonical_body_gc_refs r + WHERE r.owner_kind = 'raw' AND r.owner_id = raw.store_id + AND r.content_hash = raw.content_hash + ) LIMIT 1", + params![progress.raw_through, last], + ) + .await + .map_err(db_error)?; + let incomplete = missing.next().await.map_err(db_error)?.is_some(); + drop(missing); + if incomplete { + return Err(integrity_error( + "canonical raw bootstrap reference was not indexed", + )); + } progress.raw_through = last; } if page.len() < limit as usize { @@ -283,8 +306,10 @@ async fn bootstrap_observations( // The caller's writer transaction rolls back every group on any failure. const MAX_PENDING_OBSERVATION_REFS: usize = 128; let mut pending = Vec::new(); + let mut expected = Vec::new(); for (owner, json) in &page { for hash in collect_body_refs(json).map_err(db_error)? { + expected.push((*owner, hash.clone())); pending.push( WriteStatement::new( "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) @@ -304,6 +329,32 @@ async fn bootstrap_observations( if !pending.is_empty() { conn.execute_statements(pending).await.map_err(db_error)?; } + if !expected.is_empty() { + // Verify the Rust-validated reference set, not a second parse of raw + // observation JSON. INSERT OR IGNORE and SQLite trigger RAISE(IGNORE) + // can both report success without creating a missing reference. + let expected_json = serde_json::to_string(&expected).map_err(db_error)?; + let mut missing = conn + .query( + "SELECT 1 FROM json_each(?1) pair + WHERE NOT EXISTS ( + SELECT 1 FROM canonical_body_gc_refs r + WHERE r.owner_kind = 'observation' + AND r.owner_id = json_extract(pair.value, '$[0]') + AND r.content_hash = json_extract(pair.value, '$[1]') + ) LIMIT 1", + params![expected_json], + ) + .await + .map_err(db_error)?; + let incomplete = missing.next().await.map_err(db_error)?.is_some(); + drop(missing); + if incomplete { + return Err(integrity_error( + "canonical observation bootstrap reference was not indexed", + )); + } + } if page.len() < limit as usize { progress.observations_through = progress.observations_until; } @@ -343,6 +394,27 @@ async fn bootstrap_bodies( ) .await .map_err(db_error)?; + // A vetoed candidate insert otherwise leaves an orphaned body behind + // the permanently committed bootstrap cursor, with no future seed. + let mut missing = conn + .query( + "SELECT 1 FROM session_canonical_bodies b + WHERE b.content_hash > ?1 AND b.content_hash <= ?2 + AND NOT EXISTS ( + SELECT 1 FROM canonical_body_gc_candidates c + WHERE c.content_hash = b.content_hash + ) LIMIT 1", + params![progress.bodies_after.as_str(), last.as_str()], + ) + .await + .map_err(db_error)?; + let incomplete = missing.next().await.map_err(db_error)?.is_some(); + drop(missing); + if incomplete { + return Err(integrity_error( + "canonical body bootstrap candidate was not indexed", + )); + } progress.bodies_after.clone_from(last); } if page.len() < limit as usize { diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs index 56da4607408..5e71811ca92 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs @@ -379,8 +379,14 @@ async fn observation_bootstrap_batches_across_a_bounded_write_submission() { ] { let (report, actual_submissions) = fixture.tick().await; assert_eq!(report.roots_scanned, scanned); - assert_eq!(report.bootstrap_frontier_after, Some(format!("observations:{frontier}"))); - assert_eq!(read_progress(&fixture.conn).await.unwrap().observations_through, frontier); + assert_eq!( + report.bootstrap_frontier_after, + Some(format!("observations:{frontier}")) + ); + assert_eq!( + read_progress(&fixture.conn).await.unwrap().observations_through, + frontier + ); assert_eq!(actual_submissions, submissions); assert_eq!( observation_owners(&fixture.conn).await, @@ -460,3 +466,148 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover } fixture.assert_bodies_preserved().await; } + +async fn assert_refused_bootstrap_rolls_back(fixture: &Fixture, diagnosis: &str) { + let before = read_progress(&fixture.conn).await.unwrap(); + let transaction = fixture + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error.to_string().contains(diagnosis), + "unexpected bootstrap refusal: {error}" + ); + transaction.rollback().await.unwrap(); + assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); + fixture.assert_bodies_preserved().await; +} + +#[asupersync::test] +async fn ignored_raw_reference_insert_refuses_checkpoint_then_recovers() { + let fixture = Fixture::open().await; + fixture + .conn + .execute_batch( + "CREATE TRIGGER ignore_raw_ref BEFORE INSERT ON canonical_body_gc_refs + WHEN NEW.owner_kind = 'raw' AND NEW.owner_id = 94 + BEGIN SELECT RAISE(IGNORE); END", + ) + .await + .unwrap(); + assert_refused_bootstrap_rolls_back(&fixture, "raw bootstrap reference was not indexed").await; + assert!(raw_owners(&fixture.conn).await.is_empty()); + fixture + .conn + .execute_batch("DROP TRIGGER ignore_raw_ref") + .await + .unwrap(); + for (scanned, frontier) in [(48_u64, 96_i64), (4, 104)] { + let (report, _) = fixture.tick().await; + assert_eq!(report.roots_scanned, scanned); + assert_eq!( + read_progress(&fixture.conn).await.unwrap().raw_through, + frontier + ); + } + assert_eq!(raw_owners(&fixture.conn).await.len(), 52); + fixture.assert_bodies_preserved().await; +} + +#[asupersync::test] +async fn ignored_observation_reference_insert_refuses_checkpoint_then_recovers() { + let fixture = Fixture::with_observations(true).await; + fixture.tick().await; + fixture.tick().await; + fixture + .conn + .execute_batch( + "CREATE TRIGGER ignore_observation_ref BEFORE INSERT ON canonical_body_gc_refs + WHEN NEW.owner_kind = 'observation' AND NEW.owner_id = 94 + BEGIN SELECT RAISE(IGNORE); END", + ) + .await + .unwrap(); + assert_refused_bootstrap_rolls_back( + &fixture, + "observation bootstrap reference was not indexed", + ) + .await; + assert!(observation_owners(&fixture.conn).await.is_empty()); + fixture + .conn + .execute_batch("DROP TRIGGER ignore_observation_ref") + .await + .unwrap(); + for (frontier, references) in [(96_i64, 141_i64), (104, 153)] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bootstrap_frontier_after, Some(format!("observations:{frontier}"))); + assert_eq!(read_progress(&fixture.conn).await.unwrap().observations_through, frontier); + assert_eq!( + scalar( + &fixture.conn, + "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'", + ) + .await, + references, + ); + } + fixture.assert_bodies_preserved().await; +} + +#[asupersync::test] +async fn ignored_candidate_insert_refuses_checkpoint_then_recovers() { + let fixture = Fixture::open().await; + fixture.tick().await; + fixture.tick().await; + let mut hashes = fixture + .bodies + .iter() + .map(|body| body.content_hash.as_str()) + .collect::>(); + hashes.sort_unstable(); + let skipped = hashes[47]; + fixture + .conn + .execute_batch(&format!( + "CREATE TRIGGER ignore_candidate_seed BEFORE INSERT ON canonical_body_gc_candidates + WHEN NEW.content_hash = '{skipped}' + BEGIN SELECT RAISE(IGNORE); END" + )) + .await + .unwrap(); + assert_refused_bootstrap_rolls_back(&fixture, "body bootstrap candidate was not indexed").await; + assert_eq!( + scalar( + &fixture.conn, + "SELECT COUNT(*) FROM canonical_body_gc_candidates", + ) + .await, + 0 + ); + fixture + .conn + .execute_batch("DROP TRIGGER ignore_candidate_seed") + .await + .unwrap(); + for (scanned, candidate_total) in [(48_u64, 48_i64), (4, 52)] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!( + scalar(&fixture.conn, "SELECT COUNT(*) FROM canonical_body_gc_candidates").await, + candidate_total + ); + } + for scanned in [48_u64, 4] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!(report.candidates_discharged, scanned); + assert_eq!(report.bodies_deleted, 0, "referenced bodies are preserved"); + } + let (idle, submissions) = fixture.tick().await; + assert!(!idle.has_more); + assert_eq!(submissions, 0); + fixture.assert_bodies_preserved().await; +} From 464e7863ced43b8d21ad2e214ed1a084598700d4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:12:43 -0800 Subject: [PATCH 29/50] style(retention): format bootstrap tests for Bazel Apply the exact rustfmt output reported by the failed repository gate without changing the retention behavior or regression assertions. Refs #4787 --- .../bootstrap_write_tests.rs | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs index 5e71811ca92..03ae1ab4bbe 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs @@ -6,7 +6,9 @@ use tracedecay_runtime_core::db::engine::{ Executor, IntoParams, QueryExecutor, Result as EngineResult, Rows, TestConnection, TransactionBehavior, WriteStatement, params, }; -use tracedecay_store::{CANONICAL_BODIES_TABLE_SQL, StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL}; +use tracedecay_store::{ + CANONICAL_BODIES_TABLE_SQL, StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL, +}; use super::{CanonicalBodyGcReport, read_progress, schema, step}; @@ -58,7 +60,9 @@ impl Fixture { async fn with_observations(include_observations: bool) -> Self { let directory = tempfile::tempdir().unwrap(); let conn = TestConnection::open(&directory.path().join("bootstrap-writes.db")); - conn.execute_batch(CANONICAL_BODIES_TABLE_SQL).await.unwrap(); + conn.execute_batch(CANONICAL_BODIES_TABLE_SQL) + .await + .unwrap(); conn.execute_batch( "CREATE TABLE lcm_raw_messages ( store_id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -104,9 +108,9 @@ impl Fixture { // Three distinct hashes from the third owner onward, so a // 48-owner page crosses the 128-write submission boundary. let current = body.content_hash.as_str(); - let previous = bodies - .last() - .map_or(current, |body: &StoredCanonicalBody| body.content_hash.as_str()); + let previous = bodies.last().map_or(current, |body: &StoredCanonicalBody| { + body.content_hash.as_str() + }); let earlier = bodies .iter() .rev() @@ -289,7 +293,11 @@ async fn bootstrap_batch_refusal_rolls_back_and_recovers() { .await .unwrap(); let error = step(&transaction, 48).await.unwrap_err(); - assert!(error.to_string().contains("injected bootstrap batch refusal")); + assert!( + error + .to_string() + .contains("injected bootstrap batch refusal") + ); transaction.rollback().await.unwrap(); assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); let count = scalar(&fixture.conn, &format!("SELECT COUNT(*) FROM {table}")).await; @@ -384,7 +392,10 @@ async fn observation_bootstrap_batches_across_a_bounded_write_submission() { Some(format!("observations:{frontier}")) ); assert_eq!( - read_progress(&fixture.conn).await.unwrap().observations_through, + read_progress(&fixture.conn) + .await + .unwrap() + .observations_through, frontier ); assert_eq!(actual_submissions, submissions); @@ -393,9 +404,12 @@ async fn observation_bootstrap_batches_across_a_bounded_write_submission() { (1..=owners).map(|ordinal| ordinal * 2).collect::>() ); assert_eq!( - scalar(&fixture.conn, "SELECT COUNT(*) FROM canonical_body_gc_refs - WHERE owner_kind = 'observation'") - .await, + scalar( + &fixture.conn, + "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'" + ) + .await, refs, "duplicate references within an observation must stay deduplicated" ); @@ -411,7 +425,10 @@ async fn observation_bootstrap_batches_across_a_bounded_write_submission() { break; } } - assert!(finished, "bounded bootstrap and candidate discharge must terminate"); + assert!( + finished, + "bounded bootstrap and candidate discharge must terminate" + ); fixture.assert_bodies_preserved().await; let (idle, submissions) = fixture.tick().await; assert!(!idle.has_more); @@ -442,7 +459,11 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover .await .unwrap(); let error = step(&transaction, 48).await.unwrap_err(); - assert!(error.to_string().contains("injected observation batch refusal")); + assert!( + error + .to_string() + .contains("injected observation batch refusal") + ); transaction.rollback().await.unwrap(); assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); assert!(observation_owners(&fixture.conn).await.is_empty()); @@ -455,12 +476,18 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover .unwrap(); for (frontier, references, submissions) in [(96_i64, 141_i64, 3_usize), (104, 153, 2)] { let (report, actual_submissions) = fixture.tick().await; - assert_eq!(report.bootstrap_frontier_after, Some(format!("observations:{frontier}"))); + assert_eq!( + report.bootstrap_frontier_after, + Some(format!("observations:{frontier}")) + ); assert_eq!(actual_submissions, submissions); assert_eq!( - scalar(&fixture.conn, "SELECT COUNT(*) FROM canonical_body_gc_refs - WHERE owner_kind = 'observation'") - .await, + scalar( + &fixture.conn, + "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'" + ) + .await, references ); } @@ -542,8 +569,17 @@ async fn ignored_observation_reference_insert_refuses_checkpoint_then_recovers() .unwrap(); for (frontier, references) in [(96_i64, 141_i64), (104, 153)] { let (report, _) = fixture.tick().await; - assert_eq!(report.bootstrap_frontier_after, Some(format!("observations:{frontier}"))); - assert_eq!(read_progress(&fixture.conn).await.unwrap().observations_through, frontier); + assert_eq!( + report.bootstrap_frontier_after, + Some(format!("observations:{frontier}")) + ); + assert_eq!( + read_progress(&fixture.conn) + .await + .unwrap() + .observations_through, + frontier + ); assert_eq!( scalar( &fixture.conn, @@ -596,7 +632,11 @@ async fn ignored_candidate_insert_refuses_checkpoint_then_recovers() { let (report, _) = fixture.tick().await; assert_eq!(report.bodies_scanned, scanned); assert_eq!( - scalar(&fixture.conn, "SELECT COUNT(*) FROM canonical_body_gc_candidates").await, + scalar( + &fixture.conn, + "SELECT COUNT(*) FROM canonical_body_gc_candidates" + ) + .await, candidate_total ); } From 59b9a771fb05728aa5f61e5909338396769888a6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:31:42 -0800 Subject: [PATCH 30/50] perf(retention): avoid idle canonical GC writer admissions Read the durable bootstrap and candidate eligibility on the reader before acquiring a writer. An idle continuation does not submit a no-op write. Keep the schema check on the idle path; active work revalidates under its normal writer transaction. A registered-store regression tests exact writer admission counts, a fresh body, real reclamation and quiescence. Refs #4787 --- .../src/observation/canonical_body_gc.rs | 12 ++++++ .../observation/canonical_body_gc/tests.rs | 38 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index a95b950ddab..5ecc001555b 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -155,6 +155,18 @@ async fn save_progress(conn: &(impl Executor + ?Sized), progress: &Progress) -> #[tracing::instrument(name = "global_db.canonical_bodies.gc", level = "trace", skip_all)] pub(crate) async fn run(database: &Database, max_rows: usize) -> Result { let started = Instant::now(); + // During another retention continuation an already-drained store must not + // repeatedly acquire the SQLite writer merely to observe an empty queue. + // Do not bypass the schema authority: an idle report is safe only after + // verifying both the completed bootstrap and its installed objects. + let reader = database.read_connection(); + if read_progress(&reader).await?.complete() && !has_candidates(&reader).await? { + schema::validate(&reader).await?; + return Ok(CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + }); + } let transaction = database.begin_write_transaction(OPERATION).await?; match step(&transaction, max_rows).await { Ok(mut report) => { diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index ba18b93368b..b11bc9e8516 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -904,3 +904,41 @@ async fn reinserted_candidate_page_refuses_false_discharge_and_recovers() { assert_eq!(idle.bodies_scanned, 0); assert!(!idle.has_more); } + +#[asupersync::test] +async fn idle_registered_gc_avoids_writer_and_rechecks_new_candidates() { + use crate::tests::harness::{RegisteredGlobalDbHarness, writer_telemetry}; + + let harness = RegisteredGlobalDbHarness::open("canonical-body-idle-writer").await; + let before = writer_telemetry(&harness.registered).transactions; + for _ in 0..3 { + assert_eq!( + harness.registered.run_canonical_body_gc(48).await.unwrap(), + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); + assert_eq!( + writer_telemetry(&harness.registered).transactions, + before, + "a drained retention store must not enter the SQLite writer" + ); + } + + let unowned = StoredCanonicalBody::pack(b"arrived after an idle retention probe").unwrap(); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + insert_body(&transaction, &unowned).await; + transaction.commit().await.unwrap(); + let after_seed = writer_telemetry(&harness.registered).transactions; + let collected = harness.registered.run_canonical_body_gc(48).await.unwrap(); + assert_eq!(collected.bodies_scanned, 1); + assert_eq!(collected.bodies_deleted, 1); + assert_eq!(collected.candidates_discharged, 1); + assert!(!collected.has_more); + assert!( + writer_telemetry(&harness.registered).transactions.commands > after_seed.commands, + "a new candidate must still enter the real writer" + ); + assert!(!body_exists(&harness.registered.read_connection(), &unowned.content_hash).await); +} From 224f0f709bed078712c9622c4428fc455a756fbd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:18:19 -0800 Subject: [PATCH 31/50] style(gc): move idle regression imports to module scope Keep test harness imports at module scope per repository guidance. No assertions, behavior or limits change. Refs #4787 --- .../src/observation/canonical_body_gc/tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index b11bc9e8516..847f0fcf6de 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -6,6 +6,7 @@ use tracedecay_store::{ }; use super::{CanonicalBodyGcReport, has_candidates, read_progress, schema, step}; +use crate::tests::harness::{RegisteredGlobalDbHarness, writer_telemetry}; struct Store { runtime: TestConnection, @@ -907,8 +908,6 @@ async fn reinserted_candidate_page_refuses_false_discharge_and_recovers() { #[asupersync::test] async fn idle_registered_gc_avoids_writer_and_rechecks_new_candidates() { - use crate::tests::harness::{RegisteredGlobalDbHarness, writer_telemetry}; - let harness = RegisteredGlobalDbHarness::open("canonical-body-idle-writer").await; let before = writer_telemetry(&harness.registered).transactions; for _ in 0..3 { From d5669e7413985a75c87c2da42e464ac193be7192 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:17:33 -0800 Subject: [PATCH 32/50] fix(db): require batched execute_statements in adapters The production `begin_write_transaction` facade never overrode `Executor::execute_statements`, so the trait default dispatched one writer admission per statement. Canonical body GC observation bootstrap therefore paid a settlement per reference instead of one per bounded batch, and the write-count regression passed the concrete engine transaction directly, so it could not see the facade. Remove the trait default so an adapter that does not forward fails to compile, forward the batched submission from every facade and adapter (including the registered global-db writer and transaction wrappers), and run the bootstrap write-count regression through the production `begin_write_transaction` wrapper. --- .../src/git_index_transactions/database.rs | 9 + .../bootstrap_write_tests.rs | 280 +++++++++++------- .../src/observation/retention/tests.rs | 13 + crates/tracedecay-global-db/src/registered.rs | 31 ++ .../src/registered/workflow_schema_tests.rs | 14 + .../src/schema_contract/invariants/audit.rs | 7 + .../tracedecay-global-db/src/schema_stages.rs | 8 + .../src/session_handle.rs | 7 + .../tracedecay-global-db/src/tests/harness.rs | 13 + crates/tracedecay-lcm/src/gc/tests.rs | 8 + crates/tracedecay-lcm/src/retention.rs | 11 + .../src/summary_convergence_tests.rs | 11 + .../src/db/connection/facade.rs | 66 +++++ .../src/db/connection/query_write.rs | 21 ++ .../src/db/engine/executor.rs | 45 ++- .../src/db/engine/mod.rs | 2 +- .../src/db/engine/test_support.rs | 7 + .../src/db/engine/transaction.rs | 4 + .../src/db/engine/write_statement.rs | 1 + crates/tracedecay-runtime-core/src/ports.rs | 28 ++ .../src/sqlite_snapshot_connection.rs | 7 + .../src/test_support.rs | 7 + 22 files changed, 475 insertions(+), 125 deletions(-) diff --git a/crates/tracedecay-global-db/src/git_index_transactions/database.rs b/crates/tracedecay-global-db/src/git_index_transactions/database.rs index 51a3ce5909a..831a54e82c0 100644 --- a/crates/tracedecay-global-db/src/git_index_transactions/database.rs +++ b/crates/tracedecay-global-db/src/git_index_transactions/database.rs @@ -59,6 +59,15 @@ impl Executor for GitMutationWriteTransaction<'_> { } } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + match self { + Self::Registered(transaction) => transaction.execute_statements(statements).await, + } + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { match self { Self::Registered(transaction) => transaction.execute_batch(sql).await, diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs index 03ae1ab4bbe..211f6b9e44c 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs @@ -1,10 +1,19 @@ //! Measure real bootstrap write submissions without replacing the SQLite executor. +//! +//! Every page runs inside the production `begin_write_transaction` wrapper, so +//! the counts cover the facade the daemon actually uses. Batching inside that +//! facade is not caller-observable: a per-statement fallback expands behind the +//! adapter's own method. `Executor::execute_statements` therefore has no +//! default, and an adapter that does not forward to the engine's batched +//! submission fails to compile. use std::sync::atomic::{AtomicUsize, Ordering}; use tracedecay_runtime_core::db::engine::{ - Executor, IntoParams, QueryExecutor, Result as EngineResult, Rows, TestConnection, - TransactionBehavior, WriteStatement, params, + Executor, IntoParams, QueryExecutor, Result as EngineResult, Rows, WriteStatement, params, +}; +use tracedecay_runtime_core::db::{ + Database, DatabaseAuthority, DatabaseWriteTransaction, TestDatabaseRuntimeMode, }; use tracedecay_store::{ CANONICAL_BODIES_TABLE_SQL, StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL, @@ -47,8 +56,9 @@ impl Executor for CountWrites<'_, E> { } struct Fixture { - conn: TestConnection, + database: Database, bodies: Vec, + _authority: DatabaseAuthority, _directory: tempfile::TempDir, } @@ -58,25 +68,42 @@ impl Fixture { } async fn with_observations(include_observations: bool) -> Self { + // Publishing a profile-scoped fixture initializes the registered + // global/session schema through the fail-closed port. Registration is + // idempotent, so this fixture does not depend on another test having + // run first. + crate::register_registered_schema_installer(); let directory = tempfile::tempdir().unwrap(); - let conn = TestConnection::open(&directory.path().join("bootstrap-writes.db")); - conn.execute_batch(CANONICAL_BODIES_TABLE_SQL) - .await - .unwrap(); - conn.execute_batch( - "CREATE TABLE lcm_raw_messages ( - store_id INTEGER PRIMARY KEY AUTOINCREMENT, - storage_kind TEXT NOT NULL, content TEXT, content_hash TEXT NOT NULL - ); - CREATE TABLE observations ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - observation_json TEXT NOT NULL - );", + let db_path = directory.path().join("bootstrap-writes.db"); + let authority = + DatabaseAuthority::acquire_test(&db_path, "canonical body bootstrap write fixture") + .unwrap(); + let (database, _) = Database::publish_test_runtime( + &db_path, + &authority, + TestDatabaseRuntimeMode::Initialize, ) .await .unwrap(); - let transaction = conn - .transaction_with_behavior(TransactionBehavior::Immediate) + let transaction = database + .begin_write_transaction("seed canonical body bootstrap fixture") + .await + .unwrap(); + transaction + .execute_batch(CANONICAL_BODIES_TABLE_SQL) + .await + .unwrap(); + transaction + .execute_batch( + "CREATE TABLE lcm_raw_messages ( + store_id INTEGER PRIMARY KEY AUTOINCREMENT, + storage_kind TEXT NOT NULL, content TEXT, content_hash TEXT NOT NULL + ); + CREATE TABLE observations ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + observation_json TEXT NOT NULL + );", + ) .await .unwrap(); let mut bodies = Vec::new(); @@ -138,36 +165,49 @@ impl Fixture { schema::install(&transaction).await.unwrap(); transaction.commit().await.unwrap(); Self { - conn, + database, bodies, + _authority: authority, _directory: directory, } } - async fn tick(&self) -> (CanonicalBodyGcReport, usize) { - let transaction = self - .conn - .transaction_with_behavior(TransactionBehavior::Immediate) + async fn begin_write(&self) -> DatabaseWriteTransaction<'_> { + self.database + .begin_write_transaction("canonical body bootstrap fixture write") .await - .unwrap(); - let (report, submissions) = { - let counted = CountWrites { - inner: &transaction, - submissions: AtomicUsize::new(0), - }; - let report = step(&counted, 48).await.unwrap(); - (report, counted.submissions.load(Ordering::Relaxed)) + .unwrap() + } + + async fn execute_batch(&self, sql: &str) { + let transaction = self.begin_write().await; + transaction.execute_batch(sql).await.unwrap(); + transaction.commit().await.unwrap(); + } + + async fn tick(&self) -> (CanonicalBodyGcReport, usize) { + let transaction = self.begin_write().await; + let counted = CountWrites { + inner: &transaction, + submissions: AtomicUsize::new(0), }; + let report = step(&counted, 48).await.unwrap(); + let submissions = counted.submissions.load(Ordering::Relaxed); transaction.commit().await.unwrap(); (report, submissions) } async fn assert_bodies_preserved(&self) { - let count = scalar(&self.conn, "SELECT COUNT(*) FROM session_canonical_bodies").await; + let count = scalar( + &self.database.read_connection(), + "SELECT COUNT(*) FROM session_canonical_bodies", + ) + .await; assert_eq!(count, self.bodies.len() as i64); for body in &self.bodies { let mut rows = self - .conn + .database + .read_connection() .query( "SELECT body FROM session_canonical_bodies WHERE content_hash = ?1", params![body.content_hash.as_str()], @@ -213,12 +253,14 @@ async fn bootstrap_pages_batch_writes_and_preserve_sparse_frontiers() { let fixture = Fixture::open().await; for (scanned, through, owners) in [(48_u64, 96_i64, 48_i64), (4, 104, 52)] { let (report, submissions) = fixture.tick().await; - let progress = read_progress(&fixture.conn).await.unwrap(); + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); assert_eq!(report.roots_scanned, scanned); assert_eq!(submissions, 2, "one seed submission plus the checkpoint"); assert_eq!(progress.raw_through, through); assert_eq!( - raw_owners(&fixture.conn).await, + raw_owners(&fixture.database.read_connection()).await, (1..=owners).map(|ordinal| ordinal * 2).collect::>() ); assert_eq!(report.bodies_deleted, 0); @@ -233,9 +275,11 @@ async fn bootstrap_pages_batch_writes_and_preserve_sparse_frontiers() { hashes.sort(); for (scanned, total) in [(48_u64, 48_usize), (4, 52)] { let (report, submissions) = fixture.tick().await; - let progress = read_progress(&fixture.conn).await.unwrap(); + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); let candidates = scalar( - &fixture.conn, + &fixture.database.read_connection(), "SELECT COUNT(*) FROM canonical_body_gc_candidates", ) .await; @@ -269,7 +313,9 @@ async fn bootstrap_batch_refusal_rolls_back_and_recovers() { fixture.tick().await; fixture.tick().await; } - let before = read_progress(&fixture.conn).await.unwrap(); + let before = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); let table = if bodies_phase { "canonical_body_gc_candidates" } else { @@ -278,20 +324,14 @@ async fn bootstrap_batch_refusal_rolls_back_and_recovers() { // FAIL deliberately leaves the first 47 inserts in the transaction. // The collector's rollback must discard them with the uncommitted page. fixture - .conn .execute_batch(&format!( "CREATE TRIGGER test_bootstrap_refusal BEFORE INSERT ON {table} WHEN (SELECT COUNT(*) FROM {table}) = 47 BEGIN SELECT RAISE(FAIL, 'injected bootstrap batch refusal'); END" )) - .await - .unwrap(); - let transaction = fixture - .conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .unwrap(); + .await; + let transaction = fixture.begin_write().await; let error = step(&transaction, 48).await.unwrap_err(); assert!( error @@ -299,15 +339,22 @@ async fn bootstrap_batch_refusal_rolls_back_and_recovers() { .contains("injected bootstrap batch refusal") ); transaction.rollback().await.unwrap(); - assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); - let count = scalar(&fixture.conn, &format!("SELECT COUNT(*) FROM {table}")).await; + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap(), + before + ); + let count = scalar( + &fixture.database.read_connection(), + &format!("SELECT COUNT(*) FROM {table}"), + ) + .await; assert_eq!(count, 0); fixture.assert_bodies_preserved().await; fixture - .conn .execute_batch("DROP TRIGGER test_bootstrap_refusal") - .await - .unwrap(); + .await; for scanned in [48_u64, 4] { let (report, submissions) = fixture.tick().await; let actual = if bodies_phase { @@ -319,7 +366,11 @@ async fn bootstrap_batch_refusal_rolls_back_and_recovers() { assert_eq!(actual, scanned); assert_eq!(report.bodies_deleted, 0); } - let count = scalar(&fixture.conn, &format!("SELECT COUNT(*) FROM {table}")).await; + let count = scalar( + &fixture.database.read_connection(), + &format!("SELECT COUNT(*) FROM {table}"), + ) + .await; assert_eq!(count, 52); fixture.assert_bodies_preserved().await; } @@ -329,25 +380,27 @@ async fn bootstrap_batch_refusal_rolls_back_and_recovers() { async fn bootstrap_batch_eligibility_does_not_change_the_page_boundary() { let fixture = Fixture::open().await; fixture - .conn .execute_batch( "UPDATE lcm_raw_messages SET storage_kind = 'external' WHERE store_id = 94; UPDATE lcm_raw_messages SET content = 'owned inline text' WHERE store_id = 96;", ) + .await; + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.database.read_connection()) .await .unwrap(); - let (report, submissions) = fixture.tick().await; - let progress = read_progress(&fixture.conn).await.unwrap(); assert_eq!(report.roots_scanned, 48); assert_eq!(submissions, 2); assert_eq!(progress.raw_through, 96); assert_eq!( - raw_owners(&fixture.conn).await, + raw_owners(&fixture.database.read_connection()).await, (1..=46_i64).map(|ordinal| ordinal * 2).collect::>() ); assert!(report.has_more); let (tail, submissions) = fixture.tick().await; - let progress = read_progress(&fixture.conn).await.unwrap(); + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); assert_eq!(tail.roots_scanned, 4); assert_eq!(submissions, 2); assert_eq!(progress.raw_through, 104); @@ -355,7 +408,10 @@ async fn bootstrap_batch_eligibility_does_not_change_the_page_boundary() { .filter(|ordinal| ![47, 48].contains(ordinal)) .map(|ordinal| ordinal * 2) .collect::>(); - assert_eq!(raw_owners(&fixture.conn).await, expected); + assert_eq!( + raw_owners(&fixture.database.read_connection()).await, + expected + ); fixture.assert_bodies_preserved().await; } @@ -392,7 +448,7 @@ async fn observation_bootstrap_batches_across_a_bounded_write_submission() { Some(format!("observations:{frontier}")) ); assert_eq!( - read_progress(&fixture.conn) + read_progress(&fixture.database.read_connection()) .await .unwrap() .observations_through, @@ -400,12 +456,12 @@ async fn observation_bootstrap_batches_across_a_bounded_write_submission() { ); assert_eq!(actual_submissions, submissions); assert_eq!( - observation_owners(&fixture.conn).await, + observation_owners(&fixture.database.read_connection()).await, (1..=owners).map(|ordinal| ordinal * 2).collect::>() ); assert_eq!( scalar( - &fixture.conn, + &fixture.database.read_connection(), "SELECT COUNT(*) FROM canonical_body_gc_refs WHERE owner_kind = 'observation'" ) @@ -442,22 +498,18 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover for _ in 0..2 { fixture.tick().await; } - let before = read_progress(&fixture.conn).await.unwrap(); + let before = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); fixture - .conn .execute_batch( "CREATE TRIGGER refuse_observation_seed BEFORE INSERT ON canonical_body_gc_refs WHEN NEW.owner_kind = 'observation' AND NEW.owner_id = 94 BEGIN SELECT RAISE(FAIL, 'injected observation batch refusal'); END", ) - .await - .unwrap(); - let transaction = fixture - .conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .unwrap(); + .await; + let transaction = fixture.begin_write().await; let error = step(&transaction, 48).await.unwrap_err(); assert!( error @@ -465,15 +517,22 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover .contains("injected observation batch refusal") ); transaction.rollback().await.unwrap(); - assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); - assert!(observation_owners(&fixture.conn).await.is_empty()); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap(), + before + ); + assert!( + observation_owners(&fixture.database.read_connection()) + .await + .is_empty() + ); fixture.assert_bodies_preserved().await; fixture - .conn .execute_batch("DROP TRIGGER refuse_observation_seed") - .await - .unwrap(); + .await; for (frontier, references, submissions) in [(96_i64, 141_i64, 3_usize), (104, 153, 2)] { let (report, actual_submissions) = fixture.tick().await; assert_eq!( @@ -483,7 +542,7 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover assert_eq!(actual_submissions, submissions); assert_eq!( scalar( - &fixture.conn, + &fixture.database.read_connection(), "SELECT COUNT(*) FROM canonical_body_gc_refs WHERE owner_kind = 'observation'" ) @@ -495,19 +554,22 @@ async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recover } async fn assert_refused_bootstrap_rolls_back(fixture: &Fixture, diagnosis: &str) { - let before = read_progress(&fixture.conn).await.unwrap(); - let transaction = fixture - .conn - .transaction_with_behavior(TransactionBehavior::Immediate) + let before = read_progress(&fixture.database.read_connection()) .await .unwrap(); + let transaction = fixture.begin_write().await; let error = step(&transaction, 48).await.unwrap_err(); assert!( error.to_string().contains(diagnosis), "unexpected bootstrap refusal: {error}" ); transaction.rollback().await.unwrap(); - assert_eq!(read_progress(&fixture.conn).await.unwrap(), before); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap(), + before + ); fixture.assert_bodies_preserved().await; } @@ -515,30 +577,34 @@ async fn assert_refused_bootstrap_rolls_back(fixture: &Fixture, diagnosis: &str) async fn ignored_raw_reference_insert_refuses_checkpoint_then_recovers() { let fixture = Fixture::open().await; fixture - .conn .execute_batch( "CREATE TRIGGER ignore_raw_ref BEFORE INSERT ON canonical_body_gc_refs WHEN NEW.owner_kind = 'raw' AND NEW.owner_id = 94 BEGIN SELECT RAISE(IGNORE); END", ) - .await - .unwrap(); + .await; assert_refused_bootstrap_rolls_back(&fixture, "raw bootstrap reference was not indexed").await; - assert!(raw_owners(&fixture.conn).await.is_empty()); - fixture - .conn - .execute_batch("DROP TRIGGER ignore_raw_ref") - .await - .unwrap(); + assert!( + raw_owners(&fixture.database.read_connection()) + .await + .is_empty() + ); + fixture.execute_batch("DROP TRIGGER ignore_raw_ref").await; for (scanned, frontier) in [(48_u64, 96_i64), (4, 104)] { let (report, _) = fixture.tick().await; assert_eq!(report.roots_scanned, scanned); assert_eq!( - read_progress(&fixture.conn).await.unwrap().raw_through, + read_progress(&fixture.database.read_connection()) + .await + .unwrap() + .raw_through, frontier ); } - assert_eq!(raw_owners(&fixture.conn).await.len(), 52); + assert_eq!( + raw_owners(&fixture.database.read_connection()).await.len(), + 52 + ); fixture.assert_bodies_preserved().await; } @@ -548,25 +614,25 @@ async fn ignored_observation_reference_insert_refuses_checkpoint_then_recovers() fixture.tick().await; fixture.tick().await; fixture - .conn .execute_batch( "CREATE TRIGGER ignore_observation_ref BEFORE INSERT ON canonical_body_gc_refs WHEN NEW.owner_kind = 'observation' AND NEW.owner_id = 94 BEGIN SELECT RAISE(IGNORE); END", ) - .await - .unwrap(); + .await; assert_refused_bootstrap_rolls_back( &fixture, "observation bootstrap reference was not indexed", ) .await; - assert!(observation_owners(&fixture.conn).await.is_empty()); + assert!( + observation_owners(&fixture.database.read_connection()) + .await + .is_empty() + ); fixture - .conn .execute_batch("DROP TRIGGER ignore_observation_ref") - .await - .unwrap(); + .await; for (frontier, references) in [(96_i64, 141_i64), (104, 153)] { let (report, _) = fixture.tick().await; assert_eq!( @@ -574,7 +640,7 @@ async fn ignored_observation_reference_insert_refuses_checkpoint_then_recovers() Some(format!("observations:{frontier}")) ); assert_eq!( - read_progress(&fixture.conn) + read_progress(&fixture.database.read_connection()) .await .unwrap() .observations_through, @@ -582,7 +648,7 @@ async fn ignored_observation_reference_insert_refuses_checkpoint_then_recovers() ); assert_eq!( scalar( - &fixture.conn, + &fixture.database.read_connection(), "SELECT COUNT(*) FROM canonical_body_gc_refs WHERE owner_kind = 'observation'", ) @@ -606,34 +672,30 @@ async fn ignored_candidate_insert_refuses_checkpoint_then_recovers() { hashes.sort_unstable(); let skipped = hashes[47]; fixture - .conn .execute_batch(&format!( "CREATE TRIGGER ignore_candidate_seed BEFORE INSERT ON canonical_body_gc_candidates WHEN NEW.content_hash = '{skipped}' BEGIN SELECT RAISE(IGNORE); END" )) - .await - .unwrap(); + .await; assert_refused_bootstrap_rolls_back(&fixture, "body bootstrap candidate was not indexed").await; assert_eq!( scalar( - &fixture.conn, + &fixture.database.read_connection(), "SELECT COUNT(*) FROM canonical_body_gc_candidates", ) .await, 0 ); fixture - .conn .execute_batch("DROP TRIGGER ignore_candidate_seed") - .await - .unwrap(); + .await; for (scanned, candidate_total) in [(48_u64, 48_i64), (4, 52)] { let (report, _) = fixture.tick().await; assert_eq!(report.bodies_scanned, scanned); assert_eq!( scalar( - &fixture.conn, + &fixture.database.read_connection(), "SELECT COUNT(*) FROM canonical_body_gc_candidates" ) .await, diff --git a/crates/tracedecay-global-db/src/observation/retention/tests.rs b/crates/tracedecay-global-db/src/observation/retention/tests.rs index c3afed8fdc0..0343f292ae1 100644 --- a/crates/tracedecay-global-db/src/observation/retention/tests.rs +++ b/crates/tracedecay-global-db/src/observation/retention/tests.rs @@ -72,6 +72,19 @@ impl Executor for RetentionTestStore { .await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.database() + .writer_connection() + .map_err(|error| { + tracedecay_runtime_core::db::engine::Error::invalid_operation(error.to_string()) + })? + .execute_statements(statements) + .await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { self.database() .writer_connection() diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index 4402e809fbf..0e7f65e63ba 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -1020,6 +1020,16 @@ impl RegisteredGlobalDbWriterConnection<'_> { .await .map_err(engine_error) } + + pub async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.database + .execute_write_statements("execute registered global database statements", statements) + .await + .map_err(engine_error) + } } pub struct RegisteredGlobalDbWriteTransaction<'a> { @@ -1052,6 +1062,13 @@ impl Executor for RegisteredGlobalDbWriteTransaction<'_> { RegisteredGlobalDbWriteTransaction::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + RegisteredGlobalDbWriteTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { RegisteredGlobalDbWriteTransaction::execute_batch(self, sql).await } @@ -1108,6 +1125,13 @@ impl RegisteredGlobalDbWriteTransaction<'_> { self.transaction.execute_batch(sql).await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.transaction.execute_statements(statements).await + } + #[tracing::instrument(name = "global_db.registered.txn.commit", level = "trace", skip_all)] pub async fn commit(self) -> tracedecay_runtime_core::db::engine::Result<()> { self.commit_with_on_commit(|| {}).await @@ -1185,6 +1209,13 @@ impl Executor for RegisteredGlobalDbWriteTx<'_, '_> { self.transaction.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.transaction.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { self.transaction.execute_batch(sql).await } diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 3dd339b74f0..1e6ccd63a57 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -1209,6 +1209,13 @@ mod published_artifact_ceiling { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> engine::Result> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> engine::Result<()> { self.note(sql); self.inner.execute_batch(sql).await @@ -1281,6 +1288,13 @@ mod published_artifact_ceiling { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> engine::Result> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> engine::Result<()> { self.check(sql)?; self.inner.execute_batch(sql).await diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 7a90f3484e5..77cd5792ba0 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -1715,6 +1715,13 @@ mod tests { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { self.inner.execute_batch(sql).await } diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 63e7e42226c..bfce9fb699b 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -1013,6 +1013,14 @@ impl Executor for CancellableSchemaTransaction<'_, T> { self.transaction.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> engine::Result> { + self.checkpoint()?; + self.transaction.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> engine::Result<()> { self.checkpoint()?; self.transaction.execute_batch(sql).await diff --git a/crates/tracedecay-global-db/src/session_handle.rs b/crates/tracedecay-global-db/src/session_handle.rs index ec12592de02..4e56ca3290f 100644 --- a/crates/tracedecay-global-db/src/session_handle.rs +++ b/crates/tracedecay-global-db/src/session_handle.rs @@ -125,6 +125,13 @@ impl Executor for RegisteredGlobalDbWriterConnection<'_> { RegisteredGlobalDbWriterConnection::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + RegisteredGlobalDbWriterConnection::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { RegisteredGlobalDbWriterConnection::execute_batch(self, sql).await } diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index bb80c1884b8..7de179fbb98 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -1606,6 +1606,19 @@ impl Executor for RegisteredGlobalDbTestFixture { .await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.database + .writer_connection() + .map_err(|error| { + tracedecay_runtime_core::db::engine::Error::invalid_operation(error.to_string()) + })? + .execute_statements(statements) + .await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { self.database .writer_connection() diff --git a/crates/tracedecay-lcm/src/gc/tests.rs b/crates/tracedecay-lcm/src/gc/tests.rs index 1d4eca91af7..5f176a4e399 100644 --- a/crates/tracedecay-lcm/src/gc/tests.rs +++ b/crates/tracedecay-lcm/src/gc/tests.rs @@ -1707,6 +1707,14 @@ impl Executor for CountingExecutor<'_, E> { self.counter.tick(); self.inner.execute_batch(sql).await } + + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.counter.tick(); + self.inner.execute_statements(statements).await + } } fn batch_ref(index: usize) -> String { diff --git a/crates/tracedecay-lcm/src/retention.rs b/crates/tracedecay-lcm/src/retention.rs index fc7bfcd9355..712fd72585d 100644 --- a/crates/tracedecay-lcm/src/retention.rs +++ b/crates/tracedecay-lcm/src/retention.rs @@ -660,6 +660,17 @@ impl Executor for RetentionWriteTransaction<'_> { Self::Connection(transaction) => transaction.execute_batch(sql).await, } } + + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + match self { + Self::Database(transaction) => transaction.execute_statements(statements).await, + #[cfg(test)] + Self::Connection(transaction) => transaction.execute_statements(statements).await, + } + } } #[derive(Clone, Copy)] diff --git a/crates/tracedecay-lcm/src/summary_convergence_tests.rs b/crates/tracedecay-lcm/src/summary_convergence_tests.rs index d0c6bc90a48..dda1cfbe9d0 100644 --- a/crates/tracedecay-lcm/src/summary_convergence_tests.rs +++ b/crates/tracedecay-lcm/src/summary_convergence_tests.rs @@ -52,6 +52,17 @@ impl Executor for CountingExecutor<'_> { self.executes.set(self.executes.get() + 1); self.inner.execute_batch(sql).await } + + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.executes.set(self.executes.get() + statements.len()); + let results = self.inner.execute_statements(statements).await?; + let changed = results.iter().copied().sum::(); + self.rows_changed.set(self.rows_changed.get() + changed); + Ok(results) + } } async fn fetch_i64(conn: &TestConnection, sql: &str, params: impl IntoParams) -> i64 { diff --git a/crates/tracedecay-runtime-core/src/db/connection/facade.rs b/crates/tracedecay-runtime-core/src/db/connection/facade.rs index 769324b242c..d8448e7dd32 100644 --- a/crates/tracedecay-runtime-core/src/db/connection/facade.rs +++ b/crates/tracedecay-runtime-core/src/db/connection/facade.rs @@ -80,6 +80,13 @@ impl DatabaseEngineWriteConnection { self.conn.execute_batch(sql).await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.conn.execute_statements(statements).await + } + pub(crate) async fn authorized_long_lease_transaction( &self, ) -> crate::db::engine::Result { @@ -114,6 +121,13 @@ impl crate::db::engine::Executor for DatabaseEngineWriteConnection { DatabaseEngineWriteConnection::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseEngineWriteConnection::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { DatabaseEngineWriteConnection::execute_batch(self, sql).await } @@ -257,6 +271,13 @@ impl DatabaseEngineLongLeaseTransaction { pub(crate) async fn rollback(self) -> crate::db::engine::Result<()> { self.transaction.rollback().await } + + pub(crate) async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.transaction.execute_statements(statements).await + } } impl crate::db::engine::QueryExecutor for DatabaseEngineLongLeaseTransaction { @@ -280,6 +301,13 @@ impl crate::db::engine::Executor for DatabaseEngineLongLeaseTransaction { self.transaction.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseEngineLongLeaseTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { self.transaction.execute_batch(sql).await } @@ -329,6 +357,18 @@ impl<'a> DatabaseMemoryTransaction<'a> { } } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + match self { + Self::Read(_) => Err(crate::db::engine::Error::Runtime( + "cannot execute a write in a memory read snapshot".to_owned(), + )), + Self::Write(transaction) => transaction.execute_statements(statements).await, + } + } + pub async fn commit_with_on_commit(self, on_commit: F) -> Result<()> where F: FnOnce() + Send + 'static, @@ -393,6 +433,13 @@ impl crate::db::engine::Executor for DatabaseMemoryTransaction<'_> { DatabaseMemoryTransaction::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseMemoryTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { DatabaseMemoryTransaction::execute_batch(self, sql).await } @@ -421,6 +468,18 @@ impl DatabaseWriteTransaction<'_> { self.transaction.execute_batch(sql).await } + /// Submits owned writes in one writer admission. + /// + /// The wrapper must forward to the engine transaction's batched + /// submission; a per-statement loop here would turn a bounded page into + /// one writer admission per statement. + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.transaction.execute_statements(statements).await + } + pub async fn execute_batch_engine(&self, sql: &str) -> crate::db::engine::Result<()> { self.transaction.execute_batch(sql).await } @@ -521,6 +580,13 @@ impl crate::db::engine::Executor for DatabaseWriteTransaction<'_> { self.execute_engine(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseWriteTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { self.execute_batch_engine(sql).await } diff --git a/crates/tracedecay-runtime-core/src/db/connection/query_write.rs b/crates/tracedecay-runtime-core/src/db/connection/query_write.rs index ef5e4a038b4..3e0edda0a2f 100644 --- a/crates/tracedecay-runtime-core/src/db/connection/query_write.rs +++ b/crates/tracedecay-runtime-core/src/db/connection/query_write.rs @@ -114,6 +114,27 @@ impl Database { .await? } + /// Executes owned parameterized writes in one brokered writer job. + #[doc(hidden)] + pub async fn execute_write_statements( + &self, + operation: &str, + statements: Vec, + ) -> Result> { + let operation_name = operation.to_owned(); + self.transact(operation, async move |tx| { + let operation_name = operation_name.clone(); + let statements = statements.clone(); + tx.execute_statements(statements) + .await + .map_err(|error| TraceDecayError::Database { + message: format!("failed to execute brokered write statements: {error}"), + operation: operation_name, + }) + }) + .await? + } + /// Executes one authority-revalidated schema or bulk-maintenance batch /// through the canonical long-lease writer path. /// diff --git a/crates/tracedecay-runtime-core/src/db/engine/executor.rs b/crates/tracedecay-runtime-core/src/db/engine/executor.rs index 2740ff8ada9..2327e007d4a 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/executor.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/executor.rs @@ -53,21 +53,11 @@ pub trait Executor: QueryExecutor { P: IntoParams; /// Executes owned parameterized writes in input order. /// - /// Concrete engine connections and transactions override this with one - /// blocking submission. The default preserves compatibility for narrow - /// executor adapters while retaining exact failed-statement attribution. - async fn execute_statements(&self, statements: Vec) -> Result> { - let mut results = Vec::with_capacity(statements.len()); - for (index, statement) in statements.into_iter().enumerate() { - let (sql, params) = statement.into_parts(); - results.push( - self.execute(&sql, params) - .await - .map_err(|error| Error::statement_batch(index, error))?, - ); - } - Ok(results) - } + /// There is deliberately no default: an adapter over a batching executor + /// must forward to it, and a silent per-statement fallback would turn a + /// bounded page into one writer admission per statement. Implementations + /// whose backend cannot batch call [`execute_statements_serially`]. + async fn execute_statements(&self, statements: Vec) -> Result>; async fn execute_batch(&self, sql: &str) -> Result<()>; } @@ -105,3 +95,28 @@ impl Executor for Transaction { Transaction::execute_batch(self, sql).await } } + +/// Executes `statements` in input order with one statement per call. +/// +/// Only for executors whose backend has no batched submission. An executor +/// that wraps a batching implementation must forward to it instead, because +/// this helper admits the writer once per statement. +pub async fn execute_statements_serially( + executor: &E, + statements: Vec, +) -> Result> +where + E: Executor + ?Sized, +{ + let mut results = Vec::with_capacity(statements.len()); + for (index, statement) in statements.into_iter().enumerate() { + let (sql, params) = statement.into_parts(); + results.push( + executor + .execute(&sql, params) + .await + .map_err(|error| Error::statement_batch(index, error))?, + ); + } + Ok(results) +} diff --git a/crates/tracedecay-runtime-core/src/db/engine/mod.rs b/crates/tracedecay-runtime-core/src/db/engine/mod.rs index f464ee92fcc..786518e36a9 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/mod.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/mod.rs @@ -14,7 +14,7 @@ mod write_statement; pub use connection::{Connection, ReadConnection, ReaderPoolSnapshot, ReaderPoolState}; pub use error::{Error, Result}; -pub use executor::{Executor, QueryExecutor}; +pub use executor::{Executor, QueryExecutor, execute_statements_serially}; pub use params::{IntoParams, IntoValue, Params, params, params_from_iter}; pub use row::{Row, Rows}; pub use snapshot::ReadSnapshot; diff --git a/crates/tracedecay-runtime-core/src/db/engine/test_support.rs b/crates/tracedecay-runtime-core/src/db/engine/test_support.rs index 550b2a00086..6d7826e4d6a 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/test_support.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/test_support.rs @@ -134,6 +134,13 @@ impl Executor for TestConnection { Connection::execute(&self.connection, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + Connection::execute_statements(&self.connection, statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { Connection::execute_batch(&self.connection, sql).await } diff --git a/crates/tracedecay-runtime-core/src/db/engine/transaction.rs b/crates/tracedecay-runtime-core/src/db/engine/transaction.rs index eda1332f82f..588f5563e3d 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/transaction.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/transaction.rs @@ -229,6 +229,10 @@ impl Executor for WriteTx<'_, '_> { Ok(self.job.execute(statement(sql, params)?)?.changed_rows as u64) } + async fn execute_statements(&self, statements: Vec) -> Result> { + super::execute_statements_serially(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> Result<()> { self.job.execute_batch(sql.to_owned())?; Ok(()) diff --git a/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs b/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs index fb52ee4576d..6a79649eebd 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs @@ -6,6 +6,7 @@ use super::{IntoParams, Result, Value}; /// /// Construction performs the same request validation as scalar execution, so /// a malformed statement is attributed before any batch reaches the writer. +#[derive(Clone)] pub struct WriteStatement { exact: ExactSqlStatement, } diff --git a/crates/tracedecay-runtime-core/src/ports.rs b/crates/tracedecay-runtime-core/src/ports.rs index 1d1d3bfa0bc..3634fc951d3 100644 --- a/crates/tracedecay-runtime-core/src/ports.rs +++ b/crates/tracedecay-runtime-core/src/ports.rs @@ -96,6 +96,13 @@ pub mod registered_schema { self.connection.execute_batch(sql).await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.connection.execute_statements(statements).await + } + /// Begins the authority-bound transaction used for one atomic schema /// installation. /// @@ -154,6 +161,13 @@ pub mod registered_schema { RegisteredSchemaInstallationV1::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + RegisteredSchemaInstallationV1::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { RegisteredSchemaInstallationV1::execute_batch(self, sql).await } @@ -184,6 +198,13 @@ pub mod registered_schema { .await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.transaction.execute_statements(statements).await + } + pub async fn commit(self) -> crate::db::engine::Result<()> { self.transaction.commit().await } @@ -214,6 +235,13 @@ pub mod registered_schema { RegisteredSchemaInstallationTransactionV1::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + RegisteredSchemaInstallationTransactionV1::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { RegisteredSchemaInstallationTransactionV1::execute_batch(self, sql).await } diff --git a/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs b/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs index c6d29dcb784..028163f1661 100644 --- a/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs +++ b/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs @@ -115,6 +115,13 @@ impl Executor for SnapshotConnection { .map_err(|error| EngineError::Runtime(format!("snapshot execute task failed: {error}")))? } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + crate::db::engine::execute_statements_serially(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { let cx = crate::native_runtime::cx().ok_or(EngineError::RuntimeContextUnavailable)?; let connection = Arc::clone(&self.connection); diff --git a/crates/tracedecay-session-temporal-store/src/test_support.rs b/crates/tracedecay-session-temporal-store/src/test_support.rs index 17aac90ce64..412099bb829 100644 --- a/crates/tracedecay-session-temporal-store/src/test_support.rs +++ b/crates/tracedecay-session-temporal-store/src/test_support.rs @@ -48,6 +48,13 @@ impl Executor for QueryCountingConnection<'_, T> { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { self.inner.execute_batch(sql).await } From 0ef208d1b0afc2e83769c0380a2264342142691e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:25:37 -0800 Subject: [PATCH 33/50] test(native): use daemon scheduling in integration journeys Native work inherits the ambient executor, so these retained-owner journeys must use the canonical daemon runtime policy. Preserve default worker sizing, the existing readiness deadline, and every behavioral assertion while keeping peer wakes stealable during a long owner poll. --- .../src/native_integration/journey_tests.rs | 58 +++++++++++++++---- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs index 1804333d454..b2cc1e14f86 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs @@ -69,6 +69,16 @@ use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; const OBSERVED_AT: UtcMicros = UtcMicros(100); const EXPIRES_AT: UtcMicros = UtcMicros(10_000); +fn run_native_journey(journey: impl Future) { + // Native owners reuse the ambient runtime. Match the daemon's scheduling + // policy so a wake issued during a long owner poll stays stealable by + // another worker instead of being held in Tokio's default LIFO slot. + native_runtime::default_builder() + .build() + .expect("native journey runtime") + .block_on(journey); +} + struct UnexpectedAnalysis; impl NativeIntegrationAnalysisPort for UnexpectedAnalysis { @@ -760,8 +770,12 @@ async fn stack_snapshot( selection.as_ref().clone() } -#[tokio::test(flavor = "multi_thread")] -async fn declared_stack_runtime_shares_native_preview_and_failure_without_consuming_result() { +#[test] +fn declared_stack_runtime_shares_native_preview_and_failure_without_consuming_result() { + run_native_journey(shared_native_preview_and_failure_journey()); +} + +async fn shared_native_preview_and_failure_journey() { for fail in [false, true] { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); @@ -928,8 +942,12 @@ async fn declared_stack_runtime_shares_native_preview_and_failure_without_consum } } -#[tokio::test(flavor = "multi_thread")] -async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { +#[test] +fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { + run_native_journey(checked_out_stack_conflict_journey()); +} + +async fn checked_out_stack_conflict_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); let source_root = directory.path().join("dependency"); @@ -1392,8 +1410,12 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { code_index.shutdown().await; } -#[tokio::test(flavor = "multi_thread")] -async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() { +#[test] +fn independent_pair_applies_supported_modes_and_survives_daemon_restart() { + run_native_journey(independent_pair_restart_journey()); +} + +async fn independent_pair_restart_journey() { for (index, mode) in [ MechanicalIntegrationModeV1::FastForward, MechanicalIntegrationModeV1::TwoParentMerge, @@ -1773,8 +1795,12 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() } } -#[tokio::test(flavor = "multi_thread")] -async fn generated_only_change_requires_semantic_review() { +#[test] +fn generated_only_change_requires_semantic_review() { + run_native_journey(generated_only_change_journey()); +} + +async fn generated_only_change_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); std::fs::create_dir_all(&repository_root).expect("repository root"); @@ -1818,8 +1844,12 @@ async fn generated_only_change_requires_semantic_review() { code_index.shutdown().await; } -#[tokio::test(flavor = "multi_thread")] -async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_tip() { +#[test] +fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_tip() { + run_native_journey(foreign_destination_ref_drift_journey()); +} + +async fn foreign_destination_ref_drift_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); std::fs::create_dir_all(&repository_root).expect("repository root"); @@ -1920,8 +1950,12 @@ async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_t /// Cleanup inspection of an owned checkout, asked while another thread walks /// that checkout, classifies it from the shared walk. Reading the walk as /// "no repository" classified the checkout as foreign. -#[tokio::test(flavor = "multi_thread")] -async fn owned_worktree_inspection_shares_the_discovery_walk_another_thread_owns() { +#[test] +fn owned_worktree_inspection_shares_the_discovery_walk_another_thread_owns() { + run_native_journey(shared_worktree_discovery_journey()); +} + +async fn shared_worktree_discovery_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); std::fs::create_dir_all(&repository_root).expect("repository root"); From aa5da6857918ac9f1a591ad62d12f836bf16b3d0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:27:47 -0800 Subject: [PATCH 34/50] test(lsp): capture receipt admission phases on ACK failure --- .../src/invocation/lsp_delivery.rs | 20 ++++++++++++++++++- .../runtime_surface_acceptance.rs | 13 ++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs index 83be58f6b98..fc4e5bfd6b8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs +++ b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs @@ -114,10 +114,28 @@ impl LspDeliverySettlementClaim { self, project_lease: ProjectRuntimeRequestLeaseV1, ) -> impl Future + Send { + let owner_event_id = self.settlement.attempt.owner_event_id.clone(); + tracing::trace!( + %owner_event_id, + phase = "submit", + "LSP delivery receipt admission" + ); let admission = tracedecay_runtime_core::native_runtime::on_blocking_pool( async move { let _project_lease = project_lease; - self.admit_inline() + tracing::trace!( + %owner_event_id, + phase = "started", + "LSP delivery receipt admission" + ); + let outcome = self.admit_inline(); + tracing::trace!( + %owner_event_id, + phase = "completed", + ?outcome, + "LSP delivery receipt admission" + ); + outcome } .with_current_subscriber(), ); diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 8d0be3c8460..3737ab21b4f 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -136,9 +136,13 @@ impl RuntimeFixture { } } - fn daemon_log_tail(&self) -> String { + fn daemon_log(&self) -> String { std::fs::read_to_string(self._environment.scratch().join("runtime-daemon.log")) .expect("read isolated runtime daemon log") + } + + fn daemon_log_tail(&self) -> String { + self.daemon_log() .lines() .rev() .take(160) @@ -224,7 +228,8 @@ async fn lsp_runtime_fixture() -> RuntimeFixture { command .env( "RUST_LOG", - "info,tracedecay_daemon_service::invocation=trace", + "info,tracedecay_daemon_service::invocation=trace,\ + tracedecay_private_fs::framed_log=trace", ) .env("TRACEDECAY_SPAN_TIMINGS", "1") .stderr(daemon_log); @@ -439,10 +444,10 @@ async fn poll_lsp_response( .unwrap_or_else(|error| { panic!( "acknowledge daemon LSP frame while awaiting response {response_id}: \ - {error:?}; frame id={:?}, method={:?}; daemon log tail:\n{}", + {error:?}; frame id={:?}, method={:?}; daemon log:\n{}", value.get("id"), value.get("method"), - fixture.daemon_log_tail(), + fixture.daemon_log(), ) }); if value.get("id").and_then(Value::as_u64) == Some(response_id) { From b0680c2a02b78ab469d26c16c46325a6138ff211 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:59:27 -0800 Subject: [PATCH 35/50] test(fixtures): align schema cancellation and retain hook diagnostics Run the schema cancellation sweep on the canonical owned native runtime and reuse the published baseline across its existing cancellation samples. Preserve exact schema identity, retirement joins and every reopen check. Carry the reviewed PostCompact fixture logging and explicit daemon guard so a temporal publication failure retains its existing phase evidence. The hook assertions and deadlines stay unchanged. --- .../src/tests/schema_install_cancellation.rs | 126 ++++++++++-------- .../codex_compaction.rs | 28 ++-- 2 files changed, 86 insertions(+), 68 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 64b90ebeb1e..b7e3aabe99a 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -96,7 +96,7 @@ impl RegisteredStore { /// Inspect a completed family with an ordinary independent read-only SQLite /// connection. Reopening the registered runtime for every observation would /// perform another full attachment lifecycle and obscure cancellation timing. - fn external_schema_objects(&self) -> Vec { + fn external_schema_objects(&self) -> Vec<(String, Option)> { let connection = rusqlite::Connection::open_with_flags( &self.path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, @@ -104,12 +104,12 @@ impl RegisteredStore { .unwrap(); let mut statement = connection .prepare( - "SELECT type || ':' || name FROM sqlite_schema + "SELECT type || ':' || name, sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", ) .unwrap(); statement - .query_map([], |row| row.get::<_, String>(0)) + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() .map(Result::unwrap) .collect() @@ -183,63 +183,71 @@ async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { assert_eq!(std::fs::read(&wal).unwrap(), foreign_wal); } -#[tokio::test(flavor = "multi_thread")] -async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { - let reference = RegisteredStore::create(); - let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; - outcome.unwrap(); - let installed = reference.external_schema_objects(); - assert!( - installed.contains(&"table:code_projects".to_owned()), - "a completed attach installs the registered schema: {installed:?}" - ); +#[test] +fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { + tracedecay_runtime_core::native_runtime::default_builder() + .build() + .expect("schema cancellation runtime") + .block_on(async { + let store = RegisteredStore::create(); + let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; + outcome.unwrap(); + let installed = store.external_schema_objects(); + assert!( + installed + .iter() + .any(|(object, _)| object == "table:code_projects"), + "a completed attach installs the registered schema: {installed:?}" + ); - let stride = (total_polls / 20).max(1); - let mut cancelled = Vec::new(); - let mut completed = Vec::new(); - for cancel_at in std::iter::once(1) - .chain((2..total_polls).step_by(stride)) - .chain([total_polls + 10]) - { - let store = RegisteredStore::create(); - // Cancellation cannot modify the already-published schema or - // prevent the next admission from reopening the same family. - let (owner, runtime, retirement) = store.publish().await.into_parts(); - drop((owner, runtime)); - retire_runtime(retirement).await; - let before_admission = store.external_schema_objects(); - assert_eq!(before_admission, installed); - let (outcome, _) = store.attach_cancelled_at(cancel_at).await; - let objects = store.external_schema_objects(); - match outcome { - Err(error) => { - assert!( - error.is_store_open_cancelled(), - "cancel at poll {cancel_at}/{total_polls} must fail typed: {error:?}" - ); - assert_eq!( - objects, before_admission, - "cancel at poll {cancel_at}/{total_polls} must preserve the published schema" - ); - cancelled.push(cancel_at); - } - Ok(()) => { - assert_eq!( - objects, installed, - "an admission completed before cancel at poll {cancel_at}/{total_polls} preserves the published schema" - ); - completed.push(cancel_at); + let stride = (total_polls / 20).max(1); + let mut cancelled = Vec::new(); + let mut completed = Vec::new(); + for cancel_at in std::iter::once(1) + .chain((2..total_polls).step_by(stride)) + .chain([total_polls + 10]) + { + // The reference admission, then each successful reopen below, leave + // this family closed with the exact baseline schema. Creating another + // family would repeat initialization without exercising another + // cancellation point. Keep real runtime retirement and reopening + // around every sampled admission. + let before_admission = store.external_schema_objects(); + assert_eq!(before_admission, installed); + let (outcome, _) = store.attach_cancelled_at(cancel_at).await; + let objects = store.external_schema_objects(); + match outcome { + Err(error) => { + assert!( + error.is_store_open_cancelled(), + "cancel at poll {cancel_at}/{total_polls} must fail typed: {error:?}" + ); + assert_eq!( + objects, before_admission, + "cancel at poll {cancel_at}/{total_polls} must preserve the published schema" + ); + cancelled.push(cancel_at); + } + Ok(()) => { + assert_eq!( + objects, installed, + "an admission completed before cancel at poll {cancel_at}/{total_polls} preserves the published schema" + ); + completed.push(cancel_at); + } + } + let (reopened, _) = store.attach_cancelled_at(usize::MAX).await; + reopened.unwrap_or_else(|error| { + panic!( + "reopen after cancel at poll {cancel_at}/{total_polls} failed: {error:?}" + ) + }); + assert_eq!(store.external_schema_objects(), installed); } - } - let (reopened, _) = store.attach_cancelled_at(usize::MAX).await; - reopened.unwrap_or_else(|error| { - panic!("reopen after cancel at poll {cancel_at}/{total_polls} failed: {error:?}") + assert!( + cancelled.contains(&1), + "a cancel before admission starts must stop it; cancelled at {cancelled:?}" + ); + assert!(completed.contains(&(total_polls + 10))); }); - assert_eq!(store.external_schema_objects(), installed); - } - assert!( - cancelled.contains(&1), - "a cancel before admission starts must stop it; cancelled at {cancelled:?}" - ); - assert!(completed.contains(&(total_polls + 10))); } diff --git a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs index 1abbd6d64e6..c06eefe1a47 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs @@ -20,7 +20,7 @@ use tracedecay_runtime_core::test_executable::write_executable_script; use tracedecay_sessions::runtime::{SessionProvider, with_transcript_source_profile}; #[cfg(unix)] -use crate::common::{spawn_tracedecay_daemon, tracedecay_command_with_home}; +use crate::common::{spawn_tracedecay_daemon_logged, tracedecay_command_with_home}; #[cfg(unix)] use crate::restart_atomicity::mark_test_project; use crate::support::setup; @@ -182,19 +182,27 @@ done ) .unwrap(); let project_id = mark_test_project(&project); + let daemon_log = tmp.path().join("codex-compaction-daemon.log"); + let daemon = spawn_tracedecay_daemon_logged(&home, &daemon_log, |command| { + command + .env( + "RUST_LOG", + "info,tracedecay_session_runtime::session_temporal_refresh_scheduler=trace,tracedecay_mcp::handlers::hook_runtime=trace", + ) + .env("TRACEDECAY_SPAN_TIMINGS", "1"); + }); // The hook resolves the project root through the initialized-store gate, // exactly like production installs: `init` creates the project store and // admits the canonical daemon-owned project route. crate::common::initialize_tracedecay_cli_project(&home, &project); // The daemon launches a summarizer only through the project's // `lcm.summarizer_executables.v1` binding, so the fake `codex` is bound - // through the production configuration surface while the managed daemon - // from init is still serving the project. + // through the production configuration surface while the same daemon + // is still serving the project. configure_codex_summarizer(&home, &project, &project_id, &codex_bin); - // Keep that production owner through the hook. An in-process enrollment + // Keep the logged production owner through the hook. An in-process enrollment // adds no fixture data and would require retiring and cold-opening the // same stores again just before the hook's bounded publication join. - let daemon = spawn_tracedecay_daemon(&home); write_codex_rollout_with_compaction(&home, &project, "codex-compact"); let event = serde_json::json!({ @@ -216,15 +224,17 @@ done .unwrap(); writeln!(child.stdin.take().unwrap(), "{event}").unwrap(); let output = child.wait_with_output().unwrap(); + let daemon_diagnostics = std::fs::read_to_string(&daemon_log) + .unwrap_or_else(|error| format!("daemon log unavailable: {error}")); assert!( output.status.success(), - "Codex PostCompact hook failed: {}", - String::from_utf8_lossy(&output.stderr) + "Codex PostCompact hook failed: {}; daemon log: {daemon_diagnostics}", + String::from_utf8_lossy(&output.stderr), ); assert!( !String::from_utf8_lossy(&output.stderr).contains("PostCompact daemon call failed"), - "the PostCompact hook must not fail open around the daemon effect: {}", - String::from_utf8_lossy(&output.stderr) + "the PostCompact hook must not fail open around the daemon effect: {}; daemon log: {daemon_diagnostics}", + String::from_utf8_lossy(&output.stderr), ); assert_eq!( std::fs::read_to_string(codex_bin.with_extension("launches")).unwrap(), From 22943951a4802ff9b7e526b90ef45d3c3d492cc4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:04:23 -0800 Subject: [PATCH 36/50] test(runtime): remove completed investigation tracing --- .../runtime_surface_acceptance.rs | 3 +-- .../tests/transcript_ingest_suite/codex_compaction.rs | 9 +-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 3737ab21b4f..1fea2c8882a 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -228,8 +228,7 @@ async fn lsp_runtime_fixture() -> RuntimeFixture { command .env( "RUST_LOG", - "info,tracedecay_daemon_service::invocation=trace,\ - tracedecay_private_fs::framed_log=trace", + "info,tracedecay_daemon_service::invocation=trace", ) .env("TRACEDECAY_SPAN_TIMINGS", "1") .stderr(daemon_log); diff --git a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs index c06eefe1a47..45c6f603565 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs @@ -183,14 +183,7 @@ done .unwrap(); let project_id = mark_test_project(&project); let daemon_log = tmp.path().join("codex-compaction-daemon.log"); - let daemon = spawn_tracedecay_daemon_logged(&home, &daemon_log, |command| { - command - .env( - "RUST_LOG", - "info,tracedecay_session_runtime::session_temporal_refresh_scheduler=trace,tracedecay_mcp::handlers::hook_runtime=trace", - ) - .env("TRACEDECAY_SPAN_TIMINGS", "1"); - }); + let daemon = spawn_tracedecay_daemon_logged(&home, &daemon_log, |_| {}); // The hook resolves the project root through the initialized-store gate, // exactly like production installs: `init` creates the project store and // admits the canonical daemon-owned project route. From 123aae98f9693a9c3797796852e0a22a686fe915 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:04:41 -0800 Subject: [PATCH 37/50] test(storage): trace global schema fixture phases --- .../tracedecay-global-db/src/tests/harness.rs | 7 ++- .../src/tests/lcm_schema/mod.rs | 46 +++++++++++++++---- crates/tracedecay-runtime-core/src/logging.rs | 26 ++++++++++- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index 7de179fbb98..4fc8d82183f 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -4,6 +4,8 @@ use std::sync::OnceLock; use std::sync::atomic::{AtomicU64, Ordering}; use tempfile::TempDir; +#[cfg(test)] +use tracing::Instrument; use crate::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1}; use tracedecay_domain::canonical_text::sha256_hex; @@ -1515,7 +1517,10 @@ async fn open_registered_test_database_with_identity( } }; let (database_owner, _runtime, _retirement) = fixture.into_parts(); - let database = RegisteredGlobalDbOwnerV1::admit_and_attach(database_owner).await?; + let attachment = RegisteredGlobalDbOwnerV1::admit_and_attach(database_owner); + #[cfg(test)] + let attachment = attachment.instrument(tracing::info_span!("global_db.fixture.attach")); + let database = attachment.await?; // The physical fixture is already opened with the requested authority. // Issuance preserves that capability; neither branch manufactures a // second raw authority after publication. diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 22a7d3611c0..93757382854 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -1,10 +1,11 @@ #![allow(clippy::collapsible_if)] // test scaffolding use std::path::Path; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tempfile::TempDir; use tokio::sync::oneshot; use tokio::time::timeout; +use tracing::{Instrument, instrument::WithSubscriber}; use tracedecay_runtime_core::db::engine::{ Connection, TestConnection, TransactionBehavior, params, @@ -12,18 +13,45 @@ use tracedecay_runtime_core::db::engine::{ use tracedecay_runtime_core::db::{ Database, DatabaseAuthority, TestDatabaseRuntimeMode, TestDatabaseRuntimeScope, }; +use tracedecay_runtime_core::logging::FormattedTracingCapture; use crate::tests::harness::open_registered_test_database_fixture; async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { - if let Some(parent) = db_path.parent() { - std::fs::create_dir_all(parent)?; - } - drop( - open_registered_test_database_fixture(db_path, TestDatabaseRuntimeScope::ProfileSessions) - .await?, - ); - Ok(TestConnection::open(db_path)) + // Carry the dispatcher on the future; the thread-local guard must leave + // scope before the caller awaits or moves this future between workers. + let opening = { + let _capture = FormattedTracingCapture::start_with_info_span_timings(); + let thread = std::thread::current(); + let started_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let span = tracing::info_span!( + "global_db.schema_fixture", + test = thread.name().unwrap_or("unnamed"), + started_at_unix_ms = started_at.as_millis() as u64, + ); + async move { + if let Some(parent) = db_path.parent() { + std::fs::create_dir_all(parent)?; + } + let fixture = open_registered_test_database_fixture( + db_path, + TestDatabaseRuntimeScope::ProfileSessions, + ) + .instrument(tracing::info_span!("global_db.fixture.admit")) + .await?; + { + let _span = tracing::info_span!("global_db.fixture.release").entered(); + drop(fixture); + } + let _span = tracing::info_span!("global_db.fixture.raw_open").entered(); + Ok(TestConnection::open(db_path)) + } + .instrument(span) + .with_current_subscriber() + }; + opening.await } async fn open_read_only_global_db( diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 94cba0b7b6c..d7acb54d0b4 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -289,10 +289,34 @@ impl FormattedTracingCapture { Self::start_with_output(true) } + /// Records high-level phase boundaries without verbose SQL trace events. + #[must_use] + pub fn start_with_info_span_timings() -> Self { + Self::start_with_settings( + true, + tracing::Level::INFO, + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + } + fn start_with_output(stderr: bool) -> Self { + Self::start_with_settings( + stderr, + tracing::Level::TRACE, + tracing_subscriber::fmt::format::FmtSpan::NONE, + ) + } + + fn start_with_settings( + stderr: bool, + level: tracing::Level, + span_events: tracing_subscriber::fmt::format::FmtSpan, + ) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() - .with_max_level(tracing::Level::TRACE) + .with_max_level(level) + .with_span_events(span_events) .with_ansi(false) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { From 767aa2147aa00e62b9722ef66da3f66426ee7ec6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:11:34 -0800 Subject: [PATCH 38/50] test(storage): remove schema phase investigation capture --- .../tracedecay-global-db/src/tests/harness.rs | 7 +-- .../src/tests/lcm_schema/mod.rs | 46 ++++--------------- crates/tracedecay-runtime-core/src/logging.rs | 26 +---------- 3 files changed, 11 insertions(+), 68 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index 4fc8d82183f..7de179fbb98 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -4,8 +4,6 @@ use std::sync::OnceLock; use std::sync::atomic::{AtomicU64, Ordering}; use tempfile::TempDir; -#[cfg(test)] -use tracing::Instrument; use crate::{RegisteredGlobalDb, RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1}; use tracedecay_domain::canonical_text::sha256_hex; @@ -1517,10 +1515,7 @@ async fn open_registered_test_database_with_identity( } }; let (database_owner, _runtime, _retirement) = fixture.into_parts(); - let attachment = RegisteredGlobalDbOwnerV1::admit_and_attach(database_owner); - #[cfg(test)] - let attachment = attachment.instrument(tracing::info_span!("global_db.fixture.attach")); - let database = attachment.await?; + let database = RegisteredGlobalDbOwnerV1::admit_and_attach(database_owner).await?; // The physical fixture is already opened with the requested authority. // Issuance preserves that capability; neither branch manufactures a // second raw authority after publication. diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 93757382854..22a7d3611c0 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -1,11 +1,10 @@ #![allow(clippy::collapsible_if)] // test scaffolding use std::path::Path; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use tempfile::TempDir; use tokio::sync::oneshot; use tokio::time::timeout; -use tracing::{Instrument, instrument::WithSubscriber}; use tracedecay_runtime_core::db::engine::{ Connection, TestConnection, TransactionBehavior, params, @@ -13,45 +12,18 @@ use tracedecay_runtime_core::db::engine::{ use tracedecay_runtime_core::db::{ Database, DatabaseAuthority, TestDatabaseRuntimeMode, TestDatabaseRuntimeScope, }; -use tracedecay_runtime_core::logging::FormattedTracingCapture; use crate::tests::harness::open_registered_test_database_fixture; async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { - // Carry the dispatcher on the future; the thread-local guard must leave - // scope before the caller awaits or moves this future between workers. - let opening = { - let _capture = FormattedTracingCapture::start_with_info_span_timings(); - let thread = std::thread::current(); - let started_at = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default(); - let span = tracing::info_span!( - "global_db.schema_fixture", - test = thread.name().unwrap_or("unnamed"), - started_at_unix_ms = started_at.as_millis() as u64, - ); - async move { - if let Some(parent) = db_path.parent() { - std::fs::create_dir_all(parent)?; - } - let fixture = open_registered_test_database_fixture( - db_path, - TestDatabaseRuntimeScope::ProfileSessions, - ) - .instrument(tracing::info_span!("global_db.fixture.admit")) - .await?; - { - let _span = tracing::info_span!("global_db.fixture.release").entered(); - drop(fixture); - } - let _span = tracing::info_span!("global_db.fixture.raw_open").entered(); - Ok(TestConnection::open(db_path)) - } - .instrument(span) - .with_current_subscriber() - }; - opening.await + if let Some(parent) = db_path.parent() { + std::fs::create_dir_all(parent)?; + } + drop( + open_registered_test_database_fixture(db_path, TestDatabaseRuntimeScope::ProfileSessions) + .await?, + ); + Ok(TestConnection::open(db_path)) } async fn open_read_only_global_db( diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index d7acb54d0b4..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -289,34 +289,10 @@ impl FormattedTracingCapture { Self::start_with_output(true) } - /// Records high-level phase boundaries without verbose SQL trace events. - #[must_use] - pub fn start_with_info_span_timings() -> Self { - Self::start_with_settings( - true, - tracing::Level::INFO, - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - } - fn start_with_output(stderr: bool) -> Self { - Self::start_with_settings( - stderr, - tracing::Level::TRACE, - tracing_subscriber::fmt::format::FmtSpan::NONE, - ) - } - - fn start_with_settings( - stderr: bool, - level: tracing::Level, - span_events: tracing_subscriber::fmt::format::FmtSpan, - ) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() - .with_max_level(level) - .with_span_events(span_events) + .with_max_level(tracing::Level::TRACE) .with_ansi(false) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { From caaff6f75d507e71fc2f92589410e0384ec9de04 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:55:43 -0800 Subject: [PATCH 39/50] test(storage): measure workflow cancellation sweep phases --- .../src/registered/workflow_schema_tests.rs | 96 +++++++++++++++---- crates/tracedecay-runtime-core/src/logging.rs | 26 ++++- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index e2c6d01b7e2..48d13a4d295 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -287,6 +287,7 @@ mod published_artifact_ceiling { use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::Poll; + use std::time::{SystemTime, UNIX_EPOCH}; use rusqlite::Connection; use tempfile::TempDir; @@ -297,12 +298,14 @@ mod published_artifact_ceiling { use tracedecay_runtime_core::db::{ RegisteredTestRuntimeRetirementControlV1, TestDatabaseRuntimeScope, }; + use tracedecay_runtime_core::logging::FormattedTracingCapture; use tracedecay_runtime_core::shard_runtime::registry::{ StoreRuntimeClientLease, StoreRuntimeRetirementOutcome, StoreRuntimeRetirementResult, }; use tracedecay_rusqlite_runtime::workflow::{ WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1, WORKFLOW_TABLE_CONTRACTS_V1, }; + use tracing::{Instrument, instrument::WithSubscriber}; use super::fs; use crate::RegisteredGlobalDbOwnerV1; @@ -431,6 +434,11 @@ mod published_artifact_ceiling { } impl WorkflowState { + #[tracing::instrument( + name = "global_db.workflow_sweep.read_state", + level = "info", + skip_all + )] fn read(path: &Path) -> Self { let connection = Connection::open(path).unwrap(); let table_sql = connection @@ -1010,11 +1018,36 @@ mod published_artifact_ceiling { /// stays typed recovering, and the next open finishes the job. #[tokio::test(flavor = "multi_thread")] async fn background_upgrade_is_atomic_at_every_cancellation_point() { + let sweep = { + let _capture = FormattedTracingCapture::start_with_info_span_timings(); + let started_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let span = tracing::info_span!( + "global_db.workflow_sweep", + started_at_unix_ms = started_at.as_millis() as u64, + ); + run_background_upgrade_cancellation_sweep() + .instrument(span) + .with_current_subscriber() + }; + sweep.await; + } + + async fn run_background_upgrade_cancellation_sweep() { + #[tracing::instrument( + name = "global_db.workflow_sweep.sample", + level = "info", + skip_all, + fields(cancel_at) + )] async fn converge_cancelled_at( path: &Path, cancel_at: usize, ) -> (Result<(), TraceDecayError>, usize) { - let (owner, convergence, (runtime, retirement)) = admit_for_daemon(path).await; + let (owner, convergence, (runtime, retirement)) = admit_for_daemon(path) + .instrument(tracing::info_span!("global_db.workflow_sweep.admit")) + .await; let lease = owner.issue_lease().unwrap(); let cancellation = CancellationToken::new(); let mut recovery = Box::pin(lease.converge_schema(convergence, &cancellation)); @@ -1026,29 +1059,37 @@ mod published_artifact_ceiling { } recovery.as_mut().poll(context) }) + .instrument(tracing::info_span!("global_db.workflow_sweep.converge")) .await; drop(recovery); if let Err(error) = &outcome && error.is_store_open_cancelled() && WorkflowState::read(path).digest == PUBLISHED_DIGEST { - assert_workflow_recovering(&lease).await; + assert_workflow_recovering(&lease) + .instrument(tracing::info_span!("global_db.workflow_sweep.recovering_gate")) + .await; } // Each sampled cancellation reopens this path. Join the exact // published runtime before the next admission can acquire it. drop((lease, owner, runtime)); - let mut reservation = match retirement - .registry() - .reserve_retirement_batch(vec![retirement.retirement_target()]) - { - StoreRuntimeRetirementResult::Reserved(reservation) => reservation, - StoreRuntimeRetirementResult::Blocked(refusal) => { - panic!("completed convergence retained runtime clients: {refusal:?}") + let mut reservation = { + let _span = tracing::info_span!("global_db.workflow_sweep.reserve_retirement") + .entered(); + match retirement + .registry() + .reserve_retirement_batch(vec![retirement.retirement_target()]) + { + StoreRuntimeRetirementResult::Reserved(reservation) => reservation, + StoreRuntimeRetirementResult::Blocked(refusal) => { + panic!("completed convergence retained runtime clients: {refusal:?}") + } } }; let committed = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { reservation.commit() }) + .instrument(tracing::info_span!("global_db.workflow_sweep.join_retirement")) .await .unwrap() .unwrap(); @@ -1063,23 +1104,36 @@ mod published_artifact_ceiling { (outcome, polls) } - let reference = DaemonPublishedStore::create().await; - reference.store.run(BULK_ROWS); - let before = reference.store.state(); + let reference = DaemonPublishedStore::create() + .instrument(tracing::info_span!("global_db.workflow_sweep.reference_fixture")) + .await; + tracing::info_span!("global_db.workflow_sweep.reference_seed") + .in_scope(|| reference.store.run(BULK_ROWS)); + let before = tracing::info_span!("global_db.workflow_sweep.reference_before") + .in_scope(|| reference.store.state()); let (outcome, total_polls) = converge_cancelled_at(&reference.store.path, usize::MAX).await; outcome.unwrap(); - let completed = reference.store.state(); + let completed = tracing::info_span!("global_db.workflow_sweep.reference_after") + .in_scope(|| reference.store.state()); assert_eq!(completed.digest, WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1); assert_eq!(completed.artifacts, before.artifacts); let stride = (total_polls / 12).max(1); + tracing::info!(total_polls, stride, "workflow cancellation sweep calibrated"); let mut cancelled_published = 0; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create().await; + let daemon_store = DaemonPublishedStore::create() + .instrument(tracing::info_span!( + "global_db.workflow_sweep.fixture", + cancel_at + )) + .await; let store = &daemon_store.store; - store.run(BULK_ROWS); + tracing::info_span!("global_db.workflow_sweep.seed", cancel_at) + .in_scope(|| store.run(BULK_ROWS)); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; - let state = store.state(); + let state = tracing::info_span!("global_db.workflow_sweep.state", cancel_at) + .in_scope(|| store.state()); match outcome { Ok(()) => assert_eq!(state, completed, "completed at poll {cancel_at}"), Err(error) => { @@ -1092,7 +1146,15 @@ mod published_artifact_ceiling { cancelled_published += 1; let (reopened, _) = converge_cancelled_at(&store.path, usize::MAX).await; reopened.unwrap(); - assert_eq!(store.state(), completed, "reopen after poll {cancel_at}"); + assert_eq!( + tracing::info_span!( + "global_db.workflow_sweep.reopened_state", + cancel_at + ) + .in_scope(|| store.state()), + completed, + "reopen after poll {cancel_at}" + ); } } } diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 94cba0b7b6c..d7acb54d0b4 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -289,10 +289,34 @@ impl FormattedTracingCapture { Self::start_with_output(true) } + /// Records high-level phase boundaries without verbose SQL trace events. + #[must_use] + pub fn start_with_info_span_timings() -> Self { + Self::start_with_settings( + true, + tracing::Level::INFO, + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + } + fn start_with_output(stderr: bool) -> Self { + Self::start_with_settings( + stderr, + tracing::Level::TRACE, + tracing_subscriber::fmt::format::FmtSpan::NONE, + ) + } + + fn start_with_settings( + stderr: bool, + level: tracing::Level, + span_events: tracing_subscriber::fmt::format::FmtSpan, + ) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() - .with_max_level(tracing::Level::TRACE) + .with_max_level(level) + .with_span_events(span_events) .with_ansi(false) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { From 569ee95fac9d817894894c1097a7a679def89251 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:48:20 -0800 Subject: [PATCH 40/50] style(storage): format workflow sweep phase spans --- .../src/registered/workflow_schema_tests.rs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 48d13a4d295..02cc051ad26 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -1067,15 +1067,17 @@ mod published_artifact_ceiling { && WorkflowState::read(path).digest == PUBLISHED_DIGEST { assert_workflow_recovering(&lease) - .instrument(tracing::info_span!("global_db.workflow_sweep.recovering_gate")) + .instrument(tracing::info_span!( + "global_db.workflow_sweep.recovering_gate" + )) .await; } // Each sampled cancellation reopens this path. Join the exact // published runtime before the next admission can acquire it. drop((lease, owner, runtime)); let mut reservation = { - let _span = tracing::info_span!("global_db.workflow_sweep.reserve_retirement") - .entered(); + let _span = + tracing::info_span!("global_db.workflow_sweep.reserve_retirement").entered(); match retirement .registry() .reserve_retirement_batch(vec![retirement.retirement_target()]) @@ -1089,7 +1091,9 @@ mod published_artifact_ceiling { let committed = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { reservation.commit() }) - .instrument(tracing::info_span!("global_db.workflow_sweep.join_retirement")) + .instrument(tracing::info_span!( + "global_db.workflow_sweep.join_retirement" + )) .await .unwrap() .unwrap(); @@ -1105,7 +1109,9 @@ mod published_artifact_ceiling { } let reference = DaemonPublishedStore::create() - .instrument(tracing::info_span!("global_db.workflow_sweep.reference_fixture")) + .instrument(tracing::info_span!( + "global_db.workflow_sweep.reference_fixture" + )) .await; tracing::info_span!("global_db.workflow_sweep.reference_seed") .in_scope(|| reference.store.run(BULK_ROWS)); @@ -1119,7 +1125,11 @@ mod published_artifact_ceiling { assert_eq!(completed.artifacts, before.artifacts); let stride = (total_polls / 12).max(1); - tracing::info!(total_polls, stride, "workflow cancellation sweep calibrated"); + tracing::info!( + total_polls, + stride, + "workflow cancellation sweep calibrated" + ); let mut cancelled_published = 0; for cancel_at in (1..total_polls).step_by(stride) { let daemon_store = DaemonPublishedStore::create() From e713de429cbddd9970947ffc398fb5757730054f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:50:00 -0800 Subject: [PATCH 41/50] test(storage): remove workflow sweep phase capture --- .../src/registered/workflow_schema_tests.rs | 106 +++--------------- crates/tracedecay-runtime-core/src/logging.rs | 26 +---- 2 files changed, 18 insertions(+), 114 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 02cc051ad26..e2c6d01b7e2 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -287,7 +287,6 @@ mod published_artifact_ceiling { use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::Poll; - use std::time::{SystemTime, UNIX_EPOCH}; use rusqlite::Connection; use tempfile::TempDir; @@ -298,14 +297,12 @@ mod published_artifact_ceiling { use tracedecay_runtime_core::db::{ RegisteredTestRuntimeRetirementControlV1, TestDatabaseRuntimeScope, }; - use tracedecay_runtime_core::logging::FormattedTracingCapture; use tracedecay_runtime_core::shard_runtime::registry::{ StoreRuntimeClientLease, StoreRuntimeRetirementOutcome, StoreRuntimeRetirementResult, }; use tracedecay_rusqlite_runtime::workflow::{ WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1, WORKFLOW_TABLE_CONTRACTS_V1, }; - use tracing::{Instrument, instrument::WithSubscriber}; use super::fs; use crate::RegisteredGlobalDbOwnerV1; @@ -434,11 +431,6 @@ mod published_artifact_ceiling { } impl WorkflowState { - #[tracing::instrument( - name = "global_db.workflow_sweep.read_state", - level = "info", - skip_all - )] fn read(path: &Path) -> Self { let connection = Connection::open(path).unwrap(); let table_sql = connection @@ -1018,36 +1010,11 @@ mod published_artifact_ceiling { /// stays typed recovering, and the next open finishes the job. #[tokio::test(flavor = "multi_thread")] async fn background_upgrade_is_atomic_at_every_cancellation_point() { - let sweep = { - let _capture = FormattedTracingCapture::start_with_info_span_timings(); - let started_at = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default(); - let span = tracing::info_span!( - "global_db.workflow_sweep", - started_at_unix_ms = started_at.as_millis() as u64, - ); - run_background_upgrade_cancellation_sweep() - .instrument(span) - .with_current_subscriber() - }; - sweep.await; - } - - async fn run_background_upgrade_cancellation_sweep() { - #[tracing::instrument( - name = "global_db.workflow_sweep.sample", - level = "info", - skip_all, - fields(cancel_at) - )] async fn converge_cancelled_at( path: &Path, cancel_at: usize, ) -> (Result<(), TraceDecayError>, usize) { - let (owner, convergence, (runtime, retirement)) = admit_for_daemon(path) - .instrument(tracing::info_span!("global_db.workflow_sweep.admit")) - .await; + let (owner, convergence, (runtime, retirement)) = admit_for_daemon(path).await; let lease = owner.issue_lease().unwrap(); let cancellation = CancellationToken::new(); let mut recovery = Box::pin(lease.converge_schema(convergence, &cancellation)); @@ -1059,41 +1026,29 @@ mod published_artifact_ceiling { } recovery.as_mut().poll(context) }) - .instrument(tracing::info_span!("global_db.workflow_sweep.converge")) .await; drop(recovery); if let Err(error) = &outcome && error.is_store_open_cancelled() && WorkflowState::read(path).digest == PUBLISHED_DIGEST { - assert_workflow_recovering(&lease) - .instrument(tracing::info_span!( - "global_db.workflow_sweep.recovering_gate" - )) - .await; + assert_workflow_recovering(&lease).await; } // Each sampled cancellation reopens this path. Join the exact // published runtime before the next admission can acquire it. drop((lease, owner, runtime)); - let mut reservation = { - let _span = - tracing::info_span!("global_db.workflow_sweep.reserve_retirement").entered(); - match retirement - .registry() - .reserve_retirement_batch(vec![retirement.retirement_target()]) - { - StoreRuntimeRetirementResult::Reserved(reservation) => reservation, - StoreRuntimeRetirementResult::Blocked(refusal) => { - panic!("completed convergence retained runtime clients: {refusal:?}") - } + let mut reservation = match retirement + .registry() + .reserve_retirement_batch(vec![retirement.retirement_target()]) + { + StoreRuntimeRetirementResult::Reserved(reservation) => reservation, + StoreRuntimeRetirementResult::Blocked(refusal) => { + panic!("completed convergence retained runtime clients: {refusal:?}") } }; let committed = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { reservation.commit() }) - .instrument(tracing::info_span!( - "global_db.workflow_sweep.join_retirement" - )) .await .unwrap() .unwrap(); @@ -1108,42 +1063,23 @@ mod published_artifact_ceiling { (outcome, polls) } - let reference = DaemonPublishedStore::create() - .instrument(tracing::info_span!( - "global_db.workflow_sweep.reference_fixture" - )) - .await; - tracing::info_span!("global_db.workflow_sweep.reference_seed") - .in_scope(|| reference.store.run(BULK_ROWS)); - let before = tracing::info_span!("global_db.workflow_sweep.reference_before") - .in_scope(|| reference.store.state()); + let reference = DaemonPublishedStore::create().await; + reference.store.run(BULK_ROWS); + let before = reference.store.state(); let (outcome, total_polls) = converge_cancelled_at(&reference.store.path, usize::MAX).await; outcome.unwrap(); - let completed = tracing::info_span!("global_db.workflow_sweep.reference_after") - .in_scope(|| reference.store.state()); + let completed = reference.store.state(); assert_eq!(completed.digest, WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1); assert_eq!(completed.artifacts, before.artifacts); let stride = (total_polls / 12).max(1); - tracing::info!( - total_polls, - stride, - "workflow cancellation sweep calibrated" - ); let mut cancelled_published = 0; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create() - .instrument(tracing::info_span!( - "global_db.workflow_sweep.fixture", - cancel_at - )) - .await; + let daemon_store = DaemonPublishedStore::create().await; let store = &daemon_store.store; - tracing::info_span!("global_db.workflow_sweep.seed", cancel_at) - .in_scope(|| store.run(BULK_ROWS)); + store.run(BULK_ROWS); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; - let state = tracing::info_span!("global_db.workflow_sweep.state", cancel_at) - .in_scope(|| store.state()); + let state = store.state(); match outcome { Ok(()) => assert_eq!(state, completed, "completed at poll {cancel_at}"), Err(error) => { @@ -1156,15 +1092,7 @@ mod published_artifact_ceiling { cancelled_published += 1; let (reopened, _) = converge_cancelled_at(&store.path, usize::MAX).await; reopened.unwrap(); - assert_eq!( - tracing::info_span!( - "global_db.workflow_sweep.reopened_state", - cancel_at - ) - .in_scope(|| store.state()), - completed, - "reopen after poll {cancel_at}" - ); + assert_eq!(store.state(), completed, "reopen after poll {cancel_at}"); } } } diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index d7acb54d0b4..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -289,34 +289,10 @@ impl FormattedTracingCapture { Self::start_with_output(true) } - /// Records high-level phase boundaries without verbose SQL trace events. - #[must_use] - pub fn start_with_info_span_timings() -> Self { - Self::start_with_settings( - true, - tracing::Level::INFO, - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - } - fn start_with_output(stderr: bool) -> Self { - Self::start_with_settings( - stderr, - tracing::Level::TRACE, - tracing_subscriber::fmt::format::FmtSpan::NONE, - ) - } - - fn start_with_settings( - stderr: bool, - level: tracing::Level, - span_events: tracing_subscriber::fmt::format::FmtSpan, - ) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() - .with_max_level(level) - .with_span_events(span_events) + .with_max_level(tracing::Level::TRACE) .with_ansi(false) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { From 778f66a956521710b2f3ddb4940233c6af8aa117 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:38:28 -0800 Subject: [PATCH 42/50] test(global-db): reuse workflow cancellation baseline --- .../src/registered/workflow_schema_tests.rs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index e2c6d01b7e2..c88c416a63e 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -1074,10 +1074,36 @@ mod published_artifact_ceiling { let stride = (total_polls / 12).max(1); let mut cancelled_published = 0; + // Each convergence joins its exact runtime retirement. Reuse the + // canonically initialized store, restoring the upgrade's inputs and + // proving the entire workflow state before admitting a fresh runtime. + let store = &reference.store; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create().await; - let store = &daemon_store.store; - store.run(BULK_ROWS); + { + let mut connection = Connection::open(&store.path).unwrap(); + let transaction = connection.transaction().unwrap(); + republish(&transaction, &before.table_sql, &before.digest); + { + let mut insert = transaction + .prepare( + "INSERT INTO workflow_artifact_payloads + (rowid, payload_digest, byte_length, payload) + VALUES (?1, ?2, ?3, ?4)", + ) + .unwrap(); + for (rowid, digest, byte_length, payload) in &before.artifacts { + insert + .execute(rusqlite::params![rowid, digest, byte_length, payload]) + .unwrap(); + } + } + transaction.commit().unwrap(); + } + assert_eq!( + store.state(), + before, + "poll {cancel_at} must start from the published workflow state" + ); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; let state = store.state(); match outcome { From 68e0a258bbf80f4cc1a86e428957d7c125d861a3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:20:45 -0800 Subject: [PATCH 43/50] test(mcp): name response phases in timeout failures Return the existing timeout error to each test expectation so a failure names the response it awaited. Preserve the five-second bound, request ordering, route permits, and every response assertion. --- .../tracedecay/src/mcp/server/connection.rs | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay/src/mcp/server/connection.rs b/crates/tracedecay/src/mcp/server/connection.rs index 2852792169e..6c0e02a3746 100644 --- a/crates/tracedecay/src/mcp/server/connection.rs +++ b/crates/tracedecay/src/mcp/server/connection.rs @@ -666,7 +666,7 @@ mod cancellable_queue_tests { async fn receive_response( responses: &mut tokio::sync::mpsc::UnboundedReceiver, - ) -> Value { + ) -> std::result::Result { tokio::time::timeout(Duration::from_secs(5), async { loop { let line = responses.recv().await.expect("connection response"); @@ -678,7 +678,6 @@ mod cancellable_queue_tests { } }) .await - .expect("connection response timeout") } #[tokio::test] @@ -727,7 +726,9 @@ mod cancellable_queue_tests { ) .expect("send independent status read"); - let fast = receive_response(&mut responses).await; + let fast = receive_response(&mut responses) + .await + .expect("independent status response"); assert_eq!( fast["id"], serde_json::json!(2), @@ -735,7 +736,9 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); - let slow = receive_response(&mut responses).await; + let slow = receive_response(&mut responses) + .await + .expect("delayed selected-read response"); assert!( slow["result"]["content"].is_array(), "selected fact read: {slow}" @@ -843,7 +846,12 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); - assert_eq!(receive_response(&mut responses).await["id"], json!(10)); + assert_eq!( + receive_response(&mut responses) + .await + .expect("read-before-effect response")["id"], + json!(10) + ); fixture.wait_for_routes(2).await; assert!( tokio::time::timeout(Duration::from_millis(50), responses.recv()) @@ -853,12 +861,16 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); assert_eq!( - receive_response(&mut responses).await["id"], + receive_response(&mut responses) + .await + .expect("effect-barrier response")["id"], json!(11), "the effect must settle after the preceding read" ); assert_eq!( - receive_response(&mut responses).await["id"], + receive_response(&mut responses) + .await + .expect("read-after-effect response")["id"], json!(12), "the later read must not overtake the effect" ); @@ -915,7 +927,9 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); - let _ = receive_response(&mut responses).await; + let _ = receive_response(&mut responses) + .await + .expect("first admitted-read response"); fixture .wait_for_routes(MAX_CONCURRENT_CONNECTION_READS + 1) .await; @@ -925,7 +939,12 @@ mod cancellable_queue_tests { let mut response_ids = HashSet::new(); while response_ids.len() < MAX_CONCURRENT_CONNECTION_READS { - response_ids.insert(receive_response(&mut responses).await["id"].clone()); + response_ids.insert( + receive_response(&mut responses) + .await + .expect("remaining admitted-read response")["id"] + .clone(), + ); } assert_eq!( response_ids.len(), From a9345b4924002cf68ae583c18268f2f1119a813d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:30:49 -0800 Subject: [PATCH 44/50] test(global-db): capture restored sweep phase timings Temporarily scope the existing formatted span capture to the workflow cancellation sweep and its registered-owner and schema phases. Time the synchronous fixture restoration and complete state reads while retaining every cancellation sample, reopen assertion, and joined runtime owner. Reuse the reviewed static-target capture helper without changing tracing dependencies or production behavior. Remove this measurement after useful owning-suite evidence is collected. --- .../src/registered/workflow_schema_tests.rs | 41 +++++++++++++++++++ crates/tracedecay-runtime-core/src/logging.rs | 30 ++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index c88c416a63e..fd66b791212 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -287,6 +287,7 @@ mod published_artifact_ceiling { use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::Poll; + use std::time::{SystemTime, UNIX_EPOCH}; use rusqlite::Connection; use tempfile::TempDir; @@ -297,12 +298,14 @@ mod published_artifact_ceiling { use tracedecay_runtime_core::db::{ RegisteredTestRuntimeRetirementControlV1, TestDatabaseRuntimeScope, }; + use tracedecay_runtime_core::logging::FormattedTracingCapture; use tracedecay_runtime_core::shard_runtime::registry::{ StoreRuntimeClientLease, StoreRuntimeRetirementOutcome, StoreRuntimeRetirementResult, }; use tracedecay_rusqlite_runtime::workflow::{ WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1, WORKFLOW_TABLE_CONTRACTS_V1, }; + use tracing::{Instrument, instrument::WithSubscriber}; use super::fs; use crate::RegisteredGlobalDbOwnerV1; @@ -431,6 +434,11 @@ mod published_artifact_ceiling { } impl WorkflowState { + #[tracing::instrument( + name = "global_db.workflow_sweep.read_state", + level = "info", + skip_all + )] fn read(path: &Path) -> Self { let connection = Connection::open(path).unwrap(); let table_sql = connection @@ -1010,6 +1018,36 @@ mod published_artifact_ceiling { /// stays typed recovering, and the next open finishes the job. #[tokio::test(flavor = "multi_thread")] async fn background_upgrade_is_atomic_at_every_cancellation_point() { + let subscriber = { + let _capture = FormattedTracingCapture::start_with_target_span_timings(&[ + module_path!(), + "tracedecay_global_db::registered", + "tracedecay_global_db::schema_stages", + ]); + tracing::dispatcher::get_default(Clone::clone) + }; + async { + let started_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + run_background_upgrade_cancellation_sweep() + .instrument(tracing::info_span!( + "global_db.workflow_sweep", + started_at_unix_ms = started_at.as_millis() as u64, + )) + .await; + } + .with_subscriber(subscriber) + .await; + } + + async fn run_background_upgrade_cancellation_sweep() { + #[tracing::instrument( + name = "global_db.workflow_sweep.sample", + level = "info", + skip_all, + fields(cancel_at = cancel_at) + )] async fn converge_cancelled_at( path: &Path, cancel_at: usize, @@ -1073,6 +1111,7 @@ mod published_artifact_ceiling { assert_eq!(completed.artifacts, before.artifacts); let stride = (total_polls / 12).max(1); + tracing::info!(total_polls, stride, "workflow cancellation sweep calibrated"); let mut cancelled_published = 0; // Each convergence joins its exact runtime retirement. Reuse the // canonically initialized store, restoring the upgrade's inputs and @@ -1080,6 +1119,8 @@ mod published_artifact_ceiling { let store = &reference.store; for cancel_at in (1..total_polls).step_by(stride) { { + let _restore = + tracing::info_span!("global_db.workflow_sweep.restore", cancel_at).entered(); let mut connection = Connection::open(&store.path).unwrap(); let transaction = connection.transaction().unwrap(); republish(&transaction, &before.table_sql, &before.digest); diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 94cba0b7b6c..b1272306069 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -15,6 +15,10 @@ use std::io::Write as _; use std::os::fd::{FromRawFd, RawFd}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Mutex, PoisonError}; +#[cfg(feature = "test-helpers")] +use tracing_subscriber::Layer; +#[cfg(feature = "test-helpers")] +use tracing_subscriber::layer::SubscriberExt; /// Opening marker of every bespoke daemon log line. Watcher recovery anchors /// on it, so `tracing` output, which never carries the marker, cannot forge @@ -289,6 +293,32 @@ impl FormattedTracingCapture { Self::start_with_output(true) } + /// Captures existing span timings for the selected targets on this thread, + /// mirroring each line to stderr so a terminated measurement retains it. + #[must_use] + pub fn start_with_target_span_timings(targets: &[&str]) -> Self { + let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); + let filter = targets.iter().fold( + tracing_subscriber::filter::Targets::new(), + |filter, target| filter.with_target(*target, tracing::Level::TRACE), + ); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .with_timer(tracing_subscriber::fmt::time::uptime()) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), true)) + .with_filter(filter), + ); + Self { + bytes, + _guard: set_tracing_capture(subscriber), + } + } + fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() From 1be43eab4ecb151622aced056d745a058ea4ed2f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:34:14 -0800 Subject: [PATCH 45/50] test(daemon): use the native provider fixture runtime Run the wall-exhaustion fixture through the existing on_installed_runtime helper used by the neighboring real-provider cases. The registered producer then inherits the canonical native host runtime. Keep the complete async body, real two-second execution budget, producer deadlines and every outcome and observability assertion unchanged. This aligns the fixture host and does not establish the cause of a prior persistence deadline. --- .../src/invocation/work_attempt_exec/tests.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs index 1bdc236b04e..22b640cbf24 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs @@ -1941,8 +1941,13 @@ async fn a_cancellation_requested_before_mark_running_seals_cancelled() { /// effect outcome. This test spends the real two-second wall on purpose, the /// stall must be a monotonic measurement, not a virtual-clock artifact. #[cfg(unix)] -#[tokio::test] -async fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_terminal() { +#[test] +fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_terminal() { + on_installed_runtime(verify_wall_exhausted_provider_terminal()); +} + +#[cfg(unix)] +async fn verify_wall_exhausted_provider_terminal() { let (_artifact_dir, workflow_artifacts) = provider_artifact_store().await; let profile = tempfile::TempDir::new().unwrap(); let directory = tempfile::TempDir::new().unwrap(); From 2a8f4807e1d271eb781a046157bb7aa899d863e1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:54:13 -0800 Subject: [PATCH 46/50] test(automation): remove completed settlement timing capture --- .../automation/effect_runtime/settlement.rs | 9 --------- crates/tracedecay-runtime-core/src/logging.rs | 19 +++---------------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs index 660a467bd01..65adf5e1f84 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs @@ -1717,15 +1717,6 @@ fn settle_bound_owner_with_budget( mut state: RetainedBoundSettlement, budget: Duration, ) -> Result> { - #[cfg(test)] - let _capture = state.phase_hook.as_ref().map(|_| { - tracedecay_runtime_core::logging::FormattedTracingCapture::start_with_span_timings() - }); - #[cfg(test)] - let _capture_span = state.phase_hook.as_ref().map(|_| { - tracing::info_span!("daemon.automation.retained_owner", run_id = %state.ledger.run_id) - .entered() - }); let started = std::time::Instant::now(); let mut delay = Duration::from_millis(25); loop { diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 701629d6f01..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -279,34 +279,21 @@ impl FormattedTracingCapture { /// before the first captured event. #[must_use] pub fn start() -> Self { - Self::start_with_output(false, false) + Self::start_with_output(false) } /// Also writes each line to stderr so a terminated measurement retains /// the events emitted before its deadline. #[must_use] pub fn start_with_stderr() -> Self { - Self::start_with_output(true, false) + Self::start_with_output(true) } - /// Captures span admission and completion timings as well as events, - /// retaining live stderr if the measurement is terminated before reporting. - #[must_use] - pub fn start_with_span_timings() -> Self { - Self::start_with_output(true, true) - } - - fn start_with_output(stderr: bool, span_timings: bool) -> Self { + fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_ansi(false) - .with_span_events(if span_timings { - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE - } else { - tracing_subscriber::fmt::format::FmtSpan::NONE - }) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { set_tracing_capture( From e4db3eb6cd8cf6aa4c1328c91191ee5222194a96 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:26:28 -0800 Subject: [PATCH 47/50] test(global-db): reuse cancellation sweep schema fixture --- .../src/registered/workflow_schema_tests.rs | 134 ++++++------------ crates/tracedecay-runtime-core/src/logging.rs | 26 +--- 2 files changed, 45 insertions(+), 115 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 02cc051ad26..c88c416a63e 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -287,7 +287,6 @@ mod published_artifact_ceiling { use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::Poll; - use std::time::{SystemTime, UNIX_EPOCH}; use rusqlite::Connection; use tempfile::TempDir; @@ -298,14 +297,12 @@ mod published_artifact_ceiling { use tracedecay_runtime_core::db::{ RegisteredTestRuntimeRetirementControlV1, TestDatabaseRuntimeScope, }; - use tracedecay_runtime_core::logging::FormattedTracingCapture; use tracedecay_runtime_core::shard_runtime::registry::{ StoreRuntimeClientLease, StoreRuntimeRetirementOutcome, StoreRuntimeRetirementResult, }; use tracedecay_rusqlite_runtime::workflow::{ WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1, WORKFLOW_TABLE_CONTRACTS_V1, }; - use tracing::{Instrument, instrument::WithSubscriber}; use super::fs; use crate::RegisteredGlobalDbOwnerV1; @@ -434,11 +431,6 @@ mod published_artifact_ceiling { } impl WorkflowState { - #[tracing::instrument( - name = "global_db.workflow_sweep.read_state", - level = "info", - skip_all - )] fn read(path: &Path) -> Self { let connection = Connection::open(path).unwrap(); let table_sql = connection @@ -1018,36 +1010,11 @@ mod published_artifact_ceiling { /// stays typed recovering, and the next open finishes the job. #[tokio::test(flavor = "multi_thread")] async fn background_upgrade_is_atomic_at_every_cancellation_point() { - let sweep = { - let _capture = FormattedTracingCapture::start_with_info_span_timings(); - let started_at = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default(); - let span = tracing::info_span!( - "global_db.workflow_sweep", - started_at_unix_ms = started_at.as_millis() as u64, - ); - run_background_upgrade_cancellation_sweep() - .instrument(span) - .with_current_subscriber() - }; - sweep.await; - } - - async fn run_background_upgrade_cancellation_sweep() { - #[tracing::instrument( - name = "global_db.workflow_sweep.sample", - level = "info", - skip_all, - fields(cancel_at) - )] async fn converge_cancelled_at( path: &Path, cancel_at: usize, ) -> (Result<(), TraceDecayError>, usize) { - let (owner, convergence, (runtime, retirement)) = admit_for_daemon(path) - .instrument(tracing::info_span!("global_db.workflow_sweep.admit")) - .await; + let (owner, convergence, (runtime, retirement)) = admit_for_daemon(path).await; let lease = owner.issue_lease().unwrap(); let cancellation = CancellationToken::new(); let mut recovery = Box::pin(lease.converge_schema(convergence, &cancellation)); @@ -1059,41 +1026,29 @@ mod published_artifact_ceiling { } recovery.as_mut().poll(context) }) - .instrument(tracing::info_span!("global_db.workflow_sweep.converge")) .await; drop(recovery); if let Err(error) = &outcome && error.is_store_open_cancelled() && WorkflowState::read(path).digest == PUBLISHED_DIGEST { - assert_workflow_recovering(&lease) - .instrument(tracing::info_span!( - "global_db.workflow_sweep.recovering_gate" - )) - .await; + assert_workflow_recovering(&lease).await; } // Each sampled cancellation reopens this path. Join the exact // published runtime before the next admission can acquire it. drop((lease, owner, runtime)); - let mut reservation = { - let _span = - tracing::info_span!("global_db.workflow_sweep.reserve_retirement").entered(); - match retirement - .registry() - .reserve_retirement_batch(vec![retirement.retirement_target()]) - { - StoreRuntimeRetirementResult::Reserved(reservation) => reservation, - StoreRuntimeRetirementResult::Blocked(refusal) => { - panic!("completed convergence retained runtime clients: {refusal:?}") - } + let mut reservation = match retirement + .registry() + .reserve_retirement_batch(vec![retirement.retirement_target()]) + { + StoreRuntimeRetirementResult::Reserved(reservation) => reservation, + StoreRuntimeRetirementResult::Blocked(refusal) => { + panic!("completed convergence retained runtime clients: {refusal:?}") } }; let committed = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { reservation.commit() }) - .instrument(tracing::info_span!( - "global_db.workflow_sweep.join_retirement" - )) .await .unwrap() .unwrap(); @@ -1108,42 +1063,49 @@ mod published_artifact_ceiling { (outcome, polls) } - let reference = DaemonPublishedStore::create() - .instrument(tracing::info_span!( - "global_db.workflow_sweep.reference_fixture" - )) - .await; - tracing::info_span!("global_db.workflow_sweep.reference_seed") - .in_scope(|| reference.store.run(BULK_ROWS)); - let before = tracing::info_span!("global_db.workflow_sweep.reference_before") - .in_scope(|| reference.store.state()); + let reference = DaemonPublishedStore::create().await; + reference.store.run(BULK_ROWS); + let before = reference.store.state(); let (outcome, total_polls) = converge_cancelled_at(&reference.store.path, usize::MAX).await; outcome.unwrap(); - let completed = tracing::info_span!("global_db.workflow_sweep.reference_after") - .in_scope(|| reference.store.state()); + let completed = reference.store.state(); assert_eq!(completed.digest, WORKFLOW_SCHEMA_DEFINITION_DIGEST_V1); assert_eq!(completed.artifacts, before.artifacts); let stride = (total_polls / 12).max(1); - tracing::info!( - total_polls, - stride, - "workflow cancellation sweep calibrated" - ); let mut cancelled_published = 0; + // Each convergence joins its exact runtime retirement. Reuse the + // canonically initialized store, restoring the upgrade's inputs and + // proving the entire workflow state before admitting a fresh runtime. + let store = &reference.store; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create() - .instrument(tracing::info_span!( - "global_db.workflow_sweep.fixture", - cancel_at - )) - .await; - let store = &daemon_store.store; - tracing::info_span!("global_db.workflow_sweep.seed", cancel_at) - .in_scope(|| store.run(BULK_ROWS)); + { + let mut connection = Connection::open(&store.path).unwrap(); + let transaction = connection.transaction().unwrap(); + republish(&transaction, &before.table_sql, &before.digest); + { + let mut insert = transaction + .prepare( + "INSERT INTO workflow_artifact_payloads + (rowid, payload_digest, byte_length, payload) + VALUES (?1, ?2, ?3, ?4)", + ) + .unwrap(); + for (rowid, digest, byte_length, payload) in &before.artifacts { + insert + .execute(rusqlite::params![rowid, digest, byte_length, payload]) + .unwrap(); + } + } + transaction.commit().unwrap(); + } + assert_eq!( + store.state(), + before, + "poll {cancel_at} must start from the published workflow state" + ); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; - let state = tracing::info_span!("global_db.workflow_sweep.state", cancel_at) - .in_scope(|| store.state()); + let state = store.state(); match outcome { Ok(()) => assert_eq!(state, completed, "completed at poll {cancel_at}"), Err(error) => { @@ -1156,15 +1118,7 @@ mod published_artifact_ceiling { cancelled_published += 1; let (reopened, _) = converge_cancelled_at(&store.path, usize::MAX).await; reopened.unwrap(); - assert_eq!( - tracing::info_span!( - "global_db.workflow_sweep.reopened_state", - cancel_at - ) - .in_scope(|| store.state()), - completed, - "reopen after poll {cancel_at}" - ); + assert_eq!(store.state(), completed, "reopen after poll {cancel_at}"); } } } diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index d7acb54d0b4..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -289,34 +289,10 @@ impl FormattedTracingCapture { Self::start_with_output(true) } - /// Records high-level phase boundaries without verbose SQL trace events. - #[must_use] - pub fn start_with_info_span_timings() -> Self { - Self::start_with_settings( - true, - tracing::Level::INFO, - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - } - fn start_with_output(stderr: bool) -> Self { - Self::start_with_settings( - stderr, - tracing::Level::TRACE, - tracing_subscriber::fmt::format::FmtSpan::NONE, - ) - } - - fn start_with_settings( - stderr: bool, - level: tracing::Level, - span_events: tracing_subscriber::fmt::format::FmtSpan, - ) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() - .with_max_level(level) - .with_span_events(span_events) + .with_max_level(tracing::Level::TRACE) .with_ansi(false) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { From ec40c73fed9a96daeb50c930fe5b8f11ee280211 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:37:07 -0800 Subject: [PATCH 48/50] test(runtime): trace scripted daemon startup phases --- crates/tracedecay-runtime-core/src/logging.rs | 30 ++++++++++++++++ .../tool_client_transport.rs | 36 ++++++++++++++++--- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 94cba0b7b6c..b1272306069 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -15,6 +15,10 @@ use std::io::Write as _; use std::os::fd::{FromRawFd, RawFd}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Mutex, PoisonError}; +#[cfg(feature = "test-helpers")] +use tracing_subscriber::Layer; +#[cfg(feature = "test-helpers")] +use tracing_subscriber::layer::SubscriberExt; /// Opening marker of every bespoke daemon log line. Watcher recovery anchors /// on it, so `tracing` output, which never carries the marker, cannot forge @@ -289,6 +293,32 @@ impl FormattedTracingCapture { Self::start_with_output(true) } + /// Captures existing span timings for the selected targets on this thread, + /// mirroring each line to stderr so a terminated measurement retains it. + #[must_use] + pub fn start_with_target_span_timings(targets: &[&str]) -> Self { + let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); + let filter = targets.iter().fold( + tracing_subscriber::filter::Targets::new(), + |filter, target| filter.with_target(*target, tracing::Level::TRACE), + ); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .with_timer(tracing_subscriber::fmt::time::uptime()) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), true)) + .with_filter(filter), + ); + Self { + bytes, + _guard: set_tracing_capture(subscriber), + } + } + fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs b/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs index c94b3bc22ef..4247a2d8fad 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs @@ -23,6 +23,7 @@ use tracedecay_daemon_protocol::{ DaemonInvocationResponse, }; use tracedecay_domain::ProjectId; +use tracedecay_runtime_core::logging::FormattedTracingCapture; use tracedecay_tool_catalog::ApplicationSurfaceOperation; const LOCAL_TIMEOUT: Duration = Duration::from_secs(5); @@ -276,19 +277,44 @@ where let (ready_tx, ready_rx) = mpsc::channel(); let (request_tx, request_rx) = mpsc::channel(); let script = Arc::new(script); + let test_name = std::thread::current() + .name() + .unwrap_or("unnamed test") + .to_owned(); let server = std::thread::spawn(move || { - let _ = std::fs::remove_file(&socket); + let capture = FormattedTracingCapture::start_with_target_span_timings(&[ + module_path!(), + "tracedecay_daemon_identity::authority", + "tracedecay_daemon_identity::profile_identity", + "tracedecay_private_fs", + ]); + let startup = tracing::info_span!( + "runtime.tool_transport.scripted_daemon.startup", + test = %test_name, + ) + .entered(); + tracing::info_span!("runtime.tool_transport.scripted_daemon.remove_socket").in_scope( + || { + let _ = std::fs::remove_file(&socket); + }, + ); let authority = DaemonAuthority::acquire( socket.parent().expect("socket parent"), &DaemonEndpoint::Unix(socket.clone()), env!("CARGO_PKG_VERSION"), ) .expect("seed fake daemon authority"); - let listener = UnixListener::bind(&socket).expect("bind fake daemon"); - listener - .set_nonblocking(true) - .expect("nonblocking listener"); + let listener = tracing::info_span!("runtime.tool_transport.scripted_daemon.bind_socket") + .in_scope(|| { + let listener = UnixListener::bind(&socket).expect("bind fake daemon"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); + listener + }); ready_tx.send(()).expect("signal daemon ready"); + drop(startup); + drop(capture); let accept_deadline = Instant::now() + ACCEPT_HANG_GUARD; let mut workers = Vec::new(); while workers.len() < connections { From 1081212c60f4a30af74be292facddb51b5134e38 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:37:12 -0800 Subject: [PATCH 49/50] chore(runtime): remove temporary daemon startup tracing --- crates/tracedecay-runtime-core/src/logging.rs | 30 ---------------- .../tool_client_transport.rs | 36 +++---------------- 2 files changed, 5 insertions(+), 61 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index b1272306069..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -15,10 +15,6 @@ use std::io::Write as _; use std::os::fd::{FromRawFd, RawFd}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Mutex, PoisonError}; -#[cfg(feature = "test-helpers")] -use tracing_subscriber::Layer; -#[cfg(feature = "test-helpers")] -use tracing_subscriber::layer::SubscriberExt; /// Opening marker of every bespoke daemon log line. Watcher recovery anchors /// on it, so `tracing` output, which never carries the marker, cannot forge @@ -293,32 +289,6 @@ impl FormattedTracingCapture { Self::start_with_output(true) } - /// Captures existing span timings for the selected targets on this thread, - /// mirroring each line to stderr so a terminated measurement retains it. - #[must_use] - pub fn start_with_target_span_timings(targets: &[&str]) -> Self { - let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); - let filter = targets.iter().fold( - tracing_subscriber::filter::Targets::new(), - |filter, target| filter.with_target(*target, tracing::Level::TRACE), - ); - let subscriber = tracing_subscriber::registry().with( - tracing_subscriber::fmt::layer() - .with_ansi(false) - .with_timer(tracing_subscriber::fmt::time::uptime()) - .with_span_events( - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), true)) - .with_filter(filter), - ); - Self { - bytes, - _guard: set_tracing_capture(subscriber), - } - } - fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs b/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs index 4247a2d8fad..c94b3bc22ef 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/tool_client_transport.rs @@ -23,7 +23,6 @@ use tracedecay_daemon_protocol::{ DaemonInvocationResponse, }; use tracedecay_domain::ProjectId; -use tracedecay_runtime_core::logging::FormattedTracingCapture; use tracedecay_tool_catalog::ApplicationSurfaceOperation; const LOCAL_TIMEOUT: Duration = Duration::from_secs(5); @@ -277,44 +276,19 @@ where let (ready_tx, ready_rx) = mpsc::channel(); let (request_tx, request_rx) = mpsc::channel(); let script = Arc::new(script); - let test_name = std::thread::current() - .name() - .unwrap_or("unnamed test") - .to_owned(); let server = std::thread::spawn(move || { - let capture = FormattedTracingCapture::start_with_target_span_timings(&[ - module_path!(), - "tracedecay_daemon_identity::authority", - "tracedecay_daemon_identity::profile_identity", - "tracedecay_private_fs", - ]); - let startup = tracing::info_span!( - "runtime.tool_transport.scripted_daemon.startup", - test = %test_name, - ) - .entered(); - tracing::info_span!("runtime.tool_transport.scripted_daemon.remove_socket").in_scope( - || { - let _ = std::fs::remove_file(&socket); - }, - ); + let _ = std::fs::remove_file(&socket); let authority = DaemonAuthority::acquire( socket.parent().expect("socket parent"), &DaemonEndpoint::Unix(socket.clone()), env!("CARGO_PKG_VERSION"), ) .expect("seed fake daemon authority"); - let listener = tracing::info_span!("runtime.tool_transport.scripted_daemon.bind_socket") - .in_scope(|| { - let listener = UnixListener::bind(&socket).expect("bind fake daemon"); - listener - .set_nonblocking(true) - .expect("nonblocking listener"); - listener - }); + let listener = UnixListener::bind(&socket).expect("bind fake daemon"); + listener + .set_nonblocking(true) + .expect("nonblocking listener"); ready_tx.send(()).expect("signal daemon ready"); - drop(startup); - drop(capture); let accept_deadline = Instant::now() + ACCEPT_HANG_GUARD; let mut workers = Vec::new(); while workers.len() < connections { From ef0e8c2b8b976ad0ce464b26d71045bf686fe537 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:59:33 -0800 Subject: [PATCH 50/50] test(global-db): avoid unused schema fixture actor startup --- .../src/tests/lcm_schema/mod.rs | 7 +++- .../src/tests/lcm_schema/temporal_catalog.rs | 18 ++++------ .../lcm_schema/temporal_catalog/admission.rs | 33 +++++++------------ .../tests/lcm_schema/temporal_constraints.rs | 15 +++------ .../src/tests/lcm_schema/temporal_cursor.rs | 15 +++------ 5 files changed, 33 insertions(+), 55 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 22a7d3611c0..fbfe8b5f1f8 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -15,7 +15,7 @@ use tracedecay_runtime_core::db::{ use crate::tests::harness::open_registered_test_database_fixture; -async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { +async fn initialize_global_db(db_path: &Path) -> tracedecay_domain::errors::Result<()> { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent)?; } @@ -23,6 +23,11 @@ async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result tracedecay_domain::errors::Result { + initialize_global_db(db_path).await?; Ok(TestConnection::open(db_path)) } diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 4dcc84fa725..1d2cb2f65d2 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -63,10 +63,9 @@ async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { async fn temporal_payload_manifest_schema_is_payload_global() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -175,10 +174,9 @@ mod admission; async fn temporal_schema_query_indexes_cover_exact_lookup_shapes() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -305,10 +303,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -608,10 +605,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh async fn temporal_schema_rejects_unexpected_indexes_without_dropping_them() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -719,10 +715,9 @@ async fn temporal_schema_refuses_markerless_malformed_fts_without_mutation() { async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -791,10 +786,9 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { async fn temporal_schema_rejects_missing_graph_publication_authority_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index a5ba897a0e1..c1ec89d7d12 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -285,10 +285,9 @@ async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_cont async fn temporal_schema_rejects_transition_storage_without_mutating_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -332,10 +331,9 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -383,10 +381,9 @@ async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutati async fn final_store_missing_temporal_marker_requires_reset_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive every final authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -556,10 +553,9 @@ async fn writer_ledger_does_not_hide_a_nonempty_unmarked_store() { async fn final_schema_admission_rejects_extra_temporal_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -598,10 +594,9 @@ async fn final_schema_admission_rejects_extra_temporal_column_metadata() { async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal indexes"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -635,10 +630,9 @@ async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() async fn final_schema_admission_rejects_missing_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -670,10 +664,9 @@ async fn final_schema_admission_rejects_missing_graph_publication_table() { async fn final_schema_admission_rejects_incompatible_graph_publication_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -712,10 +705,9 @@ async fn final_schema_admission_rejects_incompatible_graph_publication_column_me async fn final_schema_admission_rejects_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -753,10 +745,9 @@ async fn final_schema_admission_rejects_extra_graph_publication_table() { async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -794,10 +785,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -835,10 +825,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index b9c1c52d30f..17c54837976 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -71,10 +71,9 @@ async fn temporal_schema_rejects_cross_session_and_generation_rows() { async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -258,10 +257,9 @@ async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -365,10 +363,9 @@ async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1160,10 +1157,9 @@ async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { async fn temporal_schema_enforces_generation_state_machine_and_durability() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1271,10 +1267,9 @@ async fn temporal_schema_enforces_generation_state_machine_and_durability() { async fn temporal_schema_keeps_append_only_authority_immutable() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index a1f6398e1b2..ad69571ce89 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -87,10 +87,9 @@ async fn temporal_schema_persists_cursor_keys_without_read_creation() { async fn temporal_schema_rejects_direct_cursor_retirement() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -121,10 +120,9 @@ async fn temporal_schema_rejects_direct_cursor_retirement() { async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -250,10 +248,9 @@ async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -304,10 +301,9 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -362,10 +358,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone();