diff --git a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs index 1804333d454..b2cc1e14f86 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs @@ -69,6 +69,16 @@ use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; const OBSERVED_AT: UtcMicros = UtcMicros(100); const EXPIRES_AT: UtcMicros = UtcMicros(10_000); +fn run_native_journey(journey: impl Future) { + // Native owners reuse the ambient runtime. Match the daemon's scheduling + // policy so a wake issued during a long owner poll stays stealable by + // another worker instead of being held in Tokio's default LIFO slot. + native_runtime::default_builder() + .build() + .expect("native journey runtime") + .block_on(journey); +} + struct UnexpectedAnalysis; impl NativeIntegrationAnalysisPort for UnexpectedAnalysis { @@ -760,8 +770,12 @@ async fn stack_snapshot( selection.as_ref().clone() } -#[tokio::test(flavor = "multi_thread")] -async fn declared_stack_runtime_shares_native_preview_and_failure_without_consuming_result() { +#[test] +fn declared_stack_runtime_shares_native_preview_and_failure_without_consuming_result() { + run_native_journey(shared_native_preview_and_failure_journey()); +} + +async fn shared_native_preview_and_failure_journey() { for fail in [false, true] { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); @@ -928,8 +942,12 @@ async fn declared_stack_runtime_shares_native_preview_and_failure_without_consum } } -#[tokio::test(flavor = "multi_thread")] -async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { +#[test] +fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { + run_native_journey(checked_out_stack_conflict_journey()); +} + +async fn checked_out_stack_conflict_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); let source_root = directory.path().join("dependency"); @@ -1392,8 +1410,12 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { code_index.shutdown().await; } -#[tokio::test(flavor = "multi_thread")] -async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() { +#[test] +fn independent_pair_applies_supported_modes_and_survives_daemon_restart() { + run_native_journey(independent_pair_restart_journey()); +} + +async fn independent_pair_restart_journey() { for (index, mode) in [ MechanicalIntegrationModeV1::FastForward, MechanicalIntegrationModeV1::TwoParentMerge, @@ -1773,8 +1795,12 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() } } -#[tokio::test(flavor = "multi_thread")] -async fn generated_only_change_requires_semantic_review() { +#[test] +fn generated_only_change_requires_semantic_review() { + run_native_journey(generated_only_change_journey()); +} + +async fn generated_only_change_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); std::fs::create_dir_all(&repository_root).expect("repository root"); @@ -1818,8 +1844,12 @@ async fn generated_only_change_requires_semantic_review() { code_index.shutdown().await; } -#[tokio::test(flavor = "multi_thread")] -async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_tip() { +#[test] +fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_tip() { + run_native_journey(foreign_destination_ref_drift_journey()); +} + +async fn foreign_destination_ref_drift_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); std::fs::create_dir_all(&repository_root).expect("repository root"); @@ -1920,8 +1950,12 @@ async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_t /// Cleanup inspection of an owned checkout, asked while another thread walks /// that checkout, classifies it from the shared walk. Reading the walk as /// "no repository" classified the checkout as foreign. -#[tokio::test(flavor = "multi_thread")] -async fn owned_worktree_inspection_shares_the_discovery_walk_another_thread_owns() { +#[test] +fn owned_worktree_inspection_shares_the_discovery_walk_another_thread_owns() { + run_native_journey(shared_worktree_discovery_journey()); +} + +async fn shared_worktree_discovery_journey() { let directory = tempfile::tempdir().expect("temporary project directory"); let repository_root = directory.path().join("repo"); std::fs::create_dir_all(&repository_root).expect("repository root"); diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs index 660a467bd01..65adf5e1f84 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs @@ -1717,15 +1717,6 @@ fn settle_bound_owner_with_budget( mut state: RetainedBoundSettlement, budget: Duration, ) -> Result> { - #[cfg(test)] - let _capture = state.phase_hook.as_ref().map(|_| { - tracedecay_runtime_core::logging::FormattedTracingCapture::start_with_span_timings() - }); - #[cfg(test)] - let _capture_span = state.phase_hook.as_ref().map(|_| { - tracing::info_span!("daemon.automation.retained_owner", run_id = %state.ledger.run_id) - .entered() - }); let started = std::time::Instant::now(); let mut delay = Duration::from_millis(25); loop { diff --git a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs index 1bdc236b04e..22b640cbf24 100644 --- a/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs +++ b/crates/tracedecay-daemon-service/src/invocation/work_attempt_exec/tests.rs @@ -1941,8 +1941,13 @@ async fn a_cancellation_requested_before_mark_running_seals_cancelled() { /// effect outcome. This test spends the real two-second wall on purpose, the /// stall must be a monotonic measurement, not a virtual-clock artifact. #[cfg(unix)] -#[tokio::test] -async fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_terminal() { +#[test] +fn a_wall_exhausted_provider_seals_timed_out_and_emits_the_no_progress_terminal() { + on_installed_runtime(verify_wall_exhausted_provider_terminal()); +} + +#[cfg(unix)] +async fn verify_wall_exhausted_provider_terminal() { let (_artifact_dir, workflow_artifacts) = provider_artifact_store().await; let profile = tempfile::TempDir::new().unwrap(); let directory = tempfile::TempDir::new().unwrap(); diff --git a/crates/tracedecay-global-db/src/git_index_transactions/database.rs b/crates/tracedecay-global-db/src/git_index_transactions/database.rs index 51a3ce5909a..831a54e82c0 100644 --- a/crates/tracedecay-global-db/src/git_index_transactions/database.rs +++ b/crates/tracedecay-global-db/src/git_index_transactions/database.rs @@ -59,6 +59,15 @@ impl Executor for GitMutationWriteTransaction<'_> { } } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + match self { + Self::Registered(transaction) => transaction.execute_statements(statements).await, + } + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { match self { Self::Registered(transaction) => transaction.execute_batch(sql).await, diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs index 6c5af50fea3..5ecc001555b 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc.rs @@ -6,12 +6,16 @@ //! bootstrap frontiers are complete. A candidate and its indexed references are //! checked under one guarded writer, so another owner cannot race collection. +use std::time::Instant; + use serde::{Deserialize, Serialize}; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::db::Database; -use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; +use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, WriteStatement, params}; use tracedecay_store::collect_body_refs; +#[cfg(test)] +mod bootstrap_write_tests; pub(crate) mod schema; #[cfg(test)] mod tests; @@ -25,6 +29,15 @@ pub const MAX_CANONICAL_BODY_GC_ROWS: usize = 48; pub struct CanonicalBodyGcReport { pub roots_scanned: u64, pub bodies_scanned: u64, + /// Wall-clock duration from writer admission through commit, not CPU time. + pub elapsed_ms: Option, + /// Candidates discharged in the committed page, even if their body stays live. + pub candidates_discharged: u64, + /// Phase-qualified SQL bootstrap cursor before the committed page. + /// The value is absent during candidate collection and idle ticks. + pub bootstrap_frontier_before: Option, + /// The corresponding persisted cursor, verified in the same writer. + pub bootstrap_frontier_after: Option, pub bodies_deleted: u64, /// Stored BLOB bytes deleted, including the effect of their existing /// compression. This is not database-file or filesystem space reclaimed. @@ -34,6 +47,16 @@ pub struct CanonicalBodyGcReport { pub has_more: bool, } +impl CanonicalBodyGcReport { + /// A bootstrap page can commit its remaining frontier after every owner in it was deleted. + pub fn has_committed_work(&self) -> bool { + self.roots_scanned != 0 + || self.bodies_scanned != 0 + || self.bootstrap_frontier_after.is_some() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] struct Progress { raw_through: i64, raw_until: i64, @@ -118,15 +141,37 @@ async fn save_progress(conn: &(impl Executor + ?Sized), progress: &Progress) -> if changed != 1 { return Err(integrity_error("canonical body ownership frontier changed")); } + // The UPDATE can report one affected row even if a database trigger + // rewrites the checkpoint. Verify the durable value before committing the + // references or candidate seed that depended on this bootstrap page. + if read_progress(conn).await? != *progress { + return Err(integrity_error( + "canonical body ownership bootstrap frontier was not persisted", + )); + } Ok(()) } #[tracing::instrument(name = "global_db.canonical_bodies.gc", level = "trace", skip_all)] pub(crate) async fn run(database: &Database, max_rows: usize) -> Result { + let started = Instant::now(); + // During another retention continuation an already-drained store must not + // repeatedly acquire the SQLite writer merely to observe an empty queue. + // Do not bypass the schema authority: an idle report is safe only after + // verifying both the completed bootstrap and its installed objects. + let reader = database.read_connection(); + if read_progress(&reader).await?.complete() && !has_candidates(&reader).await? { + schema::validate(&reader).await?; + return Ok(CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + }); + } let transaction = database.begin_write_transaction(OPERATION).await?; match step(&transaction, max_rows).await { - Ok(report) => { + Ok(mut report) => { transaction.commit().await?; + report.elapsed_ms = Some(started.elapsed().as_millis()); Ok(report) } Err(error) => match transaction.rollback().await { @@ -144,14 +189,34 @@ async fn step(conn: &(impl Executor + ?Sized), max_rows: usize) -> Result Result { let mut rows = conn .query( - "SELECT store_id, CASE WHEN storage_kind = 'inline' AND content IS NULL - THEN content_hash END - FROM lcm_raw_messages WHERE store_id > ?1 AND store_id <= ?2 - ORDER BY store_id LIMIT ?3", + "SELECT store_id FROM lcm_raw_messages + WHERE store_id > ?1 AND store_id <= ?2 ORDER BY store_id LIMIT ?3", params![progress.raw_through, progress.raw_until, limit], ) .await .map_err(db_error)?; let mut page = Vec::new(); while let Some(row) = rows.next().await.map_err(db_error)? { - page.push(( - row.get::(0).map_err(db_error)?, - row.get::>(1).map_err(db_error)?, - )); + page.push(row.get::(0).map_err(db_error)?); } drop(rows); - for (owner, hash) in &page { - if let Some(hash) = hash { - conn.execute( - "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) - VALUES ('raw', ?1, ?2)", - params![*owner, hash.as_str()], + if let Some(&last) = page.last() { + // Both reads share the writer transaction. The last selected identity + // bounds this indexed insert to the same page, including sparse IDs, + // without one worker submission for every referenced owner. + conn.execute( + "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + SELECT 'raw', store_id, content_hash FROM lcm_raw_messages + WHERE store_id > ?1 AND store_id <= ?2 + AND storage_kind = 'inline' AND content IS NULL", + params![progress.raw_through, last], + ) + .await + .map_err(db_error)?; + // A suppressed INSERT must not advance bootstrap past an owner whose + // canonical body would subsequently appear unreferenced. + let mut missing = conn + .query( + "SELECT 1 FROM lcm_raw_messages raw + WHERE raw.store_id > ?1 AND raw.store_id <= ?2 + AND raw.storage_kind = 'inline' AND raw.content IS NULL + AND NOT EXISTS ( + SELECT 1 FROM canonical_body_gc_refs r + WHERE r.owner_kind = 'raw' AND r.owner_id = raw.store_id + AND r.content_hash = raw.content_hash + ) LIMIT 1", + params![progress.raw_through, last], ) .await .map_err(db_error)?; + let incomplete = missing.next().await.map_err(db_error)?.is_some(); + drop(missing); + if incomplete { + return Err(integrity_error( + "canonical raw bootstrap reference was not indexed", + )); } - progress.raw_through = *owner; + progress.raw_through = last; } if page.len() < limit as usize { progress.raw_through = progress.raw_until; @@ -226,17 +312,60 @@ async fn bootstrap_observations( )); } drop(rows); + // Parsing preserves the canonical Rust reference contract, including + // value deduplication. Submit validated writes in bounded groups so an + // observation page cannot schedule one storage-worker hop per body hash. + // The caller's writer transaction rolls back every group on any failure. + const MAX_PENDING_OBSERVATION_REFS: usize = 128; + let mut pending = Vec::new(); + let mut expected = Vec::new(); for (owner, json) in &page { for hash in collect_body_refs(json).map_err(db_error)? { - conn.execute( - "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) - VALUES ('observation', ?1, ?2)", - params![*owner, hash], + expected.push((*owner, hash.clone())); + pending.push( + WriteStatement::new( + "INSERT OR IGNORE INTO canonical_body_gc_refs(owner_kind, owner_id, content_hash) + VALUES ('observation', ?1, ?2)", + params![*owner, hash], + ) + .map_err(db_error)?, + ); + if pending.len() == MAX_PENDING_OBSERVATION_REFS { + conn.execute_statements(std::mem::take(&mut pending)) + .await + .map_err(db_error)?; + } + } + progress.observations_through = *owner; + } + if !pending.is_empty() { + conn.execute_statements(pending).await.map_err(db_error)?; + } + if !expected.is_empty() { + // Verify the Rust-validated reference set, not a second parse of raw + // observation JSON. INSERT OR IGNORE and SQLite trigger RAISE(IGNORE) + // can both report success without creating a missing reference. + let expected_json = serde_json::to_string(&expected).map_err(db_error)?; + let mut missing = conn + .query( + "SELECT 1 FROM json_each(?1) pair + WHERE NOT EXISTS ( + SELECT 1 FROM canonical_body_gc_refs r + WHERE r.owner_kind = 'observation' + AND r.owner_id = json_extract(pair.value, '$[0]') + AND r.content_hash = json_extract(pair.value, '$[1]') + ) LIMIT 1", + params![expected_json], ) .await .map_err(db_error)?; + let incomplete = missing.next().await.map_err(db_error)?.is_some(); + drop(missing); + if incomplete { + return Err(integrity_error( + "canonical observation bootstrap reference was not indexed", + )); } - progress.observations_through = *owner; } if page.len() < limit as usize { progress.observations_through = progress.observations_until; @@ -266,14 +395,39 @@ async fn bootstrap_bodies( page.push(row.get::(0).map_err(db_error)?); } drop(rows); - for hash in &page { + if let Some(last) = page.last() { + // Seed the selected key range in one statement, not one asynchronous + // writer submission per body. No body payload is materialized here. conn.execute( - "INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) VALUES (?1)", - params![hash.as_str()], + "INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) + SELECT content_hash FROM session_canonical_bodies + WHERE content_hash > ?1 AND content_hash <= ?2", + params![progress.bodies_after.as_str(), last.as_str()], ) .await .map_err(db_error)?; - progress.bodies_after.clone_from(hash); + // A vetoed candidate insert otherwise leaves an orphaned body behind + // the permanently committed bootstrap cursor, with no future seed. + let mut missing = conn + .query( + "SELECT 1 FROM session_canonical_bodies b + WHERE b.content_hash > ?1 AND b.content_hash <= ?2 + AND NOT EXISTS ( + SELECT 1 FROM canonical_body_gc_candidates c + WHERE c.content_hash = b.content_hash + ) LIMIT 1", + params![progress.bodies_after.as_str(), last.as_str()], + ) + .await + .map_err(db_error)?; + let incomplete = missing.next().await.map_err(db_error)?.is_some(); + drop(missing); + if incomplete { + return Err(integrity_error( + "canonical body bootstrap candidate was not indexed", + )); + } + progress.bodies_after.clone_from(last); } if page.len() < limit as usize { progress.bodies_after.clone_from(&progress.bodies_until); @@ -321,15 +475,65 @@ async fn collect_candidates( report.stored_bytes_deleted = report.stored_bytes_deleted.saturating_add(bytes); } drop(deleted); + // An ignored or otherwise suppressed body DELETE must not silently + // discharge an unreferenced candidate. Once that candidate disappears, + // no owner-triggered re-enqueue is guaranteed, and the body can persist + // indefinitely without another collection attempt. + let mut uncollected = conn + .query( + "SELECT 1 FROM session_canonical_bodies b + WHERE b.content_hash IN (SELECT value FROM json_each(?1)) + AND NOT EXISTS (SELECT 1 FROM canonical_body_gc_refs r + WHERE r.content_hash = b.content_hash) + LIMIT 1", + params![hashes.as_str()], + ) + .await + .map_err(db_error)?; + if uncollected.next().await.map_err(db_error)?.is_some() { + return Err(integrity_error( + "canonical body collection left an unreferenced body behind", + )); + } + drop(uncollected); // Referenced candidates are discharged too. The final owner removal // records them again; idle ticks never revisit every retained body. - conn.execute( - "DELETE FROM canonical_body_gc_candidates - WHERE content_hash IN (SELECT value FROM json_each(?1))", - params![hashes], - ) - .await - .map_err(db_error)?; + // Deleting an unreferenced body cascades to its candidate row. The + // remaining explicit removals discharge referenced bodies that stay live. + let explicitly_discharged = conn + .execute( + "DELETE FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1))", + params![hashes.as_str()], + ) + .await + .map_err(db_error)?; + // A DELETE row count is not a durable discharge receipt: an AFTER DELETE + // trigger may have re-enqueued the very same candidates. Refuse the page + // under this writer instead of recommitting an identical 48-row batch. + let mut remaining = conn + .query( + "SELECT 1 FROM canonical_body_gc_candidates + WHERE content_hash IN (SELECT value FROM json_each(?1)) LIMIT 1", + params![hashes.as_str()], + ) + .await + .map_err(db_error)?; + if remaining.next().await.map_err(db_error)?.is_some() { + return Err(integrity_error( + "canonical body candidate discharge was not persisted", + )); + } + drop(remaining); + let discharged = explicitly_discharged + .checked_add(report.bodies_deleted) + .ok_or_else(|| integrity_error("canonical body candidate discharge overflow"))?; + if discharged != report.bodies_scanned { + return Err(integrity_error( + "canonical body candidate page was not fully discharged", + )); + } + report.candidates_discharged = discharged; Ok(()) } diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs new file mode 100644 index 00000000000..802c2488043 --- /dev/null +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/bootstrap_write_tests.rs @@ -0,0 +1,715 @@ +//! Measure real bootstrap write submissions without replacing the SQLite executor. +//! +//! Every page runs inside the production `begin_write_transaction` wrapper, so +//! the counts cover the facade the daemon actually uses. Batching inside that +//! facade is not caller-observable: a per-statement fallback expands behind the +//! adapter's own method. `Executor::execute_statements` therefore has no +//! default, and an adapter that does not forward to the engine's batched +//! submission fails to compile. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use tracedecay_runtime_core::db::engine::{ + Executor, IntoParams, QueryExecutor, Result as EngineResult, Rows, WriteStatement, params, +}; +use tracedecay_runtime_core::db::{ + Database, DatabaseAuthority, DatabaseWriteTransaction, TestDatabaseRuntimeMode, +}; +use tracedecay_store::{ + CANONICAL_BODIES_TABLE_SQL, StoredCanonicalBody, UPSERT_CANONICAL_BODY_SQL, +}; + +use super::{CanonicalBodyGcReport, read_progress, schema, step}; + +struct CountWrites<'a, E: ?Sized> { + inner: &'a E, + submissions: AtomicUsize, +} + +impl QueryExecutor for CountWrites<'_, E> { + async fn query

(&self, sql: &str, params: P) -> EngineResult + where + P: IntoParams, + { + self.inner.query(sql, params).await + } +} + +impl Executor for CountWrites<'_, E> { + async fn execute

(&self, sql: &str, params: P) -> EngineResult + where + P: IntoParams, + { + self.submissions.fetch_add(1, Ordering::Relaxed); + self.inner.execute(sql, params).await + } + + async fn execute_statements(&self, statements: Vec) -> EngineResult> { + self.submissions.fetch_add(1, Ordering::Relaxed); + self.inner.execute_statements(statements).await + } + + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { + self.submissions.fetch_add(1, Ordering::Relaxed); + self.inner.execute_batch(sql).await + } +} + +struct Fixture { + database: Database, + bodies: Vec, + _authority: DatabaseAuthority, + _directory: tempfile::TempDir, +} + +impl Fixture { + async fn open() -> Self { + Self::with_observations(false).await + } + + async fn with_observations(include_observations: bool) -> Self { + // Publishing a profile-scoped fixture initializes the registered + // global/session schema through the fail-closed port. Registration is + // idempotent, so this fixture does not depend on another test having + // run first. + crate::register_registered_schema_installer(); + let directory = tempfile::tempdir().unwrap(); + let db_path = directory.path().join("bootstrap-writes.db"); + let authority = + DatabaseAuthority::acquire_test(&db_path, "canonical body bootstrap write fixture") + .unwrap(); + let (database, _) = Database::publish_test_runtime( + &db_path, + &authority, + TestDatabaseRuntimeMode::Initialize, + ) + .await + .unwrap(); + let transaction = database + .begin_write_transaction("seed canonical body bootstrap fixture") + .await + .unwrap(); + transaction + .execute_batch(CANONICAL_BODIES_TABLE_SQL) + .await + .unwrap(); + transaction + .execute_batch( + "CREATE TABLE lcm_raw_messages ( + store_id INTEGER PRIMARY KEY AUTOINCREMENT, + storage_kind TEXT NOT NULL, content TEXT, content_hash TEXT NOT NULL + ); + CREATE TABLE observations ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + observation_json TEXT NOT NULL + );", + ) + .await + .unwrap(); + let mut bodies = Vec::new(); + for ordinal in 1..=52_i64 { + let text = format!("retained bootstrap body {ordinal}"); + let body = StoredCanonicalBody::pack(text.as_bytes()).unwrap(); + transaction + .execute( + UPSERT_CANONICAL_BODY_SQL, + params![ + body.content_hash.as_str(), + body.encoding, + body.blob.as_slice(), + body.uncompressed_bytes + ], + ) + .await + .unwrap(); + // Holes in the key space must not turn the page into an ID count. + transaction + .execute( + "INSERT INTO lcm_raw_messages(store_id, storage_kind, content_hash) + VALUES (?1, 'inline', ?2)", + params![ordinal * 2, body.content_hash.as_str()], + ) + .await + .unwrap(); + if include_observations { + // Three distinct hashes from the third owner onward, so a + // 48-owner page crosses the 128-write submission boundary. + let current = body.content_hash.as_str(); + let previous = bodies.last().map_or(current, |body: &StoredCanonicalBody| { + body.content_hash.as_str() + }); + let earlier = bodies + .iter() + .rev() + .nth(1) + .map_or(current, |body| body.content_hash.as_str()); + let json = serde_json::json!({ + "payload": {"a": null, "b": null, "c": null}, + "_tracedecay_canonical_body_refs": { + "/payload/a": current, + "/payload/b": previous, + "/payload/c": earlier + } + }) + .to_string(); + transaction + .execute( + "INSERT INTO observations(sequence, observation_json) VALUES (?1, ?2)", + params![ordinal * 2, json], + ) + .await + .unwrap(); + } + bodies.push(body); + } + schema::install(&transaction).await.unwrap(); + transaction.commit().await.unwrap(); + Self { + database, + bodies, + _authority: authority, + _directory: directory, + } + } + + async fn begin_write(&self) -> DatabaseWriteTransaction<'_> { + self.database + .begin_write_transaction("canonical body bootstrap fixture write") + .await + .unwrap() + } + + async fn execute_batch(&self, sql: &str) { + let transaction = self.begin_write().await; + transaction.execute_batch(sql).await.unwrap(); + transaction.commit().await.unwrap(); + } + + async fn tick(&self) -> (CanonicalBodyGcReport, usize) { + let transaction = self.begin_write().await; + let counted = CountWrites { + inner: &transaction, + submissions: AtomicUsize::new(0), + }; + let report = step(&counted, 48).await.unwrap(); + let submissions = counted.submissions.load(Ordering::Relaxed); + transaction.commit().await.unwrap(); + (report, submissions) + } + + async fn assert_bodies_preserved(&self) { + let count = scalar( + &self.database.read_connection(), + "SELECT COUNT(*) FROM session_canonical_bodies", + ) + .await; + assert_eq!(count, self.bodies.len() as i64); + for body in &self.bodies { + let mut rows = self + .database + .read_connection() + .query( + "SELECT body FROM session_canonical_bodies WHERE content_hash = ?1", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + let stored = rows.next().await.unwrap().unwrap(); + assert_eq!(stored.get::>(0).unwrap(), body.blob); + } + } +} + +async fn scalar(conn: &(impl QueryExecutor + ?Sized), sql: &str) -> i64 { + conn.query(sql, ()) + .await + .unwrap() + .next() + .await + .unwrap() + .unwrap() + .get(0) + .unwrap() +} + +async fn raw_owners(conn: &(impl QueryExecutor + ?Sized)) -> Vec { + let mut rows = conn + .query( + "SELECT owner_id FROM canonical_body_gc_refs WHERE owner_kind = 'raw' + ORDER BY owner_id", + (), + ) + .await + .unwrap(); + let mut owners = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + owners.push(row.get(0).unwrap()); + } + owners +} + +#[tokio::test] +async fn bootstrap_pages_batch_writes_and_preserve_sparse_frontiers() { + let fixture = Fixture::open().await; + for (scanned, through, owners) in [(48_u64, 96_i64, 48_i64), (4, 104, 52)] { + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + assert_eq!(report.roots_scanned, scanned); + assert_eq!(submissions, 2, "one seed submission plus the checkpoint"); + assert_eq!(progress.raw_through, through); + assert_eq!( + raw_owners(&fixture.database.read_connection()).await, + (1..=owners).map(|ordinal| ordinal * 2).collect::>() + ); + assert_eq!(report.bodies_deleted, 0); + assert!(report.has_more); + } + + let mut hashes = fixture + .bodies + .iter() + .map(|body| body.content_hash.clone()) + .collect::>(); + hashes.sort(); + for (scanned, total) in [(48_u64, 48_usize), (4, 52)] { + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + let candidates = scalar( + &fixture.database.read_connection(), + "SELECT COUNT(*) FROM canonical_body_gc_candidates", + ) + .await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!(submissions, 2, "one candidate seed plus the checkpoint"); + assert_eq!(progress.bodies_after, hashes[total - 1]); + assert_eq!(candidates, total as i64); + assert_eq!(report.bodies_deleted, 0); + assert!(report.has_more); + } + + for scanned in [48_u64, 4] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!(report.candidates_discharged, scanned); + assert_eq!(report.bodies_deleted, 0); + assert_eq!(report.has_more, scanned == 48); + } + let (idle, submissions) = fixture.tick().await; + assert!(!idle.has_committed_work()); + assert!(!idle.has_more); + assert_eq!(submissions, 0); + fixture.assert_bodies_preserved().await; +} + +#[tokio::test] +async fn bootstrap_batch_refusal_rolls_back_and_recovers() { + for bodies_phase in [false, true] { + let fixture = Fixture::open().await; + if bodies_phase { + fixture.tick().await; + fixture.tick().await; + } + let before = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + let table = if bodies_phase { + "canonical_body_gc_candidates" + } else { + "canonical_body_gc_refs" + }; + // FAIL deliberately leaves the first 47 inserts in the transaction. + // The collector's rollback must discard them with the uncommitted page. + fixture + .execute_batch(&format!( + "CREATE TRIGGER test_bootstrap_refusal BEFORE INSERT ON {table} + WHEN (SELECT COUNT(*) FROM {table}) = 47 BEGIN + SELECT RAISE(FAIL, 'injected bootstrap batch refusal'); + END" + )) + .await; + let transaction = fixture.begin_write().await; + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error + .to_string() + .contains("injected bootstrap batch refusal") + ); + transaction.rollback().await.unwrap(); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap(), + before + ); + let count = scalar( + &fixture.database.read_connection(), + &format!("SELECT COUNT(*) FROM {table}"), + ) + .await; + assert_eq!(count, 0); + fixture.assert_bodies_preserved().await; + fixture + .execute_batch("DROP TRIGGER test_bootstrap_refusal") + .await; + for scanned in [48_u64, 4] { + let (report, submissions) = fixture.tick().await; + let actual = if bodies_phase { + report.bodies_scanned + } else { + report.roots_scanned + }; + assert_eq!(submissions, 2); + assert_eq!(actual, scanned); + assert_eq!(report.bodies_deleted, 0); + } + let count = scalar( + &fixture.database.read_connection(), + &format!("SELECT COUNT(*) FROM {table}"), + ) + .await; + assert_eq!(count, 52); + fixture.assert_bodies_preserved().await; + } +} + +#[tokio::test] +async fn bootstrap_batch_eligibility_does_not_change_the_page_boundary() { + let fixture = Fixture::open().await; + fixture + .execute_batch( + "UPDATE lcm_raw_messages SET storage_kind = 'external' WHERE store_id = 94; + UPDATE lcm_raw_messages SET content = 'owned inline text' WHERE store_id = 96;", + ) + .await; + let (report, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + assert_eq!(report.roots_scanned, 48); + assert_eq!(submissions, 2); + assert_eq!(progress.raw_through, 96); + assert_eq!( + raw_owners(&fixture.database.read_connection()).await, + (1..=46_i64).map(|ordinal| ordinal * 2).collect::>() + ); + assert!(report.has_more); + let (tail, submissions) = fixture.tick().await; + let progress = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + assert_eq!(tail.roots_scanned, 4); + assert_eq!(submissions, 2); + assert_eq!(progress.raw_through, 104); + let expected = (1..=52_i64) + .filter(|ordinal| ![47, 48].contains(ordinal)) + .map(|ordinal| ordinal * 2) + .collect::>(); + assert_eq!( + raw_owners(&fixture.database.read_connection()).await, + expected + ); + fixture.assert_bodies_preserved().await; +} + +async fn observation_owners(conn: &(impl QueryExecutor + ?Sized)) -> Vec { + let mut rows = conn + .query( + "SELECT DISTINCT owner_id FROM canonical_body_gc_refs + WHERE owner_kind = 'observation' ORDER BY owner_id", + (), + ) + .await + .unwrap(); + let mut owners = Vec::new(); + while let Some(row) = rows.next().await.unwrap() { + owners.push(row.get(0).unwrap()); + } + owners +} + +#[tokio::test] +async fn observation_bootstrap_batches_across_a_bounded_write_submission() { + let fixture = Fixture::with_observations(true).await; + for _ in 0..2 { + fixture.tick().await; + } + for (scanned, frontier, owners, refs, submissions) in [ + (48_u64, 96_i64, 48_i64, 141_i64, 3_usize), + (4, 104, 52, 153, 2), + ] { + let (report, actual_submissions) = fixture.tick().await; + assert_eq!(report.roots_scanned, scanned); + assert_eq!( + report.bootstrap_frontier_after, + Some(format!("observations:{frontier}")) + ); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap() + .observations_through, + frontier + ); + assert_eq!(actual_submissions, submissions); + assert_eq!( + observation_owners(&fixture.database.read_connection()).await, + (1..=owners).map(|ordinal| ordinal * 2).collect::>() + ); + assert_eq!( + scalar( + &fixture.database.read_connection(), + "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'" + ) + .await, + refs, + "duplicate references within an observation must stay deduplicated" + ); + assert_eq!(report.bodies_deleted, 0); + assert!(report.has_more); + } + let mut finished = false; + for _ in 0..6 { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_deleted, 0); + if !report.has_more { + finished = true; + break; + } + } + assert!( + finished, + "bounded bootstrap and candidate discharge must terminate" + ); + fixture.assert_bodies_preserved().await; + let (idle, submissions) = fixture.tick().await; + assert!(!idle.has_more); + assert!(!idle.has_committed_work()); + assert_eq!(submissions, 0); +} + +#[tokio::test] +async fn observation_bootstrap_failure_after_a_full_batch_rolls_back_and_recovers() { + let fixture = Fixture::with_observations(true).await; + for _ in 0..2 { + fixture.tick().await; + } + let before = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + fixture + .execute_batch( + "CREATE TRIGGER refuse_observation_seed BEFORE INSERT ON canonical_body_gc_refs + WHEN NEW.owner_kind = 'observation' AND NEW.owner_id = 94 BEGIN + SELECT RAISE(FAIL, 'injected observation batch refusal'); + END", + ) + .await; + let transaction = fixture.begin_write().await; + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error + .to_string() + .contains("injected observation batch refusal") + ); + transaction.rollback().await.unwrap(); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap(), + before + ); + assert!( + observation_owners(&fixture.database.read_connection()) + .await + .is_empty() + ); + fixture.assert_bodies_preserved().await; + + fixture + .execute_batch("DROP TRIGGER refuse_observation_seed") + .await; + for (frontier, references, submissions) in [(96_i64, 141_i64, 3_usize), (104, 153, 2)] { + let (report, actual_submissions) = fixture.tick().await; + assert_eq!( + report.bootstrap_frontier_after, + Some(format!("observations:{frontier}")) + ); + assert_eq!(actual_submissions, submissions); + assert_eq!( + scalar( + &fixture.database.read_connection(), + "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'" + ) + .await, + references + ); + } + fixture.assert_bodies_preserved().await; +} + +async fn assert_refused_bootstrap_rolls_back(fixture: &Fixture, diagnosis: &str) { + let before = read_progress(&fixture.database.read_connection()) + .await + .unwrap(); + let transaction = fixture.begin_write().await; + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error.to_string().contains(diagnosis), + "unexpected bootstrap refusal: {error}" + ); + transaction.rollback().await.unwrap(); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap(), + before + ); + fixture.assert_bodies_preserved().await; +} + +#[tokio::test] +async fn ignored_raw_reference_insert_refuses_checkpoint_then_recovers() { + let fixture = Fixture::open().await; + fixture + .execute_batch( + "CREATE TRIGGER ignore_raw_ref BEFORE INSERT ON canonical_body_gc_refs + WHEN NEW.owner_kind = 'raw' AND NEW.owner_id = 94 + BEGIN SELECT RAISE(IGNORE); END", + ) + .await; + assert_refused_bootstrap_rolls_back(&fixture, "raw bootstrap reference was not indexed").await; + assert!( + raw_owners(&fixture.database.read_connection()) + .await + .is_empty() + ); + fixture.execute_batch("DROP TRIGGER ignore_raw_ref").await; + for (scanned, frontier) in [(48_u64, 96_i64), (4, 104)] { + let (report, _) = fixture.tick().await; + assert_eq!(report.roots_scanned, scanned); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap() + .raw_through, + frontier + ); + } + assert_eq!( + raw_owners(&fixture.database.read_connection()).await.len(), + 52 + ); + fixture.assert_bodies_preserved().await; +} + +#[tokio::test] +async fn ignored_observation_reference_insert_refuses_checkpoint_then_recovers() { + let fixture = Fixture::with_observations(true).await; + fixture.tick().await; + fixture.tick().await; + fixture + .execute_batch( + "CREATE TRIGGER ignore_observation_ref BEFORE INSERT ON canonical_body_gc_refs + WHEN NEW.owner_kind = 'observation' AND NEW.owner_id = 94 + BEGIN SELECT RAISE(IGNORE); END", + ) + .await; + assert_refused_bootstrap_rolls_back( + &fixture, + "observation bootstrap reference was not indexed", + ) + .await; + assert!( + observation_owners(&fixture.database.read_connection()) + .await + .is_empty() + ); + fixture + .execute_batch("DROP TRIGGER ignore_observation_ref") + .await; + for (frontier, references) in [(96_i64, 141_i64), (104, 153)] { + let (report, _) = fixture.tick().await; + assert_eq!( + report.bootstrap_frontier_after, + Some(format!("observations:{frontier}")) + ); + assert_eq!( + read_progress(&fixture.database.read_connection()) + .await + .unwrap() + .observations_through, + frontier + ); + assert_eq!( + scalar( + &fixture.database.read_connection(), + "SELECT COUNT(*) FROM canonical_body_gc_refs + WHERE owner_kind = 'observation'", + ) + .await, + references, + ); + } + fixture.assert_bodies_preserved().await; +} + +#[tokio::test] +async fn ignored_candidate_insert_refuses_checkpoint_then_recovers() { + let fixture = Fixture::open().await; + fixture.tick().await; + fixture.tick().await; + let mut hashes = fixture + .bodies + .iter() + .map(|body| body.content_hash.as_str()) + .collect::>(); + hashes.sort_unstable(); + let skipped = hashes[47]; + fixture + .execute_batch(&format!( + "CREATE TRIGGER ignore_candidate_seed BEFORE INSERT ON canonical_body_gc_candidates + WHEN NEW.content_hash = '{skipped}' + BEGIN SELECT RAISE(IGNORE); END" + )) + .await; + assert_refused_bootstrap_rolls_back(&fixture, "body bootstrap candidate was not indexed").await; + assert_eq!( + scalar( + &fixture.database.read_connection(), + "SELECT COUNT(*) FROM canonical_body_gc_candidates", + ) + .await, + 0 + ); + fixture + .execute_batch("DROP TRIGGER ignore_candidate_seed") + .await; + for (scanned, candidate_total) in [(48_u64, 48_i64), (4, 52)] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!( + scalar( + &fixture.database.read_connection(), + "SELECT COUNT(*) FROM canonical_body_gc_candidates" + ) + .await, + candidate_total + ); + } + for scanned in [48_u64, 4] { + let (report, _) = fixture.tick().await; + assert_eq!(report.bodies_scanned, scanned); + assert_eq!(report.candidates_discharged, scanned); + assert_eq!(report.bodies_deleted, 0, "referenced bodies are preserved"); + } + let (idle, submissions) = fixture.tick().await; + assert!(!idle.has_more); + assert_eq!(submissions, 0); + fixture.assert_bodies_preserved().await; +} diff --git a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs index 39d3af4b4ee..f9452dfd050 100644 --- a/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs +++ b/crates/tracedecay-global-db/src/observation/canonical_body_gc/tests.rs @@ -6,6 +6,7 @@ use tracedecay_store::{ }; use super::{CanonicalBodyGcReport, has_candidates, read_progress, schema, step}; +use crate::tests::harness::{RegisteredGlobalDbHarness, writer_telemetry}; struct Store { runtime: TestConnection, @@ -92,6 +93,14 @@ async fn body_exists(conn: &(impl QueryExecutor + ?Sized), hash: &str) -> bool { .is_some() } +async fn candidate_count(conn: &(impl QueryExecutor + ?Sized)) -> i64 { + let mut rows = conn + .query("SELECT COUNT(*) FROM canonical_body_gc_candidates", ()) + .await + .unwrap(); + rows.next().await.unwrap().unwrap().get(0).unwrap() +} + fn observation(hash: &str) -> String { serde_json::json!({ "payload": {"text": null}, @@ -506,3 +515,429 @@ async fn nonpositive_legacy_owner_ids_cannot_be_skipped_by_bootstrap() { ); } } + +#[tokio::test] +async fn ignored_body_delete_rolls_back_candidates_then_reclaims_distinct_pages() { + let store = Store::open().await; + store.install().await; + let mut hashes = Vec::new(); + for ordinal in 0..52 { + let body = + StoredCanonicalBody::pack(format!("unreferenced candidate {ordinal}").as_bytes()) + .unwrap(); + insert_body(store.conn(), &body).await; + hashes.push(body.content_hash); + } + assert_eq!(candidate_count(store.conn()).await, 52); + store + .conn() + .execute_batch( + "CREATE TRIGGER refuse_body_gc_delete + BEFORE DELETE ON session_canonical_bodies BEGIN + SELECT RAISE(IGNORE); + END", + ) + .await + .unwrap(); + + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error + .to_string() + .contains("collection left an unreferenced body behind") + ); + transaction.rollback().await.unwrap(); + assert_eq!( + candidate_count(store.conn()).await, + 52, + "a vetoed DELETE cannot discard the only future collection opportunity" + ); + for hash in &hashes { + assert!(body_exists(store.conn(), hash).await); + } + + store + .conn() + .execute_batch("DROP TRIGGER refuse_body_gc_delete") + .await + .unwrap(); + let first = store.tick(48).await; + assert_eq!(first.bodies_scanned, 48); + assert_eq!(first.bodies_deleted, 48); + assert_eq!(first.candidates_discharged, 48); + assert!(first.has_more); + assert_eq!(candidate_count(store.conn()).await, 4); + let second = store.tick(48).await; + assert_eq!(second.bodies_scanned, 4); + assert_eq!(second.bodies_deleted, 4); + assert_eq!(second.candidates_discharged, 4); + assert!(!second.has_more); + assert_eq!(candidate_count(store.conn()).await, 0); + for hash in &hashes { + assert!(!body_exists(store.conn(), hash).await); + } + assert_eq!( + store.tick(48).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); +} + +#[tokio::test] +async fn zero_delete_pages_discharge_distinct_candidates_and_quiesce() { + let store = Store::open().await; + for ordinal in 0..52 { + let body = + StoredCanonicalBody::pack(format!("still referenced {ordinal}").as_bytes()).unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + + let mut discharged_pages = Vec::new(); + for _ in 0..16 { + let report = store.tick(48).await; + assert_eq!(report.bodies_deleted, 0, "referenced bodies stay live"); + if report.candidates_discharged > 0 { + assert_eq!(report.candidates_discharged, report.bodies_scanned); + discharged_pages.push(report.candidates_discharged); + } + if !report.has_more { + break; + } + } + assert_eq!(discharged_pages, vec![48, 4]); + assert!(!has_candidates(store.conn()).await.unwrap()); + assert_eq!( + store.tick(48).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); + let mut rows = store + .conn() + .query("SELECT COUNT(*) FROM session_canonical_bodies", ()) + .await + .unwrap(); + let count: i64 = rows.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(count, 52); +} + +#[tokio::test] +async fn bootstrap_reports_each_committed_sql_frontier_without_claiming_collection() { + let store = Store::open().await; + let body = StoredCanonicalBody::pack(b"referenced bootstrap progress").unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + store + .conn() + .execute( + "INSERT INTO observations(observation_json) VALUES (?1)", + params![observation(&body.content_hash)], + ) + .await + .unwrap(); + store.install().await; + + let frontiers = [ + ("raw:0".to_owned(), "raw:1".to_owned()), + ("observations:0".to_owned(), "observations:1".to_owned()), + ( + "bodies:".to_owned(), + format!("bodies:{}", body.content_hash), + ), + ]; + for (phase, (before, after)) in frontiers.into_iter().enumerate() { + let report = store.tick(1).await; + assert_eq!( + report.bootstrap_frontier_before.as_deref(), + Some(before.as_str()) + ); + assert_eq!( + report.bootstrap_frontier_after.as_deref(), + Some(after.as_str()) + ); + let persisted = read_progress(store.conn()).await.unwrap(); + let stored_after = match phase { + 0 => format!("raw:{}", persisted.raw_through), + 1 => format!("observations:{}", persisted.observations_through), + _ => format!("bodies:{}", persisted.bodies_after), + }; + assert_eq!(stored_after, after, "reported cursor must be durable"); + assert_eq!(report.roots_scanned + report.bodies_scanned, 1); + assert_eq!(report.bodies_deleted, 0); + assert_eq!(report.candidates_discharged, 0); + assert!(report.has_more); + } + let discharged = store.tick(1).await; + assert_eq!(discharged.candidates_discharged, 1); + assert_eq!(discharged.bodies_deleted, 0); + assert!(discharged.bootstrap_frontier_before.is_none()); + assert!(discharged.bootstrap_frontier_after.is_none()); + assert!(!discharged.has_more); + assert!(body_exists(store.conn(), &body.content_hash).await); + assert_eq!( + store.tick(1).await, + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); +} + +#[tokio::test] +async fn empty_bootstrap_page_reports_its_committed_frontier_once() { + let store = Store::open().await; + store.conn().execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) VALUES ('inline', 'text', 'fixture')", () + ).await.unwrap(); + store.install().await; + store + .conn() + .execute("DELETE FROM lcm_raw_messages", ()) + .await + .unwrap(); + + let page = store.tick(48).await; + assert_eq!((page.roots_scanned, page.bodies_scanned), (0, 0)); + assert_eq!(page.bootstrap_frontier_before.as_deref(), Some("raw:0")); + assert_eq!(page.bootstrap_frontier_after.as_deref(), Some("raw:1")); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 1); + assert!(page.has_committed_work()); + assert!(!store.tick(48).await.has_committed_work()); +} + +#[tokio::test] +async fn rewritten_bootstrap_checkpoint_refuses_commit_and_recovers_across_48_row_pages() { + let store = Store::open().await; + let shared = StoredCanonicalBody::pack(b"shared canonical body").unwrap(); + insert_body(store.conn(), &shared).await; + for _ in 0..52 { + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![shared.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + // Even with recursive triggers enabled, the inner write terminates at zero. + // SQLite reports one updated row for the outer statement, but a plain + // affected-row check would miss the nonadvancing durable frontier. + store + .conn() + .execute_batch( + "CREATE TRIGGER corrupt_gc_bootstrap AFTER UPDATE ON canonical_body_gc_progress + WHEN NEW.raw_through > 0 BEGIN + UPDATE canonical_body_gc_progress SET raw_through = 0 WHERE singleton = 1; + END", + ) + .await + .unwrap(); + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("frontier was not persisted")); + let mut staged = transaction + .query("SELECT COUNT(*) FROM canonical_body_gc_refs", ()) + .await + .unwrap(); + let staged_count: i64 = staged.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!( + staged_count, 48, + "the page was enrolled before checkpoint refusal" + ); + drop(staged); + transaction.rollback().await.unwrap(); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 0); + let mut durable = store + .conn() + .query("SELECT COUNT(*) FROM canonical_body_gc_refs", ()) + .await + .unwrap(); + let durable_count: i64 = durable.next().await.unwrap().unwrap().get(0).unwrap(); + assert_eq!(durable_count, 0, "the refused page must roll back its refs"); + drop(durable); + assert!(body_exists(store.conn(), &shared.content_hash).await); + + store + .conn() + .execute_batch("DROP TRIGGER corrupt_gc_bootstrap") + .await + .unwrap(); + let first = store.tick(48).await; + assert_eq!(first.roots_scanned, 48); + assert_eq!(first.bootstrap_frontier_before.as_deref(), Some("raw:0")); + assert_eq!(first.bootstrap_frontier_after.as_deref(), Some("raw:48")); + assert!(first.has_more); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 48); + let second = store.tick(48).await; + assert_eq!(second.roots_scanned, 4); + assert_eq!(second.bootstrap_frontier_before.as_deref(), Some("raw:48")); + assert_eq!(second.bootstrap_frontier_after.as_deref(), Some("raw:52")); + assert_eq!(read_progress(store.conn()).await.unwrap().raw_through, 52); + let (deleted, bytes) = drain(&store, 48).await; + assert_eq!((deleted, bytes), (0, 0), "live bodies remain referenced"); + assert!(body_exists(store.conn(), &shared.content_hash).await); + assert!(!has_candidates(store.conn()).await.unwrap()); + let idle = store.tick(48).await; + assert!(idle.bootstrap_complete); + assert_eq!(idle.roots_scanned, 0); + assert_eq!(idle.bodies_scanned, 0); + assert!(!idle.has_more); +} + +#[tokio::test] +async fn reinserted_candidate_page_refuses_false_discharge_and_recovers() { + let store = Store::open().await; + for ordinal in 0..52 { + let body = StoredCanonicalBody::pack(format!("retained {ordinal}").as_bytes()).unwrap(); + insert_body(store.conn(), &body).await; + store + .conn() + .execute( + "INSERT INTO lcm_raw_messages(storage_kind, content, content_hash) + VALUES ('inline', NULL, ?1)", + params![body.content_hash.as_str()], + ) + .await + .unwrap(); + } + store.install().await; + let mut bootstrap_finished = false; + for _ in 0..16 { + if store.tick(48).await.bootstrap_complete { + bootstrap_finished = true; + break; + } + } + assert!( + bootstrap_finished, + "candidate collection must become eligible" + ); + assert_eq!(candidate_count(store.conn()).await, 52); + + // The explicit DELETE returns 48 affected rows even when an AFTER DELETE + // trigger restores all of them. Only a same-transaction readback catches + // the false progress that otherwise repeats this exact 48-row page. + store + .conn() + .execute_batch( + "CREATE TRIGGER keep_candidates_after_delete + AFTER DELETE ON canonical_body_gc_candidates BEGIN + INSERT OR IGNORE INTO canonical_body_gc_candidates(content_hash) + VALUES (OLD.content_hash); + END", + ) + .await + .unwrap(); + let transaction = store + .conn() + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!( + error + .to_string() + .contains("candidate discharge was not persisted"), + "an apparent 48-row DELETE must not count as committed progress: {error}" + ); + assert_eq!( + candidate_count(&transaction).await, + 52, + "the trigger left every candidate in the still-open transaction" + ); + transaction.rollback().await.unwrap(); + assert_eq!( + candidate_count(store.conn()).await, + 52, + "the refused page must leave all candidate identities durable" + ); + store + .conn() + .execute_batch("DROP TRIGGER keep_candidates_after_delete") + .await + .unwrap(); + + let first = store.tick(48).await; + assert_eq!(first.bodies_scanned, 48); + assert_eq!(first.candidates_discharged, 48); + assert_eq!(first.bodies_deleted, 0); + assert!(first.has_more); + let second = store.tick(48).await; + assert_eq!(second.bodies_scanned, 4); + assert_eq!(second.candidates_discharged, 4); + assert_eq!(second.bodies_deleted, 0); + assert!(!second.has_more); + assert_eq!(candidate_count(store.conn()).await, 0); + let idle = store.tick(48).await; + assert_eq!(idle.bodies_scanned, 0); + assert!(!idle.has_more); +} + +#[tokio::test] +async fn idle_registered_gc_avoids_writer_and_rechecks_new_candidates() { + let harness = RegisteredGlobalDbHarness::open("canonical-body-idle-writer").await; + let before = writer_telemetry(&harness.registered).transactions; + for _ in 0..3 { + assert_eq!( + harness.registered.run_canonical_body_gc(48).await.unwrap(), + CanonicalBodyGcReport { + bootstrap_complete: true, + ..CanonicalBodyGcReport::default() + } + ); + assert_eq!( + writer_telemetry(&harness.registered).transactions, + before, + "a drained retention store must not enter the SQLite writer" + ); + } + + let unowned = StoredCanonicalBody::pack(b"arrived after an idle retention probe").unwrap(); + let transaction = harness.registered.begin_write_transaction().await.unwrap(); + insert_body(&transaction, &unowned).await; + transaction.commit().await.unwrap(); + let after_seed = writer_telemetry(&harness.registered).transactions; + let collected = harness.registered.run_canonical_body_gc(48).await.unwrap(); + assert_eq!(collected.bodies_scanned, 1); + assert_eq!(collected.bodies_deleted, 1); + assert_eq!(collected.candidates_discharged, 1); + assert!(!collected.has_more); + assert!( + writer_telemetry(&harness.registered).transactions.commands > after_seed.commands, + "a new candidate must still enter the real writer" + ); + assert!(!body_exists(&harness.registered.read_connection(), &unowned.content_hash).await); +} diff --git a/crates/tracedecay-global-db/src/observation/retention/tests.rs b/crates/tracedecay-global-db/src/observation/retention/tests.rs index d55c4ed0e2a..bb62ea1cade 100644 --- a/crates/tracedecay-global-db/src/observation/retention/tests.rs +++ b/crates/tracedecay-global-db/src/observation/retention/tests.rs @@ -72,6 +72,19 @@ impl Executor for RetentionTestStore { .await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.database() + .writer_connection() + .map_err(|error| { + tracedecay_runtime_core::db::engine::Error::invalid_operation(error.to_string()) + })? + .execute_statements(statements) + .await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { self.database() .writer_connection() diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index 8a983decb4e..8d859d548f3 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -1036,6 +1036,16 @@ impl RegisteredGlobalDbWriterConnection<'_> { .await .map_err(engine_error) } + + pub async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.database + .execute_write_statements("execute registered global database statements", statements) + .await + .map_err(engine_error) + } } pub struct RegisteredGlobalDbWriteTransaction<'a> { @@ -1068,6 +1078,13 @@ impl Executor for RegisteredGlobalDbWriteTransaction<'_> { RegisteredGlobalDbWriteTransaction::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + RegisteredGlobalDbWriteTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { RegisteredGlobalDbWriteTransaction::execute_batch(self, sql).await } @@ -1124,6 +1141,13 @@ impl RegisteredGlobalDbWriteTransaction<'_> { self.transaction.execute_batch(sql).await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.transaction.execute_statements(statements).await + } + #[tracing::instrument(name = "global_db.registered.txn.commit", level = "trace", skip_all)] pub async fn commit(self) -> tracedecay_runtime_core::db::engine::Result<()> { self.commit_with_on_commit(|| {}).await @@ -1201,6 +1225,13 @@ impl Executor for RegisteredGlobalDbWriteTx<'_, '_> { self.transaction.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.transaction.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { self.transaction.execute_batch(sql).await } diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 01e87f7f53c..c88c416a63e 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -1074,10 +1074,36 @@ mod published_artifact_ceiling { let stride = (total_polls / 12).max(1); let mut cancelled_published = 0; + // Each convergence joins its exact runtime retirement. Reuse the + // canonically initialized store, restoring the upgrade's inputs and + // proving the entire workflow state before admitting a fresh runtime. + let store = &reference.store; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create().await; - let store = &daemon_store.store; - store.run(BULK_ROWS); + { + let mut connection = Connection::open(&store.path).unwrap(); + let transaction = connection.transaction().unwrap(); + republish(&transaction, &before.table_sql, &before.digest); + { + let mut insert = transaction + .prepare( + "INSERT INTO workflow_artifact_payloads + (rowid, payload_digest, byte_length, payload) + VALUES (?1, ?2, ?3, ?4)", + ) + .unwrap(); + for (rowid, digest, byte_length, payload) in &before.artifacts { + insert + .execute(rusqlite::params![rowid, digest, byte_length, payload]) + .unwrap(); + } + } + transaction.commit().unwrap(); + } + assert_eq!( + store.state(), + before, + "poll {cancel_at} must start from the published workflow state" + ); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; let state = store.state(); match outcome { @@ -1319,6 +1345,13 @@ mod published_artifact_ceiling { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> engine::Result> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> engine::Result<()> { self.note(sql); self.inner.execute_batch(sql).await @@ -1391,6 +1424,13 @@ mod published_artifact_ceiling { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> engine::Result> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> engine::Result<()> { self.check(sql)?; self.inner.execute_batch(sql).await diff --git a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs index 7d3f869f2da..4cf9bd7fa02 100644 --- a/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs +++ b/crates/tracedecay-global-db/src/schema_contract/invariants/audit.rs @@ -1715,6 +1715,13 @@ mod tests { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { self.inner.execute_batch(sql).await } diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 0ac0e0287b5..d3be4174fd1 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -1011,6 +1011,14 @@ impl Executor for CancellableSchemaTransaction<'_, T> { self.transaction.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> engine::Result> { + self.checkpoint()?; + self.transaction.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> engine::Result<()> { self.checkpoint()?; self.transaction.execute_batch(sql).await diff --git a/crates/tracedecay-global-db/src/session_handle.rs b/crates/tracedecay-global-db/src/session_handle.rs index ec12592de02..4e56ca3290f 100644 --- a/crates/tracedecay-global-db/src/session_handle.rs +++ b/crates/tracedecay-global-db/src/session_handle.rs @@ -125,6 +125,13 @@ impl Executor for RegisteredGlobalDbWriterConnection<'_> { RegisteredGlobalDbWriterConnection::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + RegisteredGlobalDbWriterConnection::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { RegisteredGlobalDbWriterConnection::execute_batch(self, sql).await } diff --git a/crates/tracedecay-global-db/src/tests/harness.rs b/crates/tracedecay-global-db/src/tests/harness.rs index bb80c1884b8..7de179fbb98 100644 --- a/crates/tracedecay-global-db/src/tests/harness.rs +++ b/crates/tracedecay-global-db/src/tests/harness.rs @@ -1606,6 +1606,19 @@ impl Executor for RegisteredGlobalDbTestFixture { .await } + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.database + .writer_connection() + .map_err(|error| { + tracedecay_runtime_core::db::engine::Error::invalid_operation(error.to_string()) + })? + .execute_statements(statements) + .await + } + async fn execute_batch(&self, sql: &str) -> tracedecay_runtime_core::db::engine::Result<()> { self.database .writer_connection() diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 22a7d3611c0..fbfe8b5f1f8 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -15,7 +15,7 @@ use tracedecay_runtime_core::db::{ use crate::tests::harness::open_registered_test_database_fixture; -async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { +async fn initialize_global_db(db_path: &Path) -> tracedecay_domain::errors::Result<()> { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent)?; } @@ -23,6 +23,11 @@ async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result tracedecay_domain::errors::Result { + initialize_global_db(db_path).await?; Ok(TestConnection::open(db_path)) } diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 4dcc84fa725..1d2cb2f65d2 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -63,10 +63,9 @@ async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { async fn temporal_payload_manifest_schema_is_payload_global() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -175,10 +174,9 @@ mod admission; async fn temporal_schema_query_indexes_cover_exact_lookup_shapes() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -305,10 +303,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -608,10 +605,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh async fn temporal_schema_rejects_unexpected_indexes_without_dropping_them() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -719,10 +715,9 @@ async fn temporal_schema_refuses_markerless_malformed_fts_without_mutation() { async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -791,10 +786,9 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { async fn temporal_schema_rejects_missing_graph_publication_authority_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index a5ba897a0e1..c1ec89d7d12 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -285,10 +285,9 @@ async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_cont async fn temporal_schema_rejects_transition_storage_without_mutating_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -332,10 +331,9 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -383,10 +381,9 @@ async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutati async fn final_store_missing_temporal_marker_requires_reset_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive every final authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -556,10 +553,9 @@ async fn writer_ledger_does_not_hide_a_nonempty_unmarked_store() { async fn final_schema_admission_rejects_extra_temporal_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -598,10 +594,9 @@ async fn final_schema_admission_rejects_extra_temporal_column_metadata() { async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal indexes"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -635,10 +630,9 @@ async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() async fn final_schema_admission_rejects_missing_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -670,10 +664,9 @@ async fn final_schema_admission_rejects_missing_graph_publication_table() { async fn final_schema_admission_rejects_incompatible_graph_publication_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -712,10 +705,9 @@ async fn final_schema_admission_rejects_incompatible_graph_publication_column_me async fn final_schema_admission_rejects_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -753,10 +745,9 @@ async fn final_schema_admission_rejects_extra_graph_publication_table() { async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -794,10 +785,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -835,10 +825,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index b9c1c52d30f..17c54837976 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -71,10 +71,9 @@ async fn temporal_schema_rejects_cross_session_and_generation_rows() { async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -258,10 +257,9 @@ async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -365,10 +363,9 @@ async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1160,10 +1157,9 @@ async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { async fn temporal_schema_enforces_generation_state_machine_and_durability() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1271,10 +1267,9 @@ async fn temporal_schema_enforces_generation_state_machine_and_durability() { async fn temporal_schema_keeps_append_only_authority_immutable() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index a1f6398e1b2..ad69571ce89 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -87,10 +87,9 @@ async fn temporal_schema_persists_cursor_keys_without_read_creation() { async fn temporal_schema_rejects_direct_cursor_retirement() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -121,10 +120,9 @@ async fn temporal_schema_rejects_direct_cursor_retirement() { async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -250,10 +248,9 @@ async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -304,10 +301,9 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -362,10 +358,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-lcm/src/gc/tests.rs b/crates/tracedecay-lcm/src/gc/tests.rs index 62b0f3b7ecd..7da03245da2 100644 --- a/crates/tracedecay-lcm/src/gc/tests.rs +++ b/crates/tracedecay-lcm/src/gc/tests.rs @@ -1707,6 +1707,14 @@ impl Executor for CountingExecutor<'_, E> { self.counter.tick(); self.inner.execute_batch(sql).await } + + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + self.counter.tick(); + self.inner.execute_statements(statements).await + } } fn batch_ref(index: usize) -> String { diff --git a/crates/tracedecay-lcm/src/retention.rs b/crates/tracedecay-lcm/src/retention.rs index fc7bfcd9355..712fd72585d 100644 --- a/crates/tracedecay-lcm/src/retention.rs +++ b/crates/tracedecay-lcm/src/retention.rs @@ -660,6 +660,17 @@ impl Executor for RetentionWriteTransaction<'_> { Self::Connection(transaction) => transaction.execute_batch(sql).await, } } + + async fn execute_statements( + &self, + statements: Vec, + ) -> tracedecay_runtime_core::db::engine::Result> { + match self { + Self::Database(transaction) => transaction.execute_statements(statements).await, + #[cfg(test)] + Self::Connection(transaction) => transaction.execute_statements(statements).await, + } + } } #[derive(Clone, Copy)] diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata.rs b/crates/tracedecay-lcm/src/retention/replay_metadata.rs index b7be38726d9..a948c9dfd1b 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata.rs @@ -5,6 +5,9 @@ //! representations keep their own content. A durable rowid frontier and a byte //! budget keep this repair off the foreground ingest path. +use std::collections::BTreeSet; +use std::time::Instant; + use serde::{Deserialize, Serialize}; use serde_json::Value; use tracedecay_runtime_core::db::Database; @@ -12,6 +15,8 @@ use tracedecay_runtime_core::db::engine::{Executor, QueryExecutor, params}; use crate::{LcmError, LcmRawMessage, gc, payload, raw, schema, security, summary_convergence}; +#[cfg(all(test, target_os = "linux"))] +mod read_cost; #[cfg(test)] mod tests; @@ -19,19 +24,29 @@ const FRONTIER_KEY: &str = "replay_metadata_compaction_store_id_v1"; const OPERATION: &str = "compact duplicate LCM replay metadata"; pub const MAX_REPLAY_METADATA_COMPACTION_ROWS: usize = 48; // Bound both receipt verification and metadata sanitization while a writer is -// held. The raw metadata envelope itself is capped at 1 MiB by ingestion. +// held. Eligibility parsing has its own equal byte budget: ineligible rows +// still cost JSON work. Ingestion caps each metadata envelope at 1 MiB. const MAX_BATCH_BYTES: u64 = 4 * 1024 * 1024; #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ReplayMetadataCompactionReport { pub rows_scanned: u64, + /// Wall-clock duration from initial frontier read through commit, not CPU time. + pub elapsed_ms: Option, + /// Transactional store-id frontier consumed by this page, absent on idle reads. + pub frontier_before: Option, + pub frontier_after: Option, pub rows_compacted: u64, - /// Eligible rows retained because their representation, sanitizer result, - /// or byte bound could not prove this narrow lossless transformation. + /// Rows retained because their representation, sanitizer result, or byte + /// bound could not prove this narrow lossless transformation, including + /// rows rejected by the eligibility check before body verification. pub rows_skipped: u64, /// Subset of skipped rows whose metadata plus canonical/stored body /// representations exceed this pass's 4 MiB read and verification budget. pub rows_skipped_oversized: u64, + /// Metadata bytes admitted to SQL eligibility checks, including ineligible + /// rows. Work deferred by the body verification budget is not parsed. + pub metadata_bytes_examined: u64, /// UTF-8 metadata bytes removed. This does not measure file shrinkage. pub metadata_bytes_removed: u64, pub has_more: bool, @@ -70,6 +85,7 @@ pub async fn compact_replay_metadata( database: &Database, max_rows: usize, ) -> Result { + let started = Instant::now(); let read = database.read_connection(); if !has_work(&read, frontier(&read).await?).await? { return Ok(ReplayMetadataCompactionReport::default()); @@ -79,8 +95,9 @@ pub async fn compact_replay_metadata( .await .map_err(db_error)?; match step(&transaction, max_rows).await { - Ok(report) => { + Ok(mut report) => { transaction.commit().await.map_err(db_error)?; + report.elapsed_ms = Some(started.elapsed().as_millis()); Ok(report) } Err(error) => match transaction.rollback().await { @@ -93,39 +110,48 @@ pub async fn compact_replay_metadata( struct Candidate { store_id: i64, eligible: bool, - read_bytes: Option, + oversized: bool, + body_size_known: bool, } -/// Read only sizes and identities until each candidate earns the byte budget. +/// Admit a prefix bounded by both metadata parsing and body-read estimates +/// before testing eligibility. Deferred rows must not be JSON-parsed again. async fn candidates( conn: &(impl QueryExecutor + ?Sized), after: i64, limit: usize, -) -> Result, LcmError> { +) -> Result<(Vec, u64), LcmError> { + // octet_length reads record metadata, unlike length(CAST(text AS BLOB)). + // Even a small scalar after a large body can load its overflow chain: + // defer storage_kind to eligibility and avoid unnecessary canonical joins. let mut rows = conn .query( - "SELECT r.store_id, - CASE WHEN r.storage_kind = 'inline' AND json_valid(r.metadata_json) - THEN json_type(r.metadata_json, '$.lcm_active_replay') = 'true' - AND json_type(r.metadata_json, '$.active_replay.content') = 'text' - ELSE 0 END, - LENGTH(CAST(r.metadata_json AS BLOB)), - COALESCE(LENGTH(CAST(r.content AS BLOB)), b.uncompressed_bytes), + "SELECT r.store_id, octet_length(r.metadata_json), + COALESCE(octet_length(r.content), + CASE WHEN LENGTH(b.body) <= ?3 THEN b.uncompressed_bytes END), COALESCE(LENGTH(b.body), 0) FROM lcm_raw_messages r - LEFT JOIN session_canonical_bodies b ON b.content_hash = r.content_hash + LEFT JOIN session_canonical_bodies b ON b.content_hash = CASE + WHEN r.content IS NULL AND octet_length(r.metadata_json) <= ?3 + THEN r.content_hash END WHERE r.store_id > ?1 ORDER BY r.store_id LIMIT ?2", params![ after, - limit.clamp(1, MAX_REPLAY_METADATA_COMPACTION_ROWS) as i64 + limit.clamp(1, MAX_REPLAY_METADATA_COMPACTION_ROWS) as i64, + MAX_BATCH_BYTES as i64 ], ) .await?; let mut result = Vec::new(); + let mut eligibility_ids = Vec::new(); + let mut remaining = MAX_BATCH_BYTES; + let mut read_remaining = MAX_BATCH_BYTES; while let Some(row) = rows.next().await? { - let metadata: Option = row.get(2)?; - let body: Option = row.get(3)?; - let stored: i64 = row.get(4)?; + let store_id: i64 = row.get(0)?; + let metadata: Option = row.get(1)?; + let body: Option = row.get(2)?; + let stored: i64 = row.get(3)?; + let metadata_bytes = u64::try_from(metadata.unwrap_or(0)).map_err(db_error)?; let read_bytes = metadata .zip(body) .and_then(|(metadata, body)| { @@ -136,62 +162,116 @@ async fn candidates( )) }) .and_then(|(metadata, body, stored)| metadata.checked_add(body)?.checked_add(stored)); + let oversized = metadata_bytes > MAX_BATCH_BYTES + || stored > MAX_BATCH_BYTES as i64 + || read_bytes.is_some_and(|bytes| bytes > MAX_BATCH_BYTES); + if !oversized { + if metadata_bytes > remaining { + break; + } + // Eligibility is not yet known. Charge each potential body read + // before parsing so deferred rows are not revisited for JSON work. + if let Some(bytes) = read_bytes { + if bytes > read_remaining { + break; + } + read_remaining -= bytes; + } + remaining -= metadata_bytes; + if metadata.is_some() { + eligibility_ids.push(store_id); + } + } result.push(Candidate { - store_id: row.get(0)?, - eligible: row.get::>(1)?.unwrap_or(0) != 0, - read_bytes, + store_id, + eligible: false, + oversized, + body_size_known: read_bytes.is_some(), }); } - Ok(result) + drop(rows); + // Resolve the admitted prefix in one query, not one query per raw row. + // Both reads share the writer transaction, so sizes and identities cannot + // change between admission and parsing. Oversized documents stay untouched. + if !eligibility_ids.is_empty() { + let ids = serde_json::to_string(&eligibility_ids).map_err(db_error)?; + let mut rows = conn + .query( + "SELECT store_id FROM lcm_raw_messages + WHERE store_id IN (SELECT value FROM json_each(?1)) + AND CASE WHEN storage_kind = 'inline' AND json_valid(metadata_json) + THEN json_type(metadata_json, '$.lcm_active_replay') = 'true' + AND json_type(metadata_json, '$.active_replay.content') = 'text' + ELSE 0 END", + params![ids], + ) + .await?; + let mut eligible = BTreeSet::new(); + while let Some(row) = rows.next().await? { + eligible.insert(row.get::(0)?); + } + for candidate in &mut result { + candidate.eligible = eligible.contains(&candidate.store_id); + } + } + Ok((result, MAX_BATCH_BYTES - remaining)) } +#[tracing::instrument( + name = "lcm.retention.replay_metadata.page", + level = "trace", + skip_all, + fields(metadata_bytes_examined = tracing::field::Empty), + err +)] async fn step( conn: &(impl Executor + ?Sized), max_rows: usize, ) -> Result { let mut after = frontier(conn).await?; - let candidates = candidates(conn, after, max_rows).await?; - let mut report = ReplayMetadataCompactionReport::default(); - let mut remaining = MAX_BATCH_BYTES; + let before = after; + let (candidates, metadata_bytes_examined) = candidates(conn, after, max_rows).await?; + tracing::Span::current().record("metadata_bytes_examined", metadata_bytes_examined); + let mut report = ReplayMetadataCompactionReport { + metadata_bytes_examined, + ..ReplayMetadataCompactionReport::default() + }; let mut guard_suspended = false; for candidate in candidates { - if candidate.eligible { - let bytes = candidate.read_bytes.ok_or_else(|| { - db_error(format!( + if candidate.oversized { + report.rows_skipped += 1; + report.rows_skipped_oversized += 1; + } else if candidate.eligible { + if !candidate.body_size_known { + return Err(db_error(format!( "raw row {} has no bounded canonical body", candidate.store_id - )) - })?; - if bytes > MAX_BATCH_BYTES { - report.rows_skipped += 1; - report.rows_skipped_oversized += 1; - } else if bytes > remaining { - break; - } else { - remaining -= bytes; - let message = raw::load_raw_message_by_store_id(conn, candidate.store_id).await?; - if let Some(metadata) = compacted_metadata(&message)? { - if !guard_suspended { - summary_convergence::suspend_raw_update_guard(conn).await?; - guard_suspended = true; - } - let changed = - apply_metadata(conn, candidate.store_id, &message, &metadata).await?; - if changed { - report.rows_compacted += 1; - report.metadata_bytes_removed += message - .metadata_json - .as_deref() - .map_or(0, str::len) - .saturating_sub(metadata.len()) - as u64; - } else { - report.rows_skipped += 1; - } + ))); + } + let message = raw::load_raw_message_by_store_id(conn, candidate.store_id).await?; + if let Some(metadata) = compacted_metadata(&message)? { + if !guard_suspended { + summary_convergence::suspend_raw_update_guard(conn).await?; + guard_suspended = true; + } + let changed = apply_metadata(conn, candidate.store_id, &message, &metadata).await?; + if changed { + report.rows_compacted += 1; + report.metadata_bytes_removed += message + .metadata_json + .as_deref() + .map_or(0, str::len) + .saturating_sub(metadata.len()) + as u64; } else { report.rows_skipped += 1; } + } else { + report.rows_skipped += 1; } + } else { + // Eligibility refusal retains the original representation too. + report.rows_skipped += 1; } report.rows_scanned += 1; after = candidate.store_id; @@ -199,7 +279,25 @@ async fn step( if guard_suspended { summary_convergence::restore_raw_update_guard(conn).await?; } - schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; + if report.rows_scanned > 0 { + if after <= before { + return Err(db_error( + "replay metadata page did not advance its frontier", + )); + } + schema::set_gc_meta(conn, FRONTIER_KEY, &after.to_string()).await?; + // An advancing local key is not proof that its checkpoint was stored. + // Verify inside the same writer so a refused or rewritten checkpoint + // rolls back this page instead of committing a false progress report. + let persisted = frontier(conn).await?; + if persisted != after { + return Err(db_error(format!( + "replay metadata frontier was not persisted: expected {after}, found {persisted}" + ))); + } + report.frontier_before = Some(before); + report.frontier_after = Some(persisted); + } report.has_more = has_work(conn, after).await?; Ok(report) } diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs new file mode 100644 index 00000000000..d34b30fa231 --- /dev/null +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/read_cost.rs @@ -0,0 +1,197 @@ +//! Isolate process-wide read accounting from concurrently running unit tests. + +use std::process::Command; + +use tracedecay_privacy::sanitize_lcm_payload_text; +use tracedecay_runtime_core::db::engine::{ + Executor, QueryExecutor, TestConnection, TransactionBehavior, params, +}; + +use super::{MAX_BATCH_BYTES, ReplayMetadataCompactionReport, frontier, step}; +use crate::{raw, retrieval_content::projected_content_hash, schema, summary_convergence}; + +fn read_chars() -> u64 { + std::fs::read_to_string("/proc/self/io") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("rchar:")) + .unwrap() + .trim() + .parse() + .unwrap() +} + +#[tokio::test] +async fn oversized_rows_skip_payload_io() { + const CHILD: &str = "TRACEDECAY_REPLAY_READ_COST_CHILD"; + const TEST: &str = "retention::replay_metadata::read_cost::oversized_rows_skip_payload_io"; + if std::env::var_os(CHILD).is_none() { + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST, "--test-threads=1"]) + .env(CHILD, "1") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "isolated read-cost test failed:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(stdout.contains("1 passed; 0 failed"), "{stdout}"); + return; + } + + let directory = tempfile::tempdir().unwrap(); + let database_path = directory.path().join("replay-read-cost.db"); + let conn = TestConnection::open(&database_path); + conn.execute_batch( + "CREATE TABLE sessions ( + provider TEXT NOT NULL, session_id TEXT NOT NULL, + PRIMARY KEY(provider, session_id) + ); INSERT INTO sessions VALUES ('codex', 'session');", + ) + .await + .unwrap(); + schema::ensure_lcm_schema(&conn).await.unwrap(); + + let small = "verified replay after oversized rows ".repeat(80); + let sanitized = sanitize_lcm_payload_text(&small).unwrap(); + let metadata = serde_json::json!({ + "lcm_active_replay": true, + "active_replay": {"role": "assistant", "content": small}, + "ingest_protection": {"sanitization_receipt": sanitized.receipt()} + }); + let metadata = raw::resanitize_stored_provider_metadata_json(&metadata.to_string()).unwrap(); + let large = "x".repeat(2 * MAX_BATCH_BYTES as usize); + let padded = format!("{metadata}{}", " ".repeat(2 * MAX_BATCH_BYTES as usize)); + let large_hash = projected_content_hash(&large); + let small_hash = projected_content_hash(&small); + conn.execute( + "INSERT INTO session_canonical_bodies(content_hash, encoding, body, uncompressed_bytes) + VALUES (?1, 'identity', ?2, ?3)", + params![large_hash.as_str(), large.as_bytes(), large.len() as i64], + ) + .await + .unwrap(); + // Direct fixture writes represent oversized retained data, not new ingress. + // Each overflow location must be rejected before its bytes are loaded: + // inline body, metadata, then a separately stored canonical body. + let fixtures = [ + ( + 1_i64, + Some(large.as_str()), + large_hash.as_str(), + metadata.as_str(), + ), + ( + 2, + Some(small.as_str()), + small_hash.as_str(), + padded.as_str(), + ), + (3, None, large_hash.as_str(), metadata.as_str()), + ( + 4, + Some(small.as_str()), + small_hash.as_str(), + metadata.as_str(), + ), + ]; + for (id, content, hash, document) in fixtures { + conn.execute( + "INSERT INTO lcm_raw_messages(provider, message_id, session_id, role, ordinal, + content, content_hash, storage_kind, metadata_json) + VALUES ('codex', ?1, 'session', 'assistant', ?2, ?3, ?4, 'inline', ?5)", + params![format!("message-{id}"), id, content, hash, document], + ) + .await + .unwrap(); + } + let original = raw::load_raw_message_by_store_id(&conn, 4).await.unwrap(); + drop(conn); + + for id in 1..=3 { + // Fresh connections preserve the oversized-read measurement without + // issuing PRAGMAs forbidden by the exact-SQL test authority. + let conn = TestConnection::open(&database_path); + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let before = read_chars(); + let report = step(&transaction, 1).await.unwrap(); + transaction.commit().await.unwrap(); + let read = read_chars() - before; + assert!( + read < MAX_BATCH_BYTES, + "rejected row {id} read {read} bytes before admission; budget={MAX_BATCH_BYTES}" + ); + assert_eq!(report.rows_scanned, 1); + assert_eq!(report.rows_skipped_oversized, 1); + assert_eq!(report.rows_compacted, 0); + assert_eq!(report.metadata_bytes_examined, 0); + assert_eq!(report.frontier_before, Some(id - 1)); + assert_eq!(report.frontier_after, Some(id)); + assert_eq!(frontier(&conn).await.unwrap(), id); + assert!(report.has_more); + } + + let conn = TestConnection::open(&database_path); + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let report = step(&transaction, 48).await.unwrap(); + transaction.commit().await.unwrap(); + assert_eq!(report.rows_scanned, 1); + assert_eq!(report.rows_compacted, 1); + assert!(report.metadata_bytes_removed > 0); + assert_eq!(report.frontier_after, Some(4)); + assert!(!report.has_more); + let repaired = raw::load_raw_message_by_store_id(&conn, 4).await.unwrap(); + assert_eq!(repaired.content, original.content); + assert_eq!( + crate::replay_transactions::raw_replay_message(&repaired), + crate::replay_transactions::raw_replay_message(&original) + ); + summary_convergence::validate_raw_update_guard(&conn) + .await + .unwrap(); + + // Check complete retained bytes outside the measured interval. + for (id, content, _, document) in &fixtures[..3] { + let mut rows = conn + .query( + "SELECT content, metadata_json FROM lcm_raw_messages WHERE store_id = ?1", + params![*id], + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().unwrap(); + assert_eq!(row.get::>(0).unwrap().as_deref(), *content); + assert_eq!(row.get::(1).unwrap(), *document); + } + let mut rows = conn + .query("SELECT body FROM session_canonical_bodies", ()) + .await + .unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::>(0) + .unwrap(), + large.as_bytes() + ); + drop(rows); + let transaction = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + assert_eq!( + step(&transaction, 48).await.unwrap(), + ReplayMetadataCompactionReport::default() + ); + transaction.commit().await.unwrap(); +} diff --git a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs index 5a874358d64..bd0b372f6ed 100644 --- a/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs +++ b/crates/tracedecay-lcm/src/retention/replay_metadata/tests.rs @@ -250,6 +250,8 @@ async fn structured_different_media_and_external_representations_are_retained() let report = store.tick(48).await; assert_eq!(report.rows_scanned, 5); assert_eq!(report.rows_compacted, 0); + assert_eq!(report.rows_skipped, 5); + assert_eq!(report.rows_skipped_oversized, 0); assert_eq!(report.metadata_bytes_removed, 0); // All five provider representations still carry their original content. assert_eq!(scalar(&store.conn, "SELECT COUNT(*) FROM lcm_raw_messages WHERE json_type(metadata_json, '$.active_replay.content') IS NOT NULL").await, 5); @@ -348,14 +350,20 @@ async fn stale_metadata_cannot_be_overwritten_and_missing_guard_refuses_repair() async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_noops() { let store = Store::open().await; let text = "bounded replay text ".repeat(35_000); + let mut metadata_bytes = Vec::new(); for ordinal in 1..=4 { - store + let id = store .insert(ordinal, &text, Value::String(text.clone()), true) .await; + let message = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + metadata_bytes.push(message.metadata_json.unwrap().len() as u64); } let first = store.tick(1).await; assert_eq!(first.rows_scanned, 1); assert_eq!(first.rows_compacted, 1); + assert_eq!(first.metadata_bytes_examined, metadata_bytes[0]); assert!(first.has_more); assert_eq!(frontier(&store.conn).await.unwrap(), 1); let second = store.tick(48).await; @@ -364,8 +372,339 @@ async fn row_and_byte_budgets_advance_a_durable_frontier_and_finished_pages_are_ "byte cap stops before the third large row" ); assert!(second.has_more); + assert_eq!( + second.metadata_bytes_examined, + metadata_bytes[1..1 + second.rows_scanned as usize] + .iter() + .sum::(), + "the body budget must not pre-parse replay metadata for deferred rows" + ); let last = store.tick(48).await; + assert_eq!( + last.metadata_bytes_examined, + metadata_bytes[1 + second.rows_scanned as usize..] + .iter() + .sum::() + ); assert!(!last.has_more); let idle = store.tick(48).await; assert_eq!(idle, ReplayMetadataCompactionReport::default()); } + +#[tokio::test] +async fn zero_compaction_pages_advance_distinct_durable_frontiers() { + let store = Store::open().await; + for ordinal in 1..=52 { + store + .insert( + ordinal, + "canonical source", + Value::String("different replay".to_owned()), + false, + ) + .await; + } + let first = store.tick(48).await; + assert_eq!(first.rows_scanned, 48); + assert_eq!(first.rows_compacted, 0); + assert_eq!(first.rows_skipped, 48); + assert_eq!(first.rows_skipped_oversized, 0); + assert_eq!(first.frontier_before, Some(0)); + assert_eq!(first.frontier_after, Some(48)); + assert!(first.has_more); + + let second = store.tick(48).await; + assert_eq!(second.rows_scanned, 4); + assert_eq!(second.rows_compacted, 0); + assert_eq!(second.rows_skipped, 4); + assert_eq!(second.rows_skipped_oversized, 0); + assert_eq!(second.frontier_before, Some(48)); + assert_eq!(second.frontier_after, Some(52)); + assert!(!second.has_more); + assert_eq!(frontier(&store.conn).await.unwrap(), 52); + + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); +} + +#[tokio::test] +async fn large_ineligible_metadata_is_bounded_before_eligibility_checks() { + let store = Store::open().await; + let structured = serde_json::json!([{ + "type": "text", + "text": "retained structured replay ".repeat(20_000) + }]); + let mut originals = Vec::new(); + for ordinal in 1..=9 { + let id = store + .insert(ordinal, "small verified body", structured.clone(), false) + .await; + originals.push(( + id, + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + )); + } + let total_metadata_bytes: u64 = originals + .iter() + .map(|(_, message)| message.metadata_json.as_ref().unwrap().len() as u64) + .sum(); + assert!(total_metadata_bytes > MAX_BATCH_BYTES); + assert!( + originals + .iter() + .all(|(_, message)| { message.metadata_json.as_ref().unwrap().len() < 1024 * 1024 }) + ); + + let mut after = 0; + let mut examined = 0; + let mut scanned = 0; + let mut ticks = 0; + for _ in 0..originals.len() { + let report = store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await; + ticks += 1; + assert!(report.rows_scanned > 0); + assert_eq!(report.rows_compacted, 0); + assert_eq!(report.rows_skipped, report.rows_scanned); + assert_eq!(report.rows_skipped_oversized, 0); + assert_eq!(report.metadata_bytes_removed, 0); + assert!(report.metadata_bytes_examined > 0); + assert!(report.metadata_bytes_examined <= MAX_BATCH_BYTES); + assert_eq!(report.frontier_before, Some(after)); + let next = report.frontier_after.unwrap(); + assert!(next > after); + assert_eq!(frontier(&store.conn).await.unwrap(), next); + after = next; + examined += report.metadata_bytes_examined; + scanned += report.rows_scanned; + if !report.has_more { + break; + } + } + assert!(ticks > 1, "ineligible JSON must obey the byte budget too"); + assert_eq!(scanned, originals.len() as u64); + assert_eq!(examined, total_metadata_bytes, "no document is revisited"); + assert_eq!(after, originals.last().unwrap().0); + assert_eq!( + store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await, + ReplayMetadataCompactionReport::default() + ); + for (id, original) in originals { + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + original, + "bounded scanning must preserve the body, replay and receipts" + ); + } +} + +#[tokio::test] +async fn oversized_metadata_is_preserved_without_blocking_later_compaction() { + let store = Store::open().await; + let text = "verified duplicate content ".repeat(80); + let oversized_id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let next_id = store + .insert(2, &text, Value::String(text.clone()), false) + .await; + // Bypass ingress only to exercise a pre-existing oversized stored document. + // The repair must not parse or rewrite it, nor pin the durable frontier. + let oversized = serde_json::json!({ + "lcm_active_replay": true, + "active_replay": {"content": text}, + "padding": "x".repeat(MAX_BATCH_BYTES as usize) + }) + .to_string(); + store + .conn + .execute( + "UPDATE lcm_raw_messages SET metadata_json = ?1 WHERE store_id = ?2", + params![oversized.as_str(), oversized_id], + ) + .await + .unwrap(); + let report = store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await; + assert_eq!(report.rows_scanned, 2); + assert_eq!(report.rows_skipped, 1); + assert_eq!(report.rows_skipped_oversized, 1); + assert_eq!(report.rows_compacted, 1); + assert!(report.metadata_bytes_examined > 0); + assert!(report.metadata_bytes_examined < MAX_BATCH_BYTES); + assert_eq!(report.frontier_after, Some(next_id)); + assert_eq!(frontier(&store.conn).await.unwrap(), next_id); + assert!(!report.has_more); + let mut rows = store + .conn + .query( + "SELECT metadata_json FROM lcm_raw_messages WHERE store_id = ?1", + params![oversized_id], + ) + .await + .unwrap(); + assert_eq!( + rows.next() + .await + .unwrap() + .unwrap() + .get::(0) + .unwrap(), + oversized + ); + drop(rows); + assert_eq!( + store.tick(MAX_REPLAY_METADATA_COMPACTION_ROWS).await, + ReplayMetadataCompactionReport::default() + ); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .unwrap(); +} + +async fn assert_checkpoint_refusal_is_atomic(store: &Store, id: i64, trigger_sql: &str) { + let checkpoint = schema::get_gc_meta(&store.conn, FRONTIER_KEY) + .await + .unwrap(); + let before = frontier(&store.conn).await.unwrap(); + let original = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + assert!(compacted_metadata(&original).unwrap().is_some()); + let revision = scalar( + &store.conn, + "SELECT raw_revision_generation FROM lcm_summary_convergence_queue", + ) + .await; + store.conn.execute_batch(trigger_sql).await.unwrap(); + let transaction = store + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .unwrap(); + let error = step(&transaction, 48).await.unwrap_err(); + assert!(error.to_string().contains("frontier was not persisted")); + let staged = raw::load_raw_message_by_store_id(&transaction, id) + .await + .unwrap(); + assert_ne!(staged.metadata_json, original.metadata_json); + assert!(metadata(&staged)["active_replay"].get("content").is_none()); + transaction.rollback().await.unwrap(); + + assert_eq!( + schema::get_gc_meta(&store.conn, FRONTIER_KEY) + .await + .unwrap(), + checkpoint, + "rollback preserves even an absent checkpoint" + ); + assert_eq!( + raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(), + original, + "a refused checkpoint must restore the complete message and receipts" + ); + assert_eq!( + scalar( + &store.conn, + "SELECT raw_revision_generation FROM lcm_summary_convergence_queue" + ) + .await, + revision + ); + summary_convergence::validate_raw_update_guard(&store.conn) + .await + .unwrap(); + + store + .conn + .execute_batch("DROP TRIGGER refuse_replay_checkpoint") + .await + .unwrap(); + let recovered = store.tick(48).await; + assert_eq!(recovered.rows_scanned, 1); + assert_eq!(recovered.rows_compacted, 1); + assert!(recovered.metadata_bytes_removed > 0); + assert_eq!(recovered.frontier_before, Some(before)); + assert_eq!(recovered.frontier_after, Some(id)); + assert_eq!(frontier(&store.conn).await.unwrap(), id); + assert!(!recovered.has_more); + let repaired = raw::load_raw_message_by_store_id(&store.conn, id) + .await + .unwrap(); + assert_eq!(repaired.content, original.content); + assert_eq!(repaired.content_hash, original.content_hash); + assert_eq!( + crate::replay_transactions::raw_replay_message(&repaired), + crate::replay_transactions::raw_replay_message(&original) + ); + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); +} + +#[tokio::test] +async fn ignored_initial_checkpoint_rolls_back_compaction() { + let store = Store::open().await; + let text = "verified replay checkpoint content ".repeat(80); + let id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let trigger = format!( + "CREATE TRIGGER refuse_replay_checkpoint BEFORE INSERT ON lcm_gc_meta + WHEN NEW.key = '{FRONTIER_KEY}' BEGIN SELECT RAISE(IGNORE); END" + ); + assert_checkpoint_refusal_is_atomic(&store, id, &trigger).await; +} + +#[tokio::test] +async fn rewritten_checkpoint_cannot_claim_durable_progress() { + let store = Store::open().await; + let text = "verified replay checkpoint content ".repeat(80); + let previous_id = store + .insert(1, &text, Value::String(text.clone()), false) + .await; + let first = store.tick(48).await; + assert_eq!(first.rows_compacted, 1); + assert_eq!(first.frontier_after, Some(previous_id)); + let id = store + .insert(2, &text, Value::String(text.clone()), false) + .await; + // A successful INSERT can still leave a different checkpoint behind. + // Counting affected rows alone cannot detect this failure. + let trigger = format!( + "CREATE TRIGGER refuse_replay_checkpoint AFTER INSERT ON lcm_gc_meta + WHEN NEW.key = '{FRONTIER_KEY}' BEGIN + UPDATE lcm_gc_meta SET value = '{previous_id}' WHERE key = NEW.key; + END" + ); + assert_checkpoint_refusal_is_atomic(&store, id, &trigger).await; +} + +#[tokio::test] +async fn idle_page_does_not_initialize_a_checkpoint() { + let store = Store::open().await; + let trigger = format!( + "CREATE TRIGGER refuse_replay_checkpoint BEFORE INSERT ON lcm_gc_meta + WHEN NEW.key = '{FRONTIER_KEY}' BEGIN + SELECT RAISE(ABORT, 'idle page attempted a checkpoint write'); + END" + ); + store.conn.execute_batch(&trigger).await.unwrap(); + assert_eq!( + store.tick(48).await, + ReplayMetadataCompactionReport::default() + ); + assert_eq!( + schema::get_gc_meta(&store.conn, FRONTIER_KEY) + .await + .unwrap(), + None + ); +} diff --git a/crates/tracedecay-lcm/src/summary_convergence_tests.rs b/crates/tracedecay-lcm/src/summary_convergence_tests.rs index 18456d731ca..58478b67998 100644 --- a/crates/tracedecay-lcm/src/summary_convergence_tests.rs +++ b/crates/tracedecay-lcm/src/summary_convergence_tests.rs @@ -52,6 +52,17 @@ impl Executor for CountingExecutor<'_> { self.executes.set(self.executes.get() + 1); self.inner.execute_batch(sql).await } + + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.executes.set(self.executes.get() + statements.len()); + let results = self.inner.execute_statements(statements).await?; + let changed = results.iter().copied().sum::(); + self.rows_changed.set(self.rows_changed.get() + changed); + Ok(results) + } } async fn fetch_i64(conn: &TestConnection, sql: &str, params: impl IntoParams) -> i64 { diff --git a/crates/tracedecay-runtime-core/src/db/connection/facade.rs b/crates/tracedecay-runtime-core/src/db/connection/facade.rs index 769324b242c..d8448e7dd32 100644 --- a/crates/tracedecay-runtime-core/src/db/connection/facade.rs +++ b/crates/tracedecay-runtime-core/src/db/connection/facade.rs @@ -80,6 +80,13 @@ impl DatabaseEngineWriteConnection { self.conn.execute_batch(sql).await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.conn.execute_statements(statements).await + } + pub(crate) async fn authorized_long_lease_transaction( &self, ) -> crate::db::engine::Result { @@ -114,6 +121,13 @@ impl crate::db::engine::Executor for DatabaseEngineWriteConnection { DatabaseEngineWriteConnection::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseEngineWriteConnection::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { DatabaseEngineWriteConnection::execute_batch(self, sql).await } @@ -257,6 +271,13 @@ impl DatabaseEngineLongLeaseTransaction { pub(crate) async fn rollback(self) -> crate::db::engine::Result<()> { self.transaction.rollback().await } + + pub(crate) async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.transaction.execute_statements(statements).await + } } impl crate::db::engine::QueryExecutor for DatabaseEngineLongLeaseTransaction { @@ -280,6 +301,13 @@ impl crate::db::engine::Executor for DatabaseEngineLongLeaseTransaction { self.transaction.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseEngineLongLeaseTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { self.transaction.execute_batch(sql).await } @@ -329,6 +357,18 @@ impl<'a> DatabaseMemoryTransaction<'a> { } } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + match self { + Self::Read(_) => Err(crate::db::engine::Error::Runtime( + "cannot execute a write in a memory read snapshot".to_owned(), + )), + Self::Write(transaction) => transaction.execute_statements(statements).await, + } + } + pub async fn commit_with_on_commit(self, on_commit: F) -> Result<()> where F: FnOnce() + Send + 'static, @@ -393,6 +433,13 @@ impl crate::db::engine::Executor for DatabaseMemoryTransaction<'_> { DatabaseMemoryTransaction::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseMemoryTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { DatabaseMemoryTransaction::execute_batch(self, sql).await } @@ -421,6 +468,18 @@ impl DatabaseWriteTransaction<'_> { self.transaction.execute_batch(sql).await } + /// Submits owned writes in one writer admission. + /// + /// The wrapper must forward to the engine transaction's batched + /// submission; a per-statement loop here would turn a bounded page into + /// one writer admission per statement. + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.transaction.execute_statements(statements).await + } + pub async fn execute_batch_engine(&self, sql: &str) -> crate::db::engine::Result<()> { self.transaction.execute_batch(sql).await } @@ -521,6 +580,13 @@ impl crate::db::engine::Executor for DatabaseWriteTransaction<'_> { self.execute_engine(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + DatabaseWriteTransaction::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { self.execute_batch_engine(sql).await } diff --git a/crates/tracedecay-runtime-core/src/db/connection/query_write.rs b/crates/tracedecay-runtime-core/src/db/connection/query_write.rs index 911c55c1f80..21b451ac4a5 100644 --- a/crates/tracedecay-runtime-core/src/db/connection/query_write.rs +++ b/crates/tracedecay-runtime-core/src/db/connection/query_write.rs @@ -114,6 +114,27 @@ impl Database { .await? } + /// Executes owned parameterized writes in one brokered writer job. + #[doc(hidden)] + pub async fn execute_write_statements( + &self, + operation: &str, + statements: Vec, + ) -> Result> { + let operation_name = operation.to_owned(); + self.transact(operation, async move |tx| { + let operation_name = operation_name.clone(); + let statements = statements.clone(); + tx.execute_statements(statements) + .await + .map_err(|error| TraceDecayError::Database { + message: format!("failed to execute brokered write statements: {error}"), + operation: operation_name, + }) + }) + .await? + } + /// Executes one authority-revalidated schema or bulk-maintenance batch /// through the canonical long-lease writer path. /// diff --git a/crates/tracedecay-runtime-core/src/db/engine/executor.rs b/crates/tracedecay-runtime-core/src/db/engine/executor.rs index 2740ff8ada9..2327e007d4a 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/executor.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/executor.rs @@ -53,21 +53,11 @@ pub trait Executor: QueryExecutor { P: IntoParams; /// Executes owned parameterized writes in input order. /// - /// Concrete engine connections and transactions override this with one - /// blocking submission. The default preserves compatibility for narrow - /// executor adapters while retaining exact failed-statement attribution. - async fn execute_statements(&self, statements: Vec) -> Result> { - let mut results = Vec::with_capacity(statements.len()); - for (index, statement) in statements.into_iter().enumerate() { - let (sql, params) = statement.into_parts(); - results.push( - self.execute(&sql, params) - .await - .map_err(|error| Error::statement_batch(index, error))?, - ); - } - Ok(results) - } + /// There is deliberately no default: an adapter over a batching executor + /// must forward to it, and a silent per-statement fallback would turn a + /// bounded page into one writer admission per statement. Implementations + /// whose backend cannot batch call [`execute_statements_serially`]. + async fn execute_statements(&self, statements: Vec) -> Result>; async fn execute_batch(&self, sql: &str) -> Result<()>; } @@ -105,3 +95,28 @@ impl Executor for Transaction { Transaction::execute_batch(self, sql).await } } + +/// Executes `statements` in input order with one statement per call. +/// +/// Only for executors whose backend has no batched submission. An executor +/// that wraps a batching implementation must forward to it instead, because +/// this helper admits the writer once per statement. +pub async fn execute_statements_serially( + executor: &E, + statements: Vec, +) -> Result> +where + E: Executor + ?Sized, +{ + let mut results = Vec::with_capacity(statements.len()); + for (index, statement) in statements.into_iter().enumerate() { + let (sql, params) = statement.into_parts(); + results.push( + executor + .execute(&sql, params) + .await + .map_err(|error| Error::statement_batch(index, error))?, + ); + } + Ok(results) +} diff --git a/crates/tracedecay-runtime-core/src/db/engine/mod.rs b/crates/tracedecay-runtime-core/src/db/engine/mod.rs index f464ee92fcc..786518e36a9 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/mod.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/mod.rs @@ -14,7 +14,7 @@ mod write_statement; pub use connection::{Connection, ReadConnection, ReaderPoolSnapshot, ReaderPoolState}; pub use error::{Error, Result}; -pub use executor::{Executor, QueryExecutor}; +pub use executor::{Executor, QueryExecutor, execute_statements_serially}; pub use params::{IntoParams, IntoValue, Params, params, params_from_iter}; pub use row::{Row, Rows}; pub use snapshot::ReadSnapshot; diff --git a/crates/tracedecay-runtime-core/src/db/engine/test_support.rs b/crates/tracedecay-runtime-core/src/db/engine/test_support.rs index 79cb11247b4..0450f982cfe 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/test_support.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/test_support.rs @@ -131,6 +131,13 @@ impl Executor for TestConnection { Connection::execute(&self.connection, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + Connection::execute_statements(&self.connection, statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { Connection::execute_batch(&self.connection, sql).await } diff --git a/crates/tracedecay-runtime-core/src/db/engine/transaction.rs b/crates/tracedecay-runtime-core/src/db/engine/transaction.rs index 0c3f87a5c58..6e64d8e34e1 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/transaction.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/transaction.rs @@ -221,6 +221,10 @@ impl Executor for WriteTx<'_, '_> { Ok(self.job.execute(statement(sql, params)?)?.changed_rows as u64) } + async fn execute_statements(&self, statements: Vec) -> Result> { + super::execute_statements_serially(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> Result<()> { self.job.execute_batch(sql.to_owned())?; Ok(()) diff --git a/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs b/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs index fb52ee4576d..6a79649eebd 100644 --- a/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs +++ b/crates/tracedecay-runtime-core/src/db/engine/write_statement.rs @@ -6,6 +6,7 @@ use super::{IntoParams, Result, Value}; /// /// Construction performs the same request validation as scalar execution, so /// a malformed statement is attributed before any batch reaches the writer. +#[derive(Clone)] pub struct WriteStatement { exact: ExactSqlStatement, } diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 701629d6f01..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -279,34 +279,21 @@ impl FormattedTracingCapture { /// before the first captured event. #[must_use] pub fn start() -> Self { - Self::start_with_output(false, false) + Self::start_with_output(false) } /// Also writes each line to stderr so a terminated measurement retains /// the events emitted before its deadline. #[must_use] pub fn start_with_stderr() -> Self { - Self::start_with_output(true, false) + Self::start_with_output(true) } - /// Captures span admission and completion timings as well as events, - /// retaining live stderr if the measurement is terminated before reporting. - #[must_use] - pub fn start_with_span_timings() -> Self { - Self::start_with_output(true, true) - } - - fn start_with_output(stderr: bool, span_timings: bool) -> Self { + fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_ansi(false) - .with_span_events(if span_timings { - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE - } else { - tracing_subscriber::fmt::format::FmtSpan::NONE - }) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { set_tracing_capture( diff --git a/crates/tracedecay-runtime-core/src/ports.rs b/crates/tracedecay-runtime-core/src/ports.rs index 442c0b5b6dc..140f570ebe7 100644 --- a/crates/tracedecay-runtime-core/src/ports.rs +++ b/crates/tracedecay-runtime-core/src/ports.rs @@ -96,6 +96,13 @@ pub mod registered_schema { self.connection.execute_batch(sql).await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.connection.execute_statements(statements).await + } + /// Begins the authority-bound transaction used for one atomic schema /// installation. /// @@ -154,6 +161,13 @@ pub mod registered_schema { RegisteredSchemaInstallationV1::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + RegisteredSchemaInstallationV1::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { RegisteredSchemaInstallationV1::execute_batch(self, sql).await } @@ -184,6 +198,13 @@ pub mod registered_schema { .await } + pub async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + self.transaction.execute_statements(statements).await + } + pub async fn commit(self) -> crate::db::engine::Result<()> { self.transaction.commit().await } @@ -214,6 +235,13 @@ pub mod registered_schema { RegisteredSchemaInstallationTransactionV1::execute(self, sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + RegisteredSchemaInstallationTransactionV1::execute_statements(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { RegisteredSchemaInstallationTransactionV1::execute_batch(self, sql).await } diff --git a/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs b/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs index 995d86eb62b..037a97126c6 100644 --- a/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs +++ b/crates/tracedecay-runtime-core/src/sqlite_snapshot_connection.rs @@ -111,6 +111,13 @@ impl Executor for SnapshotConnection { .map_err(|error| EngineError::Runtime(format!("snapshot execute task failed: {error}")))? } + async fn execute_statements( + &self, + statements: Vec, + ) -> crate::db::engine::Result> { + crate::db::engine::execute_statements_serially(self, statements).await + } + async fn execute_batch(&self, sql: &str) -> crate::db::engine::Result<()> { let cx = crate::native_runtime::handle().ok_or(EngineError::RuntimeContextUnavailable)?; let connection = Arc::clone(&self.connection); diff --git a/crates/tracedecay-session-temporal-store/src/test_support.rs b/crates/tracedecay-session-temporal-store/src/test_support.rs index 17aac90ce64..412099bb829 100644 --- a/crates/tracedecay-session-temporal-store/src/test_support.rs +++ b/crates/tracedecay-session-temporal-store/src/test_support.rs @@ -48,6 +48,13 @@ impl Executor for QueryCountingConnection<'_, T> { self.inner.execute(sql, params).await } + async fn execute_statements( + &self, + statements: Vec, + ) -> EngineResult> { + self.inner.execute_statements(statements).await + } + async fn execute_batch(&self, sql: &str) -> EngineResult<()> { self.inner.execute_batch(sql).await } diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 7318fd570bd..4b83d4cdf58 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -530,8 +530,10 @@ async fn run_registered_store_retention( Ok(_) => {} } log_payload_gc_outcome(&report.payload_gc); - let canonical_body_outcome = record_canonical_body_gc_outcome(&report.canonical_body_gc); - let replay_metadata_outcome = record_replay_metadata_outcome(&report.replay_metadata); + let canonical_body_outcome = + record_canonical_body_gc_outcome(database.db_path(), &report.canonical_body_gc); + let replay_metadata_outcome = + record_replay_metadata_outcome(database.db_path(), &report.replay_metadata); let outcome = if report.succeeded() { session_storage_outcome .combine(canonical_body_outcome) @@ -672,12 +674,19 @@ async fn run_registered_storage_continuation( .await; let terminal = run_registered_terminal_housekeeping(database).await; record_session_lcm_outcome(&session_lcm) - .combine(record_canonical_body_gc_outcome(&canonical_body_gc)) - .combine(record_replay_metadata_outcome(&replay_metadata)) + .combine(record_canonical_body_gc_outcome( + database.db_path(), + &canonical_body_gc, + )) + .combine(record_replay_metadata_outcome( + database.db_path(), + &replay_metadata, + )) .combine(terminal) } fn record_canonical_body_gc_outcome( + store: &Path, outcome: &Result< tracedecay_global_db::observation::CanonicalBodyGcReport, tracedecay_maintenance::retention::registered_store::RegisteredStoreRetentionErrorV1, @@ -685,12 +694,39 @@ fn record_canonical_body_gc_outcome( ) -> MaintenanceTickOutcome { match outcome { Ok(report) => { - if report.roots_scanned > 0 || report.bodies_scanned > 0 { + if report.has_committed_work() { log_daemon_event( "retention_canonical_bodies", &[ + ("store", store.display().to_string()), ("roots_scanned", report.roots_scanned.to_string()), ("bodies_scanned", report.bodies_scanned.to_string()), + ( + "bootstrap_frontier_before", + report + .bootstrap_frontier_before + .as_deref() + .unwrap_or("not_scanned") + .to_owned(), + ), + ( + "bootstrap_frontier_after", + report + .bootstrap_frontier_after + .as_deref() + .unwrap_or("not_scanned") + .to_owned(), + ), + ( + "elapsed_ms", + report + .elapsed_ms + .map_or_else(|| "unmeasured".to_owned(), |value| value.to_string()), + ), + ( + "candidates_discharged", + report.candidates_discharged.to_string(), + ), ("bodies_deleted", report.bodies_deleted.to_string()), ( "stored_bytes_deleted", @@ -712,6 +748,7 @@ fn record_canonical_body_gc_outcome( "retention_degraded", &[ ("pass", "canonical_body_gc".to_owned()), + ("store", store.display().to_string()), ("failure", error.diagnostic().to_owned()), ], ); @@ -721,6 +758,7 @@ fn record_canonical_body_gc_outcome( } fn record_replay_metadata_outcome( + store: &Path, outcome: &Result< tracedecay_lcm::retention::ReplayMetadataCompactionReport, tracedecay_maintenance::retention::registered_store::RegisteredStoreRetentionErrorV1, @@ -732,7 +770,32 @@ fn record_replay_metadata_outcome( log_daemon_event( "retention_replay_metadata", &[ + ("store", store.display().to_string()), ("rows_scanned", report.rows_scanned.to_string()), + ( + "elapsed_ms", + report + .elapsed_ms + .map_or_else(|| "unmeasured".to_owned(), |value| value.to_string()), + ), + ( + "metadata_bytes_examined", + report.metadata_bytes_examined.to_string(), + ), + ( + "frontier_before", + report.frontier_before.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), + ( + "frontier_after", + report.frontier_after.map_or_else( + || "not_scanned".to_owned(), + |value| value.to_string(), + ), + ), ("rows_compacted", report.rows_compacted.to_string()), ("rows_skipped", report.rows_skipped.to_string()), ( @@ -758,6 +821,7 @@ fn record_replay_metadata_outcome( "retention_degraded", &[ ("pass", "replay_metadata_compaction".to_owned()), + ("store", store.display().to_string()), ("failure", error.diagnostic().to_owned()), ], ); @@ -3454,4 +3518,82 @@ mod tests { .expect("native runtime"); assert_eq!(crate::test_support::native_runtime_hazards(), hazards); } + + /// Two mounted databases may commit identical 48-row pages in one tick. + /// Operator logs must not conflate their independent SQL frontiers. + #[test] + fn retention_progress_logs_distinguish_mounted_stores() { + const CHILD: &str = "TRACEDECAY_RETENTION_STORE_LOG_CHILD"; + const TEST: &str = + "daemon::maintenance::tests::retention_progress_logs_distinguish_mounted_stores"; + let stores = ["/project-a/sessions.db", "/project-b/sessions.db"]; + + if std::env::var_os(CHILD).is_some() { + for store in stores { + let canonical = tracedecay_global_db::observation::CanonicalBodyGcReport { + roots_scanned: 48, + bootstrap_frontier_before: Some("raw:0".to_owned()), + bootstrap_frontier_after: Some("raw:48".to_owned()), + has_more: true, + ..Default::default() + }; + let _ = super::record_canonical_body_gc_outcome(Path::new(store), &Ok(canonical)); + let replay = tracedecay_lcm::retention::ReplayMetadataCompactionReport { + rows_scanned: 48, + rows_skipped: 48, + frontier_before: Some(0), + frontier_after: Some(48), + has_more: true, + ..Default::default() + }; + let _ = super::record_replay_metadata_outcome(Path::new(store), &Ok(replay)); + } + return; + } + + // Capture the actual operator stderr lines in a child process. The + // normal daemon logger deliberately bypasses the test output capture. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST, "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output() + .expect("spawn isolated operator log test"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success() && stdout.contains("1 passed; 0 failed"), + "isolated test did not execute:\n{stdout}\n{stderr}" + ); + + let lines = stderr + .lines() + .filter(|line| { + line.contains("event=retention_canonical_bodies ") + || line.contains("event=retention_replay_metadata ") + }) + .collect::>(); + assert_eq!(lines.len(), 4, "expected two events per store: {stderr}"); + for store in stores { + for (event, frontier) in [ + ( + "retention_canonical_bodies", + "bootstrap_frontier_after=raw:48", + ), + ("retention_replay_metadata", "frontier_after=48"), + ] { + let count = lines + .iter() + .filter(|line| { + line.contains(&format!("event={event} ")) + && line.contains(&format!("store={store} ")) + && line.contains(frontier) + }) + .count(); + assert_eq!( + count, 1, + "missing distinct {event} page for {store}: {stderr}" + ); + } + } + } } diff --git a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs index d45218c298a..405d6adfe2f 100644 --- a/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/drop_shutdown_journey_test.rs @@ -185,7 +185,7 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() .expect("refused store root"); std::fs::write(&refused_graph, b"not a project graph database").expect("refused graph"); - let error = match ProductionProjectCompositionHarnessV1::open( + let error = match ProductionProjectCompositionHarnessV1::open_for_session_retrieval( isolation.path(), [first.clone(), refused.clone()], ) @@ -209,7 +209,7 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() let stores = [ profile.join("global.db"), profile.join("user-sessions.db"), - first_layout.data_root.join("sessions.db"), + first_layout.sessions_db_path, first_layout.graph_db_path, ]; for store in &stores { @@ -231,14 +231,23 @@ async fn a_failed_later_mount_releases_the_whole_composition_before_returning() } std::fs::remove_file(&refused_graph).expect("remove the fixture refusal"); - let harness = ProductionProjectCompositionHarnessV1::open( + let harness = ProductionProjectCompositionHarnessV1::open_for_session_retrieval( isolation.path(), [first.clone(), refused.clone()], ) .await .expect("both exact projects reopen after failed-open rollback"); - assert!(harness.server(&first).is_ok()); - assert!(harness.server(&refused).is_ok()); + for project in [&first, &refused] { + let server = harness.server(project).expect("reopened project server"); + let store = server + .project_session_db() + .expect("mounted project sessions"); + let layout = default_profile_sharded_layout(project, &profile).unwrap(); + assert_eq!( + store.db_path().canonicalize().unwrap(), + layout.sessions_db_path.canonicalize().unwrap() + ); + } harness.shutdown().await; } diff --git a/crates/tracedecay/src/mcp/server/connection.rs b/crates/tracedecay/src/mcp/server/connection.rs index 2852792169e..6c0e02a3746 100644 --- a/crates/tracedecay/src/mcp/server/connection.rs +++ b/crates/tracedecay/src/mcp/server/connection.rs @@ -666,7 +666,7 @@ mod cancellable_queue_tests { async fn receive_response( responses: &mut tokio::sync::mpsc::UnboundedReceiver, - ) -> Value { + ) -> std::result::Result { tokio::time::timeout(Duration::from_secs(5), async { loop { let line = responses.recv().await.expect("connection response"); @@ -678,7 +678,6 @@ mod cancellable_queue_tests { } }) .await - .expect("connection response timeout") } #[tokio::test] @@ -727,7 +726,9 @@ mod cancellable_queue_tests { ) .expect("send independent status read"); - let fast = receive_response(&mut responses).await; + let fast = receive_response(&mut responses) + .await + .expect("independent status response"); assert_eq!( fast["id"], serde_json::json!(2), @@ -735,7 +736,9 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); - let slow = receive_response(&mut responses).await; + let slow = receive_response(&mut responses) + .await + .expect("delayed selected-read response"); assert!( slow["result"]["content"].is_array(), "selected fact read: {slow}" @@ -843,7 +846,12 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); - assert_eq!(receive_response(&mut responses).await["id"], json!(10)); + assert_eq!( + receive_response(&mut responses) + .await + .expect("read-before-effect response")["id"], + json!(10) + ); fixture.wait_for_routes(2).await; assert!( tokio::time::timeout(Duration::from_millis(50), responses.recv()) @@ -853,12 +861,16 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); assert_eq!( - receive_response(&mut responses).await["id"], + receive_response(&mut responses) + .await + .expect("effect-barrier response")["id"], json!(11), "the effect must settle after the preceding read" ); assert_eq!( - receive_response(&mut responses).await["id"], + receive_response(&mut responses) + .await + .expect("read-after-effect response")["id"], json!(12), "the later read must not overtake the effect" ); @@ -915,7 +927,9 @@ mod cancellable_queue_tests { ); fixture.route_release.add_permits(1); - let _ = receive_response(&mut responses).await; + let _ = receive_response(&mut responses) + .await + .expect("first admitted-read response"); fixture .wait_for_routes(MAX_CONCURRENT_CONNECTION_READS + 1) .await; @@ -925,7 +939,12 @@ mod cancellable_queue_tests { let mut response_ids = HashSet::new(); while response_ids.len() < MAX_CONCURRENT_CONNECTION_READS { - response_ids.insert(receive_response(&mut responses).await["id"].clone()); + response_ids.insert( + receive_response(&mut responses) + .await + .expect("remaining admitted-read response")["id"] + .clone(), + ); } assert_eq!( response_ids.len(),