From 456a5e95f53368468edbfc0cde710a9684709de4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:42:48 -0800 Subject: [PATCH 01/85] fix(storage): reject unreadable profile registries Refs #4099 --- .../src/retention/storage_report.rs | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 84e56e9fbbb..0159172759c 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -606,10 +606,14 @@ async fn build_storage_report_with_walk_budget( let global_db_path = profile_root.join(GLOBAL_DB_FILENAME); // The registry rows come from an off-profile copy, so reading them first // leaves the profile untouched. - let registered_projects = if global_db_path.exists() { - load_registered_project_rows(profile_root, &global_db_path).await? - } else { - Vec::new() + // `Path::exists` hides permission errors and dangling symlinks as + // absence. That would misclassify every registered store as unregistered. + let registered_projects = match std::fs::symlink_metadata(&global_db_path) { + Ok(_) => load_registered_project_rows(profile_root, &global_db_path).await?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => { + return Err(report_error("inspect profile registry for storage report", error)); + } }; // Capture the profile's physical footprint before opening any SQLite // readers. A report must describe the profile supplied by its caller, not @@ -2446,6 +2450,38 @@ mod tests { ); } + #[cfg(unix)] + #[asupersync::test] + async fn dangling_profile_registry_cannot_make_registered_stores_look_unregistered() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + let project = profile_root.join("projects/proj_retained"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("sessions.db"), b"retained sessions").unwrap(); + + let registry = profile_root.join(GLOBAL_DB_FILENAME); + std::os::unix::fs::symlink(temporary.path().join("missing-global.db"), ®istry) + .unwrap(); + assert!(std::fs::symlink_metadata(®istry).is_ok()); + assert!(!registry.exists(), "the old existence test silently lost this registry"); + + let error = build_storage_report(&profile_root) + .await + .expect_err("a broken registry link must not imply zero registered projects"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "the unreadable registry must have a typed storage failure: {error}" + ); + + std::fs::remove_file(®istry).unwrap(); + let absent = build_storage_report(&profile_root) + .await + .expect("a genuinely absent registry is still reportable"); + assert!(absent.stores.is_empty()); + assert_eq!(absent.unregistered_dir_count, 1); + assert!(absent.unregistered_bytes >= 17); + } + #[asupersync::test] async fn storage_report_names_user_sessions_db_family() { let tmp = tempfile::TempDir::new().unwrap(); From 6df1dce6f9cf5e65c13f1b9ae080eac9c1dac192 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:12:25 -0800 Subject: [PATCH 02/85] style(storage): format registry-report regression Refs #4099 --- .../src/retention/storage_report.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 0159172759c..2d1a8bda2ac 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -612,7 +612,10 @@ async fn build_storage_report_with_walk_budget( Ok(_) => load_registered_project_rows(profile_root, &global_db_path).await?, Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), Err(error) => { - return Err(report_error("inspect profile registry for storage report", error)); + return Err(report_error( + "inspect profile registry for storage report", + error, + )); } }; // Capture the profile's physical footprint before opening any SQLite @@ -2460,10 +2463,12 @@ mod tests { std::fs::write(project.join("sessions.db"), b"retained sessions").unwrap(); let registry = profile_root.join(GLOBAL_DB_FILENAME); - std::os::unix::fs::symlink(temporary.path().join("missing-global.db"), ®istry) - .unwrap(); + std::os::unix::fs::symlink(temporary.path().join("missing-global.db"), ®istry).unwrap(); assert!(std::fs::symlink_metadata(®istry).is_ok()); - assert!(!registry.exists(), "the old existence test silently lost this registry"); + assert!( + !registry.exists(), + "the old existence test silently lost this registry" + ); let error = build_storage_report(&profile_root) .await From d2ea4f5621a26ae0eae766c2d33f52b2445b025d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:13:41 -0800 Subject: [PATCH 03/85] fix(storage): refuse linked foreign registries in footprint reports Refs #4099 --- .../src/retention/storage_report.rs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 2d1a8bda2ac..45dc45fd2d1 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -607,8 +607,15 @@ async fn build_storage_report_with_walk_budget( // The registry rows come from an off-profile copy, so reading them first // leaves the profile untouched. // `Path::exists` hides permission errors and dangling symlinks as - // absence. That would misclassify every registered store as unregistered. + // absence. A valid symlink is also not a profile-owned registry: following + // it would read a foreign database the footprint walker does not charge. let registered_projects = match std::fs::symlink_metadata(&global_db_path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(report_error( + "inspect profile registry for storage report", + "registry path is a symbolic link; its target is outside the measured registry authority", + )); + } Ok(_) => load_registered_project_rows(profile_root, &global_db_path).await?, Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), Err(error) => { @@ -2487,6 +2494,33 @@ mod tests { assert!(absent.unregistered_bytes >= 17); } + #[cfg(unix)] + #[asupersync::test] + async fn linked_foreign_registry_is_not_misattributed_to_the_profile() { + let temporary = tempfile::TempDir::new().unwrap(); + let foreign_root = temporary.path().join("foreign"); + std::fs::create_dir_all(&foreign_root).unwrap(); + seed_global_db(&foreign_root, &[("proj_foreign", "/repos/foreign")]).await; + + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let registry = profile_root.join(GLOBAL_DB_FILENAME); + std::os::unix::fs::symlink(foreign_root.join(GLOBAL_DB_FILENAME), ®istry).unwrap(); + assert!(registry.is_file(), "the old path probe follows the foreign registry"); + + let error = build_storage_report(&profile_root) + .await + .expect_err("a foreign registry cannot supply project identity for this profile"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "the symlink must be refused as a typed storage error: {error}" + ); + assert!( + error.to_string().contains("symbolic link"), + "the error must name the profile registry's disallowed link: {error}" + ); + } + #[asupersync::test] async fn storage_report_names_user_sessions_db_family() { let tmp = tempfile::TempDir::new().unwrap(); From 43cef92bc121f1bf9b451de1a3b92c8e40fd624f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:28:14 -0800 Subject: [PATCH 04/85] style(storage): format linked-registry refusal test --- .../tracedecay-maintenance/src/retention/storage_report.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 45dc45fd2d1..b8653747322 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -2506,7 +2506,10 @@ mod tests { std::fs::create_dir_all(&profile_root).unwrap(); let registry = profile_root.join(GLOBAL_DB_FILENAME); std::os::unix::fs::symlink(foreign_root.join(GLOBAL_DB_FILENAME), ®istry).unwrap(); - assert!(registry.is_file(), "the old path probe follows the foreign registry"); + assert!( + registry.is_file(), + "the old path probe follows the foreign registry" + ); let error = build_storage_report(&profile_root) .await From 8b403d1eeaa2245eae042ade7068fab6671c10bb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:47:57 -0800 Subject: [PATCH 05/85] fix(storage): stream table growth census without page sorting Refs #4099. Aggregate each table's dbstat payload inside a correlated subquery instead of materializing all pages for GROUP BY. Preserve the existing reader snapshot, cancellation, deadline and typed-error path. Add nonzero filesystem-backed regressions for exact payload attribution, growth, logical deletion versus physical compaction, VM-work reduction, and interruption followed by a successful retry. No retention policy or measurement budget is relaxed; full 24-hour profile acceptance remains open. --- .../src/reader/worker.rs | 26 +-- .../src/reader/worker/table_sizes.rs | 214 ++++++++++++++++++ 2 files changed, 217 insertions(+), 23 deletions(-) create mode 100644 crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs index 9edee1472e7..59189d8fdbc 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs @@ -27,6 +27,8 @@ use crate::telemetry::{ReaderAdmissionRecorder, take_observed_vm}; use super::{ExistingReaderLocator, ReaderStartError}; +mod table_sizes; + const REPLY_POLL_QUANTUM: Duration = Duration::from_millis(5); /// How often an idle snapshot checks whether the checkpoint fence revoked it. @@ -770,29 +772,7 @@ async fn run_snapshot( let _ = reply.send(result); } SnapshotCommand::TableSizes { reply } => { - let read = || -> Result, rusqlite::Error> { - let mut statement = transaction.prepare_cached( - "SELECT schema_entry.name, COALESCE(SUM(dbstat.payload), 0) \ - FROM sqlite_schema AS schema_entry \ - LEFT JOIN dbstat ON dbstat.name = schema_entry.name \ - WHERE schema_entry.type = 'table' \ - AND schema_entry.name NOT LIKE 'sqlite_%' \ - GROUP BY schema_entry.name \ - ORDER BY schema_entry.name", - )?; - statement - .query_map([], |row| { - let bytes = row.get::<_, i64>(1)?; - Ok(TableSizeTelemetrySample { - table_name: row.get(0)?, - bytes: u64::try_from(bytes).map_err(|_| { - rusqlite::Error::IntegralValueOutOfRange(1, bytes) - })?, - }) - })? - .collect() - }; - let result = read().map_err(|error| { + let result = table_sizes::read(transaction).map_err(|error| { ReaderWorkerError::Storage(StorageRuntimeErrorV1::Infrastructure { operation: format!("read table size telemetry: {error}"), }) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs new file mode 100644 index 00000000000..217f18be5ac --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -0,0 +1,214 @@ +//! Payload census on the reader's existing snapshot, without a page-sized sorter. + +use rusqlite::Connection; + +use super::TableSizeTelemetrySample; + +/// Aggregate one table at a time. Joining every `dbstat` page before `GROUP BY` +/// materializes a temporary sorter proportional to the database's page count, +/// just to return one row per table. This correlated aggregate streams the pages +/// into one `SUM` and sorts only the completed table totals. +/// +/// Keep `dbstat` in page mode: `SQLite` can observe interruption between pages, +/// rather than spending an entire large btree inside one aggregate-mode step. +/// Payload bytes deliberately exclude indexes, free pages and WAL/SHM files; +/// the separate store/profile footprint owns those physical-size measurements. +pub(super) fn read(connection: &Connection) -> rusqlite::Result> { + let mut statement = connection.prepare_cached( + "SELECT schema_entry.name, \ + (SELECT COALESCE(SUM(payload), 0) FROM dbstat \ + WHERE name = schema_entry.name) \ + FROM sqlite_schema AS schema_entry \ + WHERE schema_entry.type = 'table' \ + AND schema_entry.name NOT LIKE 'sqlite_%' \ + ORDER BY schema_entry.name", + )?; + statement + .query_map([], |row| { + let bytes = row.get::<_, i64>(1)?; + Ok(TableSizeTelemetrySample { + table_name: row.get(0)?, + bytes: u64::try_from(bytes) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(1, bytes))?, + }) + })? + .collect() +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + use super::*; + + // Independent, per-page reference for the previous census. Keeping it in + // the test also makes the work regression falsifiable on the same SQLite + // build, without wall-clock thresholds or query-plan string assertions. + const GROUPED_CENSUS: &str = + "SELECT schema_entry.name, COALESCE(SUM(dbstat.payload), 0) \ + FROM sqlite_schema AS schema_entry \ + LEFT JOIN dbstat ON dbstat.name = schema_entry.name \ + WHERE schema_entry.type = 'table' \ + AND schema_entry.name NOT LIKE 'sqlite_%' \ + GROUP BY schema_entry.name ORDER BY schema_entry.name"; + + fn fixture() -> (tempfile::TempDir, Connection) { + let directory = tempfile::tempdir().unwrap(); + let connection = Connection::open(directory.path().join("growth.db")).unwrap(); + connection + .execute_batch( + "PRAGMA page_size = 4096; + PRAGMA auto_vacuum = NONE; + CREATE TABLE events(id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB); + CREATE INDEX event_size ON events(length(body)); + CREATE TABLE empty_table(body BLOB); + CREATE TABLE keyed(name TEXT PRIMARY KEY, body BLOB) WITHOUT ROWID; + CREATE VIRTUAL TABLE page_probe USING dbstat; + INSERT INTO keyed VALUES ('one', zeroblob(32768)); + WITH RECURSIVE n(x) AS ( + SELECT 1 UNION ALL SELECT x + 1 FROM n WHERE x < 16 + ) INSERT INTO events(body) SELECT zeroblob(262144) FROM n;", + ) + .unwrap(); + (directory, connection) + } + + fn reference(connection: &Connection) -> Vec { + let mut statement = connection.prepare(GROUPED_CENSUS).unwrap(); + statement + .query_map([], |row| { + let bytes = row.get::<_, i64>(1)?; + Ok(TableSizeTelemetrySample { + table_name: row.get(0)?, + bytes: u64::try_from(bytes).unwrap(), + }) + }) + .unwrap() + .collect::>() + .unwrap() + } + + fn measure_vm(connection: &Connection, run: impl FnOnce() -> T) -> (T, usize) { + let steps = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&steps); + connection + .progress_handler( + 1, + Some(move || { + observed.fetch_add(1, Ordering::Relaxed); + false + }), + ) + .unwrap(); + let result = run(); + connection.progress_handler(0, None:: bool>).unwrap(); + (result, steps.load(Ordering::Relaxed)) + } + + fn page_count(connection: &Connection) -> i64 { + connection + .pragma_query_value(None, "page_count", |row| row.get(0)) + .unwrap() + } + + #[test] + fn payload_census_tracks_growth_and_reclamation_without_counting_indexes() { + let (_directory, connection) = fixture(); + let before = read(&connection).unwrap(); + assert_eq!(before, reference(&connection)); + let bytes: BTreeMap<_, _> = before + .iter() + .map(|sample| (sample.table_name.as_str(), sample.bytes)) + .collect(); + assert!(bytes["events"] > 16 * 262_144); + assert!(bytes["keyed"] > 32_768); + assert_eq!(bytes["empty_table"], 0); + assert_eq!( + bytes["page_probe"], + 0, + "rootless tables still have a sample" + ); + assert!(!bytes.contains_key("event_size")); + assert!(!bytes.contains_key("sqlite_sequence")); + assert!( + before + .windows(2) + .all(|pair| pair[0].table_name < pair[1].table_name) + ); + + connection + .execute("INSERT INTO events(body) VALUES (zeroblob(262144))", []) + .unwrap(); + let enlarged = read(&connection).unwrap(); + assert_eq!(enlarged, reference(&connection)); + assert!(enlarged.iter().any(|sample| { + sample.table_name == "events" && sample.bytes > bytes["events"] + })); + + let grown_pages = page_count(&connection); + connection + .execute("DELETE FROM events WHERE id > 1", []) + .unwrap(); + let reclaimed = read(&connection).unwrap(); + assert_eq!(reclaimed, reference(&connection)); + let retained = reclaimed + .iter() + .find(|sample| sample.table_name == "events") + .unwrap(); + assert!(retained.bytes > 262_144); + assert!(retained.bytes < bytes["events"]); + assert_eq!( + page_count(&connection), + grown_pages, + "free pages are not live payload" + ); + + // Only this disposable fixture is vacuumed. Payload accounting must + // not confuse physical compaction with another logical deletion. + connection.execute_batch("VACUUM").unwrap(); + assert!(page_count(&connection) < grown_pages); + assert_eq!(read(&connection).unwrap(), reclaimed); + } + + #[test] + fn page_census_avoids_group_sort_work_under_the_same_vm_budget() { + let (_directory, connection) = fixture(); + let (expected, grouped_steps) = measure_vm(&connection, || reference(&connection)); + let (actual, streamed_steps) = measure_vm(&connection, || read(&connection).unwrap()); + assert_eq!(actual, expected); + assert!( + actual + .iter() + .any(|sample| sample.bytes > 4 * 1024 * 1024) + ); + assert!(streamed_steps > 0); + assert!( + streamed_steps < grouped_steps / 2, + "page sorting consumed the census budget: streamed={streamed_steps}, grouped={grouped_steps}" + ); + } + + #[test] + fn interrupted_census_returns_an_error_not_partial_or_zero_sizes_and_can_retry() { + let (_directory, connection) = fixture(); + connection.progress_handler(100, Some(|| true)).unwrap(); + let interrupted = read(&connection); + connection.progress_handler(0, None:: bool>).unwrap(); + assert!(matches!( + interrupted, + Err(rusqlite::Error::SqliteFailure(error, _)) + if error.code == rusqlite::ErrorCode::OperationInterrupted + )); + let retried = read(&connection).unwrap(); + assert_eq!(retried, reference(&connection)); + assert!( + retried + .iter() + .any(|sample| sample.bytes > 4 * 1024 * 1024) + ); + } +} From d9559c838c33df295148a655305167972f4c13fd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:12:26 -0800 Subject: [PATCH 06/85] style(storage): align census regressions with pinned rustfmt Refs #4099. Apply the exact rustfmt delta reported by the hosted Repository gates; no production logic or acceptance criteria changed. --- .../src/reader/worker/table_sizes.rs | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs index 217f18be5ac..adb05e762a1 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -48,8 +48,7 @@ mod tests { // Independent, per-page reference for the previous census. Keeping it in // the test also makes the work regression falsifiable on the same SQLite // build, without wall-clock thresholds or query-plan string assertions. - const GROUPED_CENSUS: &str = - "SELECT schema_entry.name, COALESCE(SUM(dbstat.payload), 0) \ + const GROUPED_CENSUS: &str = "SELECT schema_entry.name, COALESCE(SUM(dbstat.payload), 0) \ FROM sqlite_schema AS schema_entry \ LEFT JOIN dbstat ON dbstat.name = schema_entry.name \ WHERE schema_entry.type = 'table' \ @@ -105,7 +104,9 @@ mod tests { ) .unwrap(); let result = run(); - connection.progress_handler(0, None:: bool>).unwrap(); + connection + .progress_handler(0, None:: bool>) + .unwrap(); (result, steps.load(Ordering::Relaxed)) } @@ -128,8 +129,7 @@ mod tests { assert!(bytes["keyed"] > 32_768); assert_eq!(bytes["empty_table"], 0); assert_eq!( - bytes["page_probe"], - 0, + bytes["page_probe"], 0, "rootless tables still have a sample" ); assert!(!bytes.contains_key("event_size")); @@ -145,9 +145,11 @@ mod tests { .unwrap(); let enlarged = read(&connection).unwrap(); assert_eq!(enlarged, reference(&connection)); - assert!(enlarged.iter().any(|sample| { - sample.table_name == "events" && sample.bytes > bytes["events"] - })); + assert!( + enlarged + .iter() + .any(|sample| { sample.table_name == "events" && sample.bytes > bytes["events"] }) + ); let grown_pages = page_count(&connection); connection @@ -180,11 +182,7 @@ mod tests { let (expected, grouped_steps) = measure_vm(&connection, || reference(&connection)); let (actual, streamed_steps) = measure_vm(&connection, || read(&connection).unwrap()); assert_eq!(actual, expected); - assert!( - actual - .iter() - .any(|sample| sample.bytes > 4 * 1024 * 1024) - ); + assert!(actual.iter().any(|sample| sample.bytes > 4 * 1024 * 1024)); assert!(streamed_steps > 0); assert!( streamed_steps < grouped_steps / 2, @@ -197,7 +195,9 @@ mod tests { let (_directory, connection) = fixture(); connection.progress_handler(100, Some(|| true)).unwrap(); let interrupted = read(&connection); - connection.progress_handler(0, None:: bool>).unwrap(); + connection + .progress_handler(0, None:: bool>) + .unwrap(); assert!(matches!( interrupted, Err(rusqlite::Error::SqliteFailure(error, _)) @@ -205,10 +205,6 @@ mod tests { )); let retried = read(&connection).unwrap(); assert_eq!(retried, reference(&connection)); - assert!( - retried - .iter() - .any(|sample| sample.bytes > 4 * 1024 * 1024) - ); + assert!(retried.iter().any(|sample| sample.bytes > 4 * 1024 * 1024)); } } From 679f6363564c5f5314a0b14d7152bb8f801bded7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:22:09 -0800 Subject: [PATCH 07/85] test(storage): satisfy clippy for empty registry assertion Refs #4099. Use a value assertion so the hosted Clippy assert_is_empty policy can display an unexpected store on failure. --- crates/tracedecay-maintenance/src/retention/storage_report.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index b8653747322..5f154fce1d6 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -2489,7 +2489,7 @@ mod tests { let absent = build_storage_report(&profile_root) .await .expect("a genuinely absent registry is still reportable"); - assert!(absent.stores.is_empty()); + assert_eq!(absent.stores, Vec::::new()); assert_eq!(absent.unregistered_dir_count, 1); assert!(absent.unregistered_bytes >= 17); } From a3654ac170b2723f2da8a0479c18dc174eb4b980 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:28:18 -0800 Subject: [PATCH 08/85] fix(storage): measure user tables with sqlite-like names Refs #4099. LIKE treats the underscore in sqlite_% as a wildcard, silently excluding valid user tables such as sqlitemessages. Match the literal reserved sqlite_ prefix instead without changing the streaming page census or its cancellation and snapshot boundaries. Add a file-backed regression that requires the omitted table's nonzero payload and subsequent growth to match an independent dbstat sum while still excluding sqlite_sequence. Preserve the concurrent CI repair. --- .../src/reader/worker/table_sizes.rs | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs index adb05e762a1..f24bfed629a 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -14,13 +14,15 @@ use super::TableSizeTelemetrySample; /// Payload bytes deliberately exclude indexes, free pages and WAL/SHM files; /// the separate store/profile footprint owns those physical-size measurements. pub(super) fn read(connection: &Connection) -> rusqlite::Result> { + // LIKE treats the underscore as a wildcard and hides ordinary tables + // such as `sqlitemessages`; only the literal `sqlite_` prefix is internal. let mut statement = connection.prepare_cached( "SELECT schema_entry.name, \ (SELECT COALESCE(SUM(payload), 0) FROM dbstat \ WHERE name = schema_entry.name) \ FROM sqlite_schema AS schema_entry \ WHERE schema_entry.type = 'table' \ - AND schema_entry.name NOT LIKE 'sqlite_%' \ + AND schema_entry.name NOT GLOB 'sqlite_*' \ ORDER BY schema_entry.name", )?; statement @@ -176,6 +178,42 @@ mod tests { assert_eq!(read(&connection).unwrap(), reclaimed); } + #[test] + fn payload_census_includes_sqlite_named_user_tables_and_their_growth() { + let (_directory, connection) = fixture(); + connection + .execute_batch( + "CREATE TABLE sqlitemessages(body BLOB); + INSERT INTO sqlitemessages VALUES (zeroblob(1048576));", + ) + .unwrap(); + let before = read(&connection).unwrap(); + let initial = before + .iter() + .find(|sample| sample.table_name == "sqlitemessages") + .expect("a user table without the reserved underscore must be measured"); + assert!(initial.bytes > 1_048_576); + + connection + .execute("INSERT INTO sqlitemessages VALUES (zeroblob(1048576))", []) + .unwrap(); + let after = read(&connection).unwrap(); + let grown = after + .iter() + .find(|sample| sample.table_name == "sqlitemessages") + .unwrap(); + let expected: i64 = connection + .query_row( + "SELECT SUM(payload) FROM dbstat WHERE name = 'sqlitemessages'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(grown.bytes, u64::try_from(expected).unwrap()); + assert!(grown.bytes > initial.bytes); + assert!(after.iter().all(|sample| sample.table_name != "sqlite_sequence")); + } + #[test] fn page_census_avoids_group_sort_work_under_the_same_vm_budget() { let (_directory, connection) = fixture(); From 4bcc943de6ecf4cc84243cb3beed2e87d8d38a2e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:37:15 -0800 Subject: [PATCH 09/85] style(storage): apply pinned rustfmt to growth regression Refs #4099. Align the new user-table growth assertion with the exact format expected by the Bazel rustfmt gate; no behavioral changes. --- .../src/reader/worker/table_sizes.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs index f24bfed629a..9db1bcead60 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -211,7 +211,11 @@ mod tests { .unwrap(); assert_eq!(grown.bytes, u64::try_from(expected).unwrap()); assert!(grown.bytes > initial.bytes); - assert!(after.iter().all(|sample| sample.table_name != "sqlite_sequence")); + assert!( + after + .iter() + .all(|sample| sample.table_name != "sqlite_sequence") + ); } #[test] From 1e3d62f5c2c687df2cae16e5791416291ef29e97 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:23:49 -0800 Subject: [PATCH 10/85] fix(storage): reject linked SQLite report sidecars Refs #4099. The offline registry snapshot followed symlinked WAL/SHM members and could copy arbitrarily large foreign files into scratch. Refuse nonregular members before copying, and keep physical family byte counts from charging symlink targets outside the inspected profile. Add real offline-report and family-copy regressions for symlinked source members, plus partial-coverage and size assertions for user sessions. No operator data is copied or deleted. --- .../src/retention/storage_report.rs | 147 +++++++++++++++++- 1 file changed, 140 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 5f154fce1d6..11f8be1429f 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1508,8 +1508,13 @@ fn user_sessions_family_bytes(profile_root: &Path) -> u64 { fn database_family_bytes(database_path: &Path) -> u64 { let mut total_bytes = 0u64; for suffix in ["", "-wal", "-shm"] { - if let Ok(metadata) = std::fs::metadata(sqlite_family_member(database_path, suffix)) { - total_bytes = total_bytes.saturating_add(metadata.len()); + // Physical profile bytes must not include a symlink's foreign target. + // The full-profile walk separately marks a linked member unavailable. + if let Ok(metadata) = std::fs::symlink_metadata(sqlite_family_member(database_path, suffix)) + { + if metadata.file_type().is_file() { + total_bytes = total_bytes.saturating_add(metadata.len()); + } } } total_bytes @@ -1519,13 +1524,32 @@ fn database_family_bytes(database_path: &Path) -> u64 { /// caller opens only `destination`, which prevents reporting from creating /// sidecars under the profile it is inspecting. fn copy_sqlite_family(source: &Path, destination: &Path) -> std::io::Result<()> { - let generation = - tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(source)?; + // Refuse nonregular and linked source members before copying any bytes. + // Path::is_file() follows symlinks, potentially copying an arbitrarily + // large foreign file into the report's temporary SQLite snapshot. + let mut members = Vec::with_capacity(3); for suffix in ["", "-wal", "-shm"] { - let source_member = sqlite_family_member(source, suffix); - if !source_member.is_file() { - continue; + let member = sqlite_family_member(source, suffix); + match std::fs::symlink_metadata(&member) { + Ok(metadata) if metadata.file_type().is_file() => { + members.push((member, suffix)); + } + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "SQLite report source member '{}' is not a regular file; symbolic links are not allowed", + member.display() + ), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), } + } + let generation = + tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(source)?; + for (source_member, suffix) in members { std::fs::copy(&source_member, sqlite_family_member(destination, suffix))?; } generation.validate() @@ -2524,6 +2548,115 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn registry_snapshot_refuses_foreign_sidecar_links_before_copying() { + for suffix in ["-wal", "-shm"] { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let connection = rusqlite::Connection::open(®istry).unwrap(); + connection + .execute_batch( + "CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, + canonical_root TEXT NOT NULL + );", + ) + .unwrap(); + drop(connection); + + let foreign = temporary.path().join("foreign.bin"); + let foreign_bytes = 512 * 1024 * 1024; + std::fs::File::create(&foreign) + .unwrap() + .set_len(foreign_bytes) + .unwrap(); + std::os::unix::fs::symlink(&foreign, sqlite_family_member(®istry, suffix)) + .unwrap(); + + let destination = temporary.path().join("snapshot.db"); + let error = copy_sqlite_family(®istry, &destination) + .expect_err("a linked SQLite family member cannot supply snapshot bytes"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains("symbolic links"), "{error}"); + assert!( + !destination.exists(), + "the invalid family must be refused before making a partial copy" + ); + assert_eq!(foreign.metadata().unwrap().len(), foreign_bytes); + } + } + + #[cfg(unix)] + #[asupersync::test] + async fn offline_report_refuses_a_foreign_registry_wal_before_sampling() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let registry = profile_root.join(GLOBAL_DB_FILENAME); + let connection = rusqlite::Connection::open(®istry).unwrap(); + connection + .execute_batch( + "CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, + canonical_root TEXT NOT NULL + );", + ) + .unwrap(); + drop(connection); + + let foreign = temporary.path().join("foreign-wal.bin"); + std::fs::File::create(&foreign) + .unwrap() + .set_len(512 * 1024 * 1024) + .unwrap(); + std::os::unix::fs::symlink(&foreign, sqlite_family_member(®istry, "-wal")).unwrap(); + + let error = build_storage_report(&profile_root) + .await + .expect_err("a linked registry WAL must fail instead of copying a foreign file"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "expected a typed storage report failure: {error}" + ); + assert!(error.to_string().contains("symbolic links"), "{error}"); + assert_eq!(foreign.metadata().unwrap().len(), 512 * 1024 * 1024); + } + + #[cfg(unix)] + #[asupersync::test] + async fn user_sessions_family_excludes_foreign_wal_bytes_and_downgrades_coverage() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + seed_global_db(&profile_root, &[]).await; + std::fs::write( + profile_root.join(USER_SESSIONS_DB_FILENAME), + vec![1u8; 4_096], + ) + .unwrap(); + let foreign = temporary.path().join("foreign-sessions-wal.bin"); + std::fs::File::create(&foreign) + .unwrap() + .set_len(512 * 1024 * 1024) + .unwrap(); + std::os::unix::fs::symlink( + &foreign, + profile_root.join(format!("{USER_SESSIONS_DB_FILENAME}-wal")), + ) + .unwrap(); + + let report = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(report.user_sessions_db_bytes, 4_096); + let profile_size = report.full_profile_size.expect("measured profile"); + assert_eq!(profile_size.state, ProfileTotalCoverageStateV1::Partial); + assert!( + profile_size.unavailable_entry_count > 0, + "the linked WAL must remain explicitly unavailable, not be counted as zero" + ); + assert_eq!(foreign.metadata().unwrap().len(), 512 * 1024 * 1024); + } + #[asupersync::test] async fn storage_report_names_user_sessions_db_family() { let tmp = tempfile::TempDir::new().unwrap(); From 8913817dbe908e6f27c0bcd317ac8f6c683679e6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:24:51 -0800 Subject: [PATCH 11/85] style(storage): apply pinned rustfmt to snapshot regression Refs #4099. Apply the exact formatter hunk reported by the hosted Repository gates on 1e3d62f. Preserve source behavior and all assertions. --- crates/tracedecay-maintenance/src/retention/storage_report.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 11f8be1429f..50369b537ec 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -2571,8 +2571,7 @@ mod tests { .unwrap() .set_len(foreign_bytes) .unwrap(); - std::os::unix::fs::symlink(&foreign, sqlite_family_member(®istry, suffix)) - .unwrap(); + std::os::unix::fs::symlink(&foreign, sqlite_family_member(®istry, suffix)).unwrap(); let destination = temporary.path().join("snapshot.db"); let error = copy_sqlite_family(®istry, &destination) From eb751efd4e5c4db616b65cd7b5ab52bc4ad237f0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:42:35 -0800 Subject: [PATCH 12/85] fix(storage): capture registry generation before WAL discovery Refs #4099. The SQLite snapshot enumerated sidecars before capturing its validation generation. A writer creating a WAL in between could commit new project registrations that the report then misclassified as unregistered. Capture the generation first, then inspect only regular members and validate after copying. Add a deterministic on-disk WAL race regression: stale generations must fail closed, while a fresh generation preserves the committed project. Also fix the pinned Clippy collapsible-if failure in family sizing. --- .../src/retention/storage_report.rs | 81 +++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 50369b537ec..68bbe64b9af 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1511,10 +1511,9 @@ fn database_family_bytes(database_path: &Path) -> u64 { // Physical profile bytes must not include a symlink's foreign target. // The full-profile walk separately marks a linked member unavailable. if let Ok(metadata) = std::fs::symlink_metadata(sqlite_family_member(database_path, suffix)) + && metadata.file_type().is_file() { - if metadata.file_type().is_file() { - total_bytes = total_bytes.saturating_add(metadata.len()); - } + total_bytes = total_bytes.saturating_add(metadata.len()); } } total_bytes @@ -1524,6 +1523,16 @@ fn database_family_bytes(database_path: &Path) -> u64 { /// caller opens only `destination`, which prevents reporting from creating /// sidecars under the profile it is inspecting. fn copy_sqlite_family(source: &Path, destination: &Path) -> std::io::Result<()> { + let generation = + tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(source)?; + copy_sqlite_family_at_generation(source, destination, &generation) +} + +fn copy_sqlite_family_at_generation( + source: &Path, + destination: &Path, + generation: &tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration, +) -> std::io::Result<()> { // Refuse nonregular and linked source members before copying any bytes. // Path::is_file() follows symlinks, potentially copying an arbitrarily // large foreign file into the report's temporary SQLite snapshot. @@ -1547,8 +1556,6 @@ fn copy_sqlite_family(source: &Path, destination: &Path) -> std::io::Result<()> Err(error) => return Err(error), } } - let generation = - tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(source)?; for (source_member, suffix) in members { std::fs::copy(&source_member, sqlite_family_member(destination, suffix))?; } @@ -2586,6 +2593,70 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn registry_snapshot_rejects_wal_committed_after_generation_capture() { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch( + "CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, + canonical_root TEXT NOT NULL + );", + ) + .unwrap(); + let generation = + tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(®istry) + .unwrap(); + // A new committed WAL must invalidate the preexisting generation. + // The old order captured membership before the generation and then + // silently copied only the main database, losing this project. + writer + .execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + INSERT INTO code_projects VALUES ('proj_new', '/repos/new');", + ) + .unwrap(); + let wal = sqlite_family_member(®istry, "-wal"); + assert!(wal.metadata().unwrap().len() > 0); + + let destination = temporary.path().join("report-snapshot.db"); + let error = copy_sqlite_family_at_generation(®istry, &destination, &generation) + .expect_err("a post-capture WAL commit must invalidate the snapshot"); + assert!( + error.to_string().contains("changed after inspection"), + "the snapshot must fail closed instead of misclassifying new projects: {error}" + ); + let canonical_root: String = writer + .query_row( + "SELECT canonical_root FROM code_projects WHERE project_id = 'proj_new'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(canonical_root, "/repos/new"); + + // A new complete generation must still be usable on the same live + // registry; the rejection is for racing changes, not WAL mode itself. + let stable = + tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(®istry) + .unwrap(); + let stable_destination = temporary.path().join("stable-snapshot.db"); + copy_sqlite_family_at_generation(®istry, &stable_destination, &stable).unwrap(); + let snapshot = rusqlite::Connection::open(&stable_destination).unwrap(); + let snapshot_root: String = snapshot + .query_row( + "SELECT canonical_root FROM code_projects WHERE project_id = 'proj_new'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(snapshot_root, "/repos/new"); + } + #[cfg(unix)] #[asupersync::test] async fn offline_report_refuses_a_foreign_registry_wal_before_sampling() { From f9827683e484336fcd6d7fd752084f6eb85932f0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:25:34 -0800 Subject: [PATCH 13/85] fix(storage): pin SQLite members against symlink swaps Refs #4099. The registry snapshot preflight rejected symlinks, but fs::copy reopened each checked path and could follow a link swapped in after inspection, copying foreign bytes into report scratch. Open through a held parent capability with no-follow leaf policy, then copy from that file handle. A real filesystem regression swaps a checked WAL member to a foreign link and proves the copy refuses without creating a scratch member. Preserve existing CI repairs. --- .../src/retention/storage_report.rs | 64 ++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 68bbe64b9af..249939800f9 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -29,6 +29,8 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::io::Read; use std::path::{Path, PathBuf}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; + use tracedecay_domain::errors::TraceDecayError; use tracedecay_domain::{ CodeGenerationId, CursorBindingMismatchV1, CursorBindingV1, decode_bound_cursor, @@ -1556,12 +1558,43 @@ fn copy_sqlite_family_at_generation( Err(error) => return Err(error), } } - for (source_member, suffix) in members { - std::fs::copy(&source_member, sqlite_family_member(destination, suffix))?; - } + copy_sqlite_members_nofollow(source, destination, &members)?; generation.validate() } +/// Open the checked member through a held directory, refusing a symlink even +/// when its name is replaced after the family preflight. A bare fs::copy would +/// follow that replacement and import foreign bytes into report scratch. +fn copy_sqlite_members_nofollow( + source: &Path, + destination: &Path, + members: &[(PathBuf, &str)], +) -> std::io::Result<()> { + let parent = source + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let directory = cap_std::fs::Dir::open_ambient_dir(parent, cap_std::ambient_authority())?; + let mut options = cap_std::fs::OpenOptions::new(); + options.read(true); + options.follow(FollowSymlinks::No); + for (member, suffix) in members { + let name = member.file_name().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "SQLite member has no filename") + })?; + let mut input = directory.open_with(name, &options)?; + if !input.metadata()?.is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "SQLite report source member is not a regular file", + )); + } + let mut output = std::fs::File::create(sqlite_family_member(destination, suffix))?; + std::io::copy(&mut input, &mut output)?; + } + Ok(()) +} + fn validate_external_scratch(profile_root: &Path, scratch_root: &Path) -> std::io::Result<()> { let profile = profile_root.canonicalize()?; let scratch = scratch_root.canonicalize()?; @@ -2593,6 +2626,31 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn registry_snapshot_member_cannot_follow_a_post_preflight_link_swap() { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let wal = sqlite_family_member(®istry, "-wal"); + std::fs::write(&wal, b"original WAL member").unwrap(); + assert!(std::fs::symlink_metadata(&wal).unwrap().file_type().is_file()); + // The production preflight recorded this pathname as a regular file. + let checked = vec![(wal.clone(), "-wal")]; + std::fs::remove_file(&wal).unwrap(); + let foreign = temporary.path().join("foreign-data.bin"); + std::fs::write(&foreign, vec![73u8; 4096]).unwrap(); + std::os::unix::fs::symlink(&foreign, &wal).unwrap(); + + let destination = temporary.path().join("report-snapshot.db"); + copy_sqlite_members_nofollow(®istry, &destination, &checked) + .expect_err("a swapped link must never supply snapshot bytes"); + assert!( + !sqlite_family_member(&destination, "-wal").exists(), + "a link swapped after preflight cannot be copied into report scratch" + ); + assert_eq!(std::fs::metadata(&foreign).unwrap().len(), 4096); + } + #[cfg(unix)] #[test] fn registry_snapshot_rejects_wal_committed_after_generation_capture() { From 2d839d80f4c1b3b6541cfc9ceb6d2158198b0d13 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:46:34 -0800 Subject: [PATCH 14/85] fix(storage): refuse replaced SQLite report members before copy Refs #4099. A no-follow open still accepts a regular-file replacement between preflight and copy, potentially copying a huge foreign WAL before post-copy generation validation rejects it. Compare the opened descriptor against checked identity and length before creating scratch; bound the copy to the inspected length plus one. Add a filesystem regression for same-sized and oversized regular-file swaps without reading the foreign bytes. --- .../src/retention/storage_report.rs | 104 ++++++++++++++++-- 1 file changed, 93 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 249939800f9..e36bfa13fbe 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1543,7 +1543,7 @@ fn copy_sqlite_family_at_generation( let member = sqlite_family_member(source, suffix); match std::fs::symlink_metadata(&member) { Ok(metadata) if metadata.file_type().is_file() => { - members.push((member, suffix)); + members.push((member, suffix, metadata)); } Ok(_) => { return Err(std::io::Error::new( @@ -1562,13 +1562,43 @@ fn copy_sqlite_family_at_generation( generation.validate() } -/// Open the checked member through a held directory, refusing a symlink even -/// when its name is replaced after the family preflight. A bare fs::copy would -/// follow that replacement and import foreign bytes into report scratch. +fn matches_checked_sqlite_member( + checked: &std::fs::Metadata, + opened: &std::fs::Metadata, +) -> bool { + if !opened.is_file() || checked.len() != opened.len() { + return false; + } + // A no-follow open blocks symlink swaps but not regular-file replacement. + // Revalidate the held file's identity before creating any scratch output. + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + checked.dev() == opened.dev() && checked.ino() == opened.ino() + } + #[cfg(not(unix))] + { + // A size-changing swap is rejected above. On platforms without the + // Unix inode API, compare timestamps and let the generation witness + // independently reject a changed source after copying. + checked + .created() + .ok() + .zip(opened.created().ok()) + .zip(checked.modified().ok().zip(opened.modified().ok())) + .is_some_and(|((created, opened_created), (modified, opened_modified))| { + created == opened_created && modified == opened_modified + }) + } +} + +/// Open the checked member through a held directory, refusing symlink or +/// regular-file replacement after preflight. A pathname copy would import +/// unrelated bytes into report scratch even if later validation rejected it. fn copy_sqlite_members_nofollow( source: &Path, destination: &Path, - members: &[(PathBuf, &str)], + members: &[(PathBuf, &str, std::fs::Metadata)], ) -> std::io::Result<()> { let parent = source .parent() @@ -1578,19 +1608,30 @@ fn copy_sqlite_members_nofollow( let mut options = cap_std::fs::OpenOptions::new(); options.read(true); options.follow(FollowSymlinks::No); - for (member, suffix) in members { + for (member, suffix, checked) in members { let name = member.file_name().ok_or_else(|| { std::io::Error::new(std::io::ErrorKind::InvalidInput, "SQLite member has no filename") })?; - let mut input = directory.open_with(name, &options)?; - if !input.metadata()?.is_file() { + let mut input = directory.open_with(name, &options)?.into_std(); + if !matches_checked_sqlite_member(checked, &input.metadata()?) { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - "SQLite report source member is not a regular file", + format!( + "SQLite report source member '{}' changed after preflight", + member.display() + ), )); } let mut output = std::fs::File::create(sqlite_family_member(destination, suffix))?; - std::io::copy(&mut input, &mut output)?; + // A live WAL may grow after inspection; do not copy an unbounded tail. + let cap = checked.len().saturating_add(1); + let copied = std::io::copy(&mut input.take(cap), &mut output)?; + if copied != checked.len() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "SQLite report source member changed size during copy", + )); + } } Ok(()) } @@ -2635,7 +2676,11 @@ mod tests { std::fs::write(&wal, b"original WAL member").unwrap(); assert!(std::fs::symlink_metadata(&wal).unwrap().file_type().is_file()); // The production preflight recorded this pathname as a regular file. - let checked = vec![(wal.clone(), "-wal")]; + let checked = vec![( + wal.clone(), + "-wal", + std::fs::symlink_metadata(&wal).unwrap(), + )]; std::fs::remove_file(&wal).unwrap(); let foreign = temporary.path().join("foreign-data.bin"); std::fs::write(&foreign, vec![73u8; 4096]).unwrap(); @@ -2651,6 +2696,43 @@ mod tests { assert_eq!(std::fs::metadata(&foreign).unwrap().len(), 4096); } + #[cfg(unix)] + #[test] + fn registry_snapshot_refuses_regular_member_replacement_before_copying() { + for replacement_bytes in [4096_u64, 512 * 1024 * 1024] { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let wal = sqlite_family_member(®istry, "-wal"); + std::fs::write(&wal, vec![17u8; 4096]).unwrap(); + let checked = vec![( + wal.clone(), + "-wal", + std::fs::symlink_metadata(&wal).unwrap(), + )]; + + // Both files are regular, so a no-follow open still succeeds. + // Include a same-sized swap: checking size alone is insufficient. + let incoming = temporary.path().join("foreign-regular.bin"); + std::fs::File::create(&incoming) + .unwrap() + .set_len(replacement_bytes) + .unwrap(); + std::fs::rename(&incoming, &wal).unwrap(); + assert!(std::fs::symlink_metadata(&wal).unwrap().file_type().is_file()); + + let destination = temporary.path().join("report-snapshot.db"); + let error = copy_sqlite_members_nofollow(®istry, &destination, &checked) + .expect_err("a regular member swapped after preflight must be refused"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains("changed after preflight"), "{error}"); + assert!( + !sqlite_family_member(&destination, "-wal").exists(), + "the replacement must be refused before any bytes enter scratch" + ); + assert_eq!(std::fs::metadata(&wal).unwrap().len(), replacement_bytes); + } + } + #[cfg(unix)] #[test] fn registry_snapshot_rejects_wal_committed_after_generation_capture() { From 7f54a8459169c810c4c052b1dbc7788e73b13c62 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:15:20 -0800 Subject: [PATCH 15/85] style(storage): honor top-level Unix metadata imports Refs #4099. Keep inode identity verification unchanged while moving MetadataExt to the module's explicit imports as required by AGENTS.md and the open review on PR #4794. --- crates/tracedecay-maintenance/src/retention/storage_report.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index e36bfa13fbe..dc145f5118e 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -27,6 +27,8 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::io::Read; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; @@ -1573,7 +1575,6 @@ fn matches_checked_sqlite_member( // Revalidate the held file's identity before creating any scratch output. #[cfg(unix)] { - use std::os::unix::fs::MetadataExt; checked.dev() == opened.dev() && checked.ino() == opened.ino() } #[cfg(not(unix))] From 32aa3888308bcfe8353bb8ae9142301a4fe5a7b3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:19:44 -0800 Subject: [PATCH 16/85] fix(storage): refuse rollback-journal registry snapshots Refs #4099. A rollback-mode writer may spill uncommitted registration pages into global.db while the undo journal remains. Copying only the main/WAL/SHM family would turn those pages into false registry truth. Fail closed on the rollback journal before writing scratch. Add direct and end-to-end real-SQLite regressions for refusal during an open transaction and correct registration after rollback. Preserve all prior snapshot generation and no-follow protections. --- .../src/retention/storage_report.rs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index dc145f5118e..d865fd43260 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1537,6 +1537,24 @@ fn copy_sqlite_family_at_generation( destination: &Path, generation: &tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration, ) -> std::io::Result<()> { + // A rollback-mode writer may spill uncommitted pages into the main file + // while their undo records remain in -journal. Copying main/WAL/SHM alone + // can report those uncommitted rows as registered projects. Refuse the + // incomplete family rather than returning false retention authority. + let journal_path = sqlite_family_member(source, "-journal"); + match std::fs::symlink_metadata(&journal_path) { + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "SQLite report source has rollback journal '{}'; main/WAL/SHM snapshot is unsafe", + journal_path.display() + ), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } // Refuse nonregular and linked source members before copying any bytes. // Path::is_file() follows symlinks, potentially copying an arbitrarily // large foreign file into the report's temporary SQLite snapshot. @@ -2630,6 +2648,86 @@ mod tests { ); } + #[test] + fn registry_snapshot_refuses_rollback_journal_and_recovers_after_rollback() { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, canonical_root TEXT NOT NULL + ); + INSERT INTO code_projects VALUES ('proj_committed', '/repos/committed'); + BEGIN IMMEDIATE; + UPDATE code_projects SET canonical_root = '/repos/uncommitted' + WHERE project_id = 'proj_committed';", + ) + .unwrap(); + let journal = sqlite_family_member(®istry, "-journal"); + assert!(journal.is_file(), "the uncommitted update needs an undo journal"); + + let destination = temporary.path().join("snapshot.db"); + let error = copy_sqlite_family(®istry, &destination) + .expect_err("a main-only snapshot must not read uncommitted rollback pages"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains("rollback journal"), "{error}"); + assert!(!destination.exists(), "refusal must happen before any copy"); + + writer.execute_batch("ROLLBACK").unwrap(); + drop(writer); + assert!(!journal.exists(), "DELETE-mode rollback removes the journal"); + copy_sqlite_family(®istry, &destination).unwrap(); + let snapshot = rusqlite::Connection::open(&destination).unwrap(); + let root: String = snapshot + .query_row( + "SELECT canonical_root FROM code_projects WHERE project_id = 'proj_committed'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(root, "/repos/committed"); + } + + #[asupersync::test] + async fn offline_report_refuses_rollback_journal_instead_of_misclassifying_stores() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + let project = profile_root.join("projects/proj_committed"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("sessions.db"), b"retained session").unwrap(); + let registry = profile_root.join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, canonical_root TEXT NOT NULL + ); + INSERT INTO code_projects VALUES ('proj_committed', '/repos/committed'); + BEGIN IMMEDIATE; + UPDATE code_projects SET canonical_root = '/repos/uncommitted' + WHERE project_id = 'proj_committed';", + ) + .unwrap(); + assert!(sqlite_family_member(®istry, "-journal").is_file()); + + let error = build_storage_report(&profile_root) + .await + .expect_err("an active rollback journal must not become registration truth"); + assert!(matches!(&error, TraceDecayError::Database { .. }), "{error}"); + assert!(error.to_string().contains("rollback journal"), "{error}"); + + writer.execute_batch("ROLLBACK").unwrap(); + drop(writer); + let report = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(report.stores.len(), 1); + assert_eq!(report.stores[0].project_id, "proj_committed"); + assert_eq!(report.stores[0].canonical_root, "/repos/committed"); + assert_eq!(report.unregistered_dir_count, 0); + } + #[cfg(unix)] #[test] fn registry_snapshot_refuses_foreign_sidecar_links_before_copying() { From c4258d8a39893a174471fab64d5b3e11f6b5f422 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:25:35 -0800 Subject: [PATCH 17/85] style(storage): format SQLite snapshot regressions Apply eight exact Rust 2024 rustfmt differences from hosted Bazel Repository gates. No behavior, schema, or test assertion changes. --- .../src/retention/storage_report.rs | 44 ++++++++++++++----- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index d865fd43260..2809d423917 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1582,10 +1582,7 @@ fn copy_sqlite_family_at_generation( generation.validate() } -fn matches_checked_sqlite_member( - checked: &std::fs::Metadata, - opened: &std::fs::Metadata, -) -> bool { +fn matches_checked_sqlite_member(checked: &std::fs::Metadata, opened: &std::fs::Metadata) -> bool { if !opened.is_file() || checked.len() != opened.len() { return false; } @@ -1629,7 +1626,10 @@ fn copy_sqlite_members_nofollow( options.follow(FollowSymlinks::No); for (member, suffix, checked) in members { let name = member.file_name().ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::InvalidInput, "SQLite member has no filename") + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "SQLite member has no filename", + ) })?; let mut input = directory.open_with(name, &options)?.into_std(); if !matches_checked_sqlite_member(checked, &input.metadata()?) { @@ -2666,7 +2666,10 @@ mod tests { ) .unwrap(); let journal = sqlite_family_member(®istry, "-journal"); - assert!(journal.is_file(), "the uncommitted update needs an undo journal"); + assert!( + journal.is_file(), + "the uncommitted update needs an undo journal" + ); let destination = temporary.path().join("snapshot.db"); let error = copy_sqlite_family(®istry, &destination) @@ -2677,7 +2680,10 @@ mod tests { writer.execute_batch("ROLLBACK").unwrap(); drop(writer); - assert!(!journal.exists(), "DELETE-mode rollback removes the journal"); + assert!( + !journal.exists(), + "DELETE-mode rollback removes the journal" + ); copy_sqlite_family(®istry, &destination).unwrap(); let snapshot = rusqlite::Connection::open(&destination).unwrap(); let root: String = snapshot @@ -2716,7 +2722,10 @@ mod tests { let error = build_storage_report(&profile_root) .await .expect_err("an active rollback journal must not become registration truth"); - assert!(matches!(&error, TraceDecayError::Database { .. }), "{error}"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "{error}" + ); assert!(error.to_string().contains("rollback journal"), "{error}"); writer.execute_batch("ROLLBACK").unwrap(); @@ -2773,7 +2782,12 @@ mod tests { let registry = temporary.path().join(GLOBAL_DB_FILENAME); let wal = sqlite_family_member(®istry, "-wal"); std::fs::write(&wal, b"original WAL member").unwrap(); - assert!(std::fs::symlink_metadata(&wal).unwrap().file_type().is_file()); + assert!( + std::fs::symlink_metadata(&wal) + .unwrap() + .file_type() + .is_file() + ); // The production preflight recorded this pathname as a regular file. let checked = vec![( wal.clone(), @@ -2817,13 +2831,21 @@ mod tests { .set_len(replacement_bytes) .unwrap(); std::fs::rename(&incoming, &wal).unwrap(); - assert!(std::fs::symlink_metadata(&wal).unwrap().file_type().is_file()); + assert!( + std::fs::symlink_metadata(&wal) + .unwrap() + .file_type() + .is_file() + ); let destination = temporary.path().join("report-snapshot.db"); let error = copy_sqlite_members_nofollow(®istry, &destination, &checked) .expect_err("a regular member swapped after preflight must be refused"); assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); - assert!(error.to_string().contains("changed after preflight"), "{error}"); + assert!( + error.to_string().contains("changed after preflight"), + "{error}" + ); assert!( !sqlite_family_member(&destination, "-wal").exists(), "the replacement must be refused before any bytes enter scratch" From 7990e35140d6aca26a161060b0a1e4ee837269f2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:35:34 -0800 Subject: [PATCH 18/85] fix(storage): recognize inert rollback journals in reports Refs #4099. Allow committed PERSIST/TRUNCATE journals while rejecting active/uncertain rollback state and verifying the journal witness across the registry snapshot copy. Cover committed, active and rolled back states with real SQLite report regressions. --- .../src/retention/storage_report.rs | 186 ++++++++++++++++-- 1 file changed, 168 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 2809d423917..1f7c8af8f18 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1537,24 +1537,11 @@ fn copy_sqlite_family_at_generation( destination: &Path, generation: &tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration, ) -> std::io::Result<()> { - // A rollback-mode writer may spill uncommitted pages into the main file - // while their undo records remain in -journal. Copying main/WAL/SHM alone - // can report those uncommitted rows as registered projects. Refuse the - // incomplete family rather than returning false retention authority. - let journal_path = sqlite_family_member(source, "-journal"); - match std::fs::symlink_metadata(&journal_path) { - Ok(_) => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!( - "SQLite report source has rollback journal '{}'; main/WAL/SHM snapshot is unsafe", - journal_path.display() - ), - )); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error), - } + // Rollback journals can outlive commits in TRUNCATE and PERSIST mode. + // Only an absent, truncated, or explicitly invalidated journal is safe + // to omit from a snapshot. Recheck its witness after copying: a writer + // must not turn a stable registry into an uncommitted main-file image. + let journal_before = inspect_inert_rollback_journal(source)?; // Refuse nonregular and linked source members before copying any bytes. // Path::is_file() follows symlinks, potentially copying an arbitrarily // large foreign file into the report's temporary SQLite snapshot. @@ -1579,9 +1566,100 @@ fn copy_sqlite_family_at_generation( } } copy_sqlite_members_nofollow(source, destination, &members)?; + let journal_after = inspect_inert_rollback_journal(source)?; + if !rollback_journal_unchanged(&journal_before, &journal_after) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "SQLite report rollback journal changed while copying registry", + )); + } generation.validate() } +fn rollback_journal_uncommitted(path: &Path) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "SQLite report source has active or uncertain rollback journal '{}'; main/WAL/SHM snapshot is unsafe", + path.display() + ), + ) +} + +/// A committed TRUNCATE journal is empty; PERSIST clears the first 28 header +/// bytes instead of deleting the file. A zero magic prefix alone is NOT enough: +/// SQLite may leave those first eight bytes zero during an active transaction, +/// while the nonce and page/sector fields are already populated. +fn inspect_inert_rollback_journal( + source: &Path, +) -> std::io::Result> { + let path = sqlite_family_member(source, "-journal"); + let checked = match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_file() => metadata, + Ok(_) => return Err(rollback_journal_uncommitted(&path)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if checked.len() == 0 { + return Ok(Some(checked)); + } + if checked.len() < 28 { + return Err(rollback_journal_uncommitted(&path)); + } + + let parent = source + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let directory = cap_std::fs::Dir::open_ambient_dir(parent, cap_std::ambient_authority())?; + let mut options = cap_std::fs::OpenOptions::new(); + options.read(true); + options.follow(FollowSymlinks::No); + let name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "SQLite rollback journal has no filename", + ) + })?; + let mut input = directory.open_with(name, &options)?.into_std(); + let opened = input.metadata()?; + if !matches_checked_sqlite_member(&checked, &opened) + || checked.modified()? != opened.modified()? + { + return Err(rollback_journal_uncommitted(&path)); + } + let mut header = [0u8; 28]; + input.read_exact(&mut header)?; + if header != [0u8; 28] { + return Err(rollback_journal_uncommitted(&path)); + } + let after_read = input.metadata()?; + if !matches_checked_sqlite_member(&checked, &after_read) + || checked.modified()? != after_read.modified()? + { + return Err(rollback_journal_uncommitted(&path)); + } + Ok(Some(checked)) +} + +fn rollback_journal_unchanged( + before: &Option, + after: &Option, +) -> bool { + match (before, after) { + (None, None) => true, + (Some(before), Some(after)) => { + matches_checked_sqlite_member(before, after) + && before + .modified() + .ok() + .zip(after.modified().ok()) + .is_some_and(|(left, right)| left == right) + } + _ => false, + } +} + fn matches_checked_sqlite_member(checked: &std::fs::Metadata, opened: &std::fs::Metadata) -> bool { if !opened.is_file() || checked.len() != opened.len() { return false; @@ -2737,6 +2815,78 @@ mod tests { assert_eq!(report.unregistered_dir_count, 0); } + #[asupersync::test] + async fn committed_persist_and_truncate_journals_remain_reportable_but_active_writes_refuse() { + for mode in ["PERSIST", "TRUNCATE"] { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + let project = profile_root.join("projects/proj_committed"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("sessions.db"), b"retained session").unwrap(); + + let registry = profile_root.join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch(&format!( + "PRAGMA journal_mode = {mode}; + CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, canonical_root TEXT NOT NULL + ); + INSERT INTO code_projects VALUES ('proj_committed', '/repos/committed');" + )) + .unwrap(); + let journal = sqlite_family_member(®istry, "-journal"); + let journal_bytes = std::fs::metadata(&journal).unwrap().len(); + if mode == "TRUNCATE" { + assert_eq!(journal_bytes, 0, "TRUNCATE retains an empty journal"); + } else { + assert!( + journal_bytes >= 28, + "PERSIST retains an invalidated journal" + ); + let mut header = [0u8; 28]; + std::fs::File::open(&journal) + .unwrap() + .read_exact(&mut header) + .unwrap(); + assert_eq!(header, [0u8; 28]); + } + + let committed = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(committed.stores.len(), 1); + assert_eq!(committed.stores[0].project_id, "proj_committed"); + assert_eq!(committed.stores[0].canonical_root, "/repos/committed"); + assert_eq!(committed.unregistered_dir_count, 0); + + writer + .execute_batch( + "BEGIN IMMEDIATE; + UPDATE code_projects SET canonical_root = '/repos/uncommitted' + WHERE project_id = 'proj_committed';", + ) + .unwrap(); + assert!( + std::fs::metadata(&journal).unwrap().len() >= 28, + "the active rollback transaction must have a journal header" + ); + let error = build_storage_report(&profile_root) + .await + .expect_err("active rollback journal must not leak uncommitted registrations"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "report refusal must be a typed storage error: {error}" + ); + assert!(error.to_string().contains("rollback journal"), "{error}"); + + writer.execute_batch("ROLLBACK").unwrap(); + let recovered = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(recovered.stores.len(), 1); + assert_eq!(recovered.stores[0].canonical_root, "/repos/committed"); + assert_eq!(recovered.unregistered_dir_count, 0); + drop(writer); + } + } + #[cfg(unix)] #[test] fn registry_snapshot_refuses_foreign_sidecar_links_before_copying() { From 0c7597a3274d17d895f767cc5dd3b70f6913b6a6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 07:15:34 -0800 Subject: [PATCH 19/85] style(storage): align journal probe with pinned rustfmt Refs #4099. Apply the exact format-only correction reported by Bazel Repository gates to the committed-journal probe. No behavior or test assertion changes; preserve the existing concurrent issue-specific work. --- crates/tracedecay-maintenance/src/retention/storage_report.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 1f7c8af8f18..ee81df4048a 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1590,9 +1590,7 @@ fn rollback_journal_uncommitted(path: &Path) -> std::io::Error { /// bytes instead of deleting the file. A zero magic prefix alone is NOT enough: /// SQLite may leave those first eight bytes zero during an active transaction, /// while the nonce and page/sector fields are already populated. -fn inspect_inert_rollback_journal( - source: &Path, -) -> std::io::Result> { +fn inspect_inert_rollback_journal(source: &Path) -> std::io::Result> { let path = sqlite_family_member(source, "-journal"); let checked = match std::fs::symlink_metadata(&path) { Ok(metadata) if metadata.file_type().is_file() => metadata, From c396deeb8e624324abb20da2b877039bf33d06b7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:23:39 -0800 Subject: [PATCH 20/85] fix(storage): satisfy Clippy for rollback journal checks --- .../src/retention/storage_report.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index ee81df4048a..32e187e73d7 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1567,7 +1567,7 @@ fn copy_sqlite_family_at_generation( } copy_sqlite_members_nofollow(source, destination, &members)?; let journal_after = inspect_inert_rollback_journal(source)?; - if !rollback_journal_unchanged(&journal_before, &journal_after) { + if !rollback_journal_unchanged(journal_before.as_ref(), journal_after.as_ref()) { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, "SQLite report rollback journal changed while copying registry", @@ -1641,8 +1641,8 @@ fn inspect_inert_rollback_journal(source: &Path) -> std::io::Result, - after: &Option, + before: Option<&std::fs::Metadata>, + after: Option<&std::fs::Metadata>, ) -> bool { match (before, after) { (None, None) => true, @@ -1707,7 +1707,7 @@ fn copy_sqlite_members_nofollow( "SQLite member has no filename", ) })?; - let mut input = directory.open_with(name, &options)?.into_std(); + let input = directory.open_with(name, &options)?.into_std(); if !matches_checked_sqlite_member(checked, &input.metadata()?) { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, From e64168b49bb667a835170fd29869d06742186cb3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:46:34 -0800 Subject: [PATCH 21/85] perf(storage): scan each SQLite page once for table growth Refs #4099. Avoid a correlated dbstat rescan per table while retaining exact payload totals, interruptible page mode and bounded table-only state. Add a many-table VM-work regression that fails under the prior query. --- .../src/reader/worker/table_sizes.rs | 98 +++++++++++++------ 1 file changed, 67 insertions(+), 31 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs index 9db1bcead60..d28106395ab 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -1,40 +1,51 @@ -//! Payload census on the reader's existing snapshot, without a page-sized sorter. +//! Payload census on the reader's existing snapshot, with one page walk. -use rusqlite::Connection; +use std::collections::BTreeMap; + +use rusqlite::{Connection, types::Type}; use super::TableSizeTelemetrySample; -/// Aggregate one table at a time. Joining every `dbstat` page before `GROUP BY` -/// materializes a temporary sorter proportional to the database's page count, -/// just to return one row per table. This correlated aggregate streams the pages -/// into one `SUM` and sorts only the completed table totals. +/// Initialize every user table (including empty and rootless tables) from +/// schema, then visit the dbstat pages exactly once. The prior correlated SUM +/// re-walked the virtual table for each schema table; many-table profiles spent +/// their measurement deadline repeating a census even without a page sorter. /// -/// Keep `dbstat` in page mode: `SQLite` can observe interruption between pages, -/// rather than spending an entire large btree inside one aggregate-mode step. -/// Payload bytes deliberately exclude indexes, free pages and WAL/SHM files; -/// the separate store/profile footprint owns those physical-size measurements. +/// Page-mode dbstat preserves the reader's interrupt checks between pages. +/// Index and free-page bytes are not table payload; the separate footprint +/// census measures all physical database/WAL/SHM bytes. pub(super) fn read(connection: &Connection) -> rusqlite::Result> { - // LIKE treats the underscore as a wildcard and hides ordinary tables - // such as `sqlitemessages`; only the literal `sqlite_` prefix is internal. - let mut statement = connection.prepare_cached( - "SELECT schema_entry.name, \ - (SELECT COALESCE(SUM(payload), 0) FROM dbstat \ - WHERE name = schema_entry.name) \ - FROM sqlite_schema AS schema_entry \ - WHERE schema_entry.type = 'table' \ - AND schema_entry.name NOT GLOB 'sqlite_*' \ - ORDER BY schema_entry.name", - )?; - statement - .query_map([], |row| { - let bytes = row.get::<_, i64>(1)?; - Ok(TableSizeTelemetrySample { - table_name: row.get(0)?, - bytes: u64::try_from(bytes) - .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(1, bytes))?, - }) - })? - .collect() + let mut totals = BTreeMap::::new(); + { + let mut statement = connection.prepare_cached( + "SELECT name FROM sqlite_schema \ + WHERE type = 'table' AND name NOT GLOB 'sqlite_*'", + )?; + for name in statement.query_map([], |row| row.get::<_, String>(0))? { + totals.insert(name?, 0); + } + } + + let mut statement = connection.prepare_cached("SELECT name, payload FROM dbstat")?; + let mut pages = statement.query([])?; + while let Some(page) = pages.next()? { + let name = page.get_ref(0)?.as_str().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(0, Type::Text, Box::new(error)) + })?; + if let Some(total) = totals.get_mut(name) { + let signed_payload = page.get::<_, i64>(1)?; + let payload = u64::try_from(signed_payload).map_err(|_| { + rusqlite::Error::IntegralValueOutOfRange(1, signed_payload) + })?; + *total = total + .checked_add(payload) + .ok_or(rusqlite::Error::IntegralValueOutOfRange(1, i64::MAX))?; + } + } + Ok(totals + .into_iter() + .map(|(table_name, bytes)| TableSizeTelemetrySample { table_name, bytes }) + .collect()) } #[cfg(test)] @@ -232,6 +243,31 @@ mod tests { ); } + #[test] + fn many_tables_do_not_repeat_the_full_page_census_per_table() { + let directory = tempfile::tempdir().unwrap(); + let connection = Connection::open(directory.path().join("many-tables.db")).unwrap(); + for index in 0..36 { + connection + .execute_batch(&format!( + "CREATE TABLE t{index}(body BLOB); + INSERT INTO t{index} VALUES(zeroblob(32768));" + )) + .unwrap(); + } + + let (expected, grouped_steps) = measure_vm(&connection, || reference(&connection)); + let (actual, single_pass_steps) = measure_vm(&connection, || read(&connection).unwrap()); + assert_eq!(actual, expected); + assert_eq!(actual.len(), 36); + assert!(actual.iter().all(|sample| sample.bytes >= 32768)); + assert!(single_pass_steps > 0); + assert!( + single_pass_steps < grouped_steps / 3, + "many-table payload census repeated work: single_pass={single_pass_steps}, grouped={grouped_steps}" + ); + } + #[test] fn interrupted_census_returns_an_error_not_partial_or_zero_sizes_and_can_retry() { let (_directory, connection) = fixture(); From 1b9bf85b00a744f9b3aa679f9d9bbec36f68e0b7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:47:12 -0800 Subject: [PATCH 22/85] fix(storage): retain borrowed dbstat value through lookup Refs #4099. Keep the SQLite page name ValueRef alive while its UTF-8 slice is used to update the bounded table totals. --- .../src/reader/worker/table_sizes.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs index d28106395ab..a85e7201487 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -29,7 +29,8 @@ pub(super) fn read(connection: &Connection) -> rusqlite::Result Date: Sun, 8 Nov 2026 10:11:08 -0800 Subject: [PATCH 23/85] style(storage): align single-pass census with pinned rustfmt Refs #4099 Apply the precise rustfmt diff from CI run 42506693304 without changing census behavior, bounds, or regression assertions. --- .../src/reader/worker/table_sizes.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs index a85e7201487..deb7cbd7ba2 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -35,9 +35,8 @@ pub(super) fn read(connection: &Connection) -> rusqlite::Result(1)?; - let payload = u64::try_from(signed_payload).map_err(|_| { - rusqlite::Error::IntegralValueOutOfRange(1, signed_payload) - })?; + let payload = u64::try_from(signed_payload) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(1, signed_payload))?; *total = total .checked_add(payload) .ok_or(rusqlite::Error::IntegralValueOutOfRange(1, i64::MAX))?; From 2fdaffbe101e24afef30a4365ef1fc33d3fbe0e4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:56:53 -0800 Subject: [PATCH 24/85] fix(storage): count retained SQLite journals in family sizes Refs #4099 A committed PERSIST journal remains allocated but the report's registry and user-sessions family totals counted only main, WAL, and SHM bytes. Count the rollback journal without following foreign symlinks and test a real persisted journal through the offline report and footprint census. --- .../src/retention/storage_report.rs | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 32e187e73d7..24985d2ec25 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1511,7 +1511,9 @@ fn user_sessions_family_bytes(profile_root: &Path) -> u64 { fn database_family_bytes(database_path: &Path) -> u64 { let mut total_bytes = 0u64; - for suffix in ["", "-wal", "-shm"] { + // PERSIST rollback journals remain allocated after a committed transaction; + // TRUNCATE journals are zero-length. Both are physical family bytes. + for suffix in ["", "-wal", "-shm", "-journal"] { // Physical profile bytes must not include a symlink's foreign target. // The full-profile walk separately marks a linked member unavailable. if let Ok(metadata) = std::fs::symlink_metadata(sqlite_family_member(database_path, suffix)) @@ -2660,6 +2662,53 @@ mod tests { ); } + #[asupersync::test] + async fn committed_persist_journal_is_charged_to_user_sessions_family() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let database = profile_root.join(USER_SESSIONS_DB_FILENAME); + let writer = rusqlite::Connection::open(&database).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = PERSIST; + CREATE TABLE receipts(payload BLOB NOT NULL); + INSERT INTO receipts VALUES (zeroblob(262144));", + ) + .unwrap(); + + let journal = sqlite_family_member(&database, "-journal"); + let main_bytes = std::fs::metadata(&database).unwrap().len(); + let journal_bytes = std::fs::metadata(&journal).unwrap().len(); + assert!(journal_bytes > 0, "PERSIST must retain a committed journal"); + let retained_header = std::fs::read(&journal).unwrap(); + assert_eq!( + &retained_header[..28], + &[0u8; 28], + "the retained journal must be committed, not active" + ); + + let report = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(report.user_sessions_db_bytes, main_bytes + journal_bytes); + let footprint = report.full_profile_size.unwrap(); + assert_eq!(footprint.total_bytes, main_bytes + journal_bytes); + assert_eq!(footprint.state, ProfileTotalCoverageStateV1::Complete); + } + + #[cfg(unix)] + #[test] + fn linked_rollback_journal_does_not_charge_foreign_storage() { + let temporary = tempfile::TempDir::new().unwrap(); + let database = temporary.path().join("owned.db"); + std::fs::write(&database, b"owned").unwrap(); + let foreign = temporary.path().join("foreign.bin"); + std::fs::write(&foreign, vec![0u8; 1024 * 1024]).unwrap(); + let journal = sqlite_family_member(&database, "-journal"); + std::os::unix::fs::symlink(&foreign, &journal).unwrap(); + + assert_eq!(database_family_bytes(&database), 5); + } + #[cfg(unix)] #[asupersync::test] async fn dangling_profile_registry_cannot_make_registered_stores_look_unregistered() { From dcca6bcad3fe5b91319b978ebe0335dc8d93f413 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 25/85] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 1a80a56eb512e691916066ff2be4deab5b3c161f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:13:43 -0800 Subject: [PATCH 26/85] fix(doctor): retain footprint when registry enumeration fails Refs #4099 --- .../src/doctor_kernel.rs | 151 +++++++++++++++++- 1 file changed, 143 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 167442e0a0f..ecd80c5c186 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -614,6 +614,31 @@ fn soft_budget_findings( storage_family_read(findings) } +/// A failed registry listing must not hide the physical profile footprint. +/// An unknown registry is not an empty registry: orphan classifications require +/// proven registration state, but file sizes and their bounds do not. +fn profile_footprint_from_registered_roots( + profile_root: &Path, + registered_roots: Option<&std::collections::BTreeMap>, + overrides: &std::collections::BTreeMap, +) -> tracedecay_maintenance::retention::footprint::ProfileFootprintV1 { + use tracedecay_maintenance::retention::footprint::{ + ProjectRootStateV1, census_profile_footprint, + }; + + let empty_roots = std::collections::BTreeMap::new(); + let mut footprint = + census_profile_footprint(profile_root, registered_roots.unwrap_or(&empty_roots), overrides); + if registered_roots.is_none() { + for entry in &mut footprint.entries { + if let Some(project) = entry.project.as_mut() { + project.state = ProjectRootStateV1::Unverifiable; + } + } + } + footprint +} + /// Size every store family under the profile against its bound. Registered /// project roots name each project store's root and whether it still exists. #[tracing::instrument(name = "daemon.doctor.profile_footprint", level = "trace", skip_all)] @@ -625,17 +650,19 @@ async fn profile_footprint_read( let registered = registry .list_code_projects(usize::MAX) .await - .ok()? - .into_iter() - .map(|project| (project.project_id, project.canonical_root)) - .collect::>(); + .ok() + .map(|projects| { + projects + .into_iter() + .map(|project| (project.project_id, project.canonical_root)) + .collect::>() + }); let overrides = retention.store_soft_budgets_bytes.clone(); let profile_root = profile_root.to_path_buf(); tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - use tracedecay_maintenance::retention::footprint::{ - census_profile_footprint, census_sqlite_temp_spill, - }; - let mut footprint = census_profile_footprint(&profile_root, ®istered, &overrides); + use tracedecay_maintenance::retention::footprint::census_sqlite_temp_spill; + let mut footprint = + profile_footprint_from_registered_roots(&profile_root, registered.as_ref(), &overrides); footprint.entries.extend(census_sqlite_temp_spill( Path::new("/proc/self/fd"), &overrides, @@ -646,6 +673,114 @@ async fn profile_footprint_read( .ok() } +#[cfg(test)] +mod profile_footprint_tests { + use std::collections::BTreeMap; + use std::path::Path; + + use tracedecay_contracts::doctor::{ + DoctorEvidenceStateV1, DoctorStorageFamilyReadV1, + }; + use tracedecay_maintenance::retention::footprint::{ + ProfileFamilyV1, ProjectRootStateV1, footprint_findings, + }; + + use super::profile_footprint_from_registered_roots; + + fn state_for_store( + profile: &tracedecay_maintenance::retention::footprint::ProfileFootprintV1, + ) -> ProjectRootStateV1 { + profile + .entries + .iter() + .find(|entry| entry.key == "projects/proj_growing") + .and_then(|entry| entry.project.as_ref()) + .expect("measured project store") + .state + } + + #[test] + fn unreadable_registry_cannot_hide_large_profile_or_mislabel_orphans() { + let temp = tempfile::tempdir().unwrap(); + let profile_root = temp.path().join("profile"); + let store_root = profile_root.join("projects/proj_growing"); + std::fs::create_dir_all(&store_root).unwrap(); + std::fs::create_dir_all(profile_root.join("lcm-payloads")).unwrap(); + std::fs::write(store_root.join("tracedecay.db-wal"), vec![0; 4096]).unwrap(); + std::fs::write(profile_root.join("lcm-payloads/blob"), vec![0; 2048]).unwrap(); + + let overrides = BTreeMap::from([ + ("projects/proj_growing".to_owned(), 1024), + ("profile".to_owned(), 1024), + ]); + // None is a failed registry read, not a successful empty listing. + let unknown = profile_footprint_from_registered_roots(&profile_root, None, &overrides); + assert_eq!(unknown.total_bytes, 6144); + assert_eq!(state_for_store(&unknown), ProjectRootStateV1::Unverifiable); + let project = unknown + .entries + .iter() + .find(|entry| entry.key == "projects/proj_growing") + .unwrap(); + assert_eq!(project.family, ProfileFamilyV1::ProjectStore); + assert_eq!(project.bytes, 4096); + assert_eq!(project.bound_bytes, 1024); + assert_eq!(project.project.as_ref().unwrap().registered_root, None); + + let DoctorStorageFamilyReadV1::Observed { findings } = footprint_findings(&unknown) else { + panic!("physical footprint must remain observable without a registry"); + }; + let named = findings + .into_iter() + .map(|finding| { + ( + finding.finding().evidence()[0] + .reference() + .as_str() + .to_owned(), + finding.finding().state(), + ) + }) + .collect::>(); + assert_eq!( + named.get("projects/proj_growing"), + Some(&DoctorEvidenceStateV1::Degraded) + ); + assert_eq!( + named.get("profile"), + Some(&DoctorEvidenceStateV1::Degraded) + ); + + // Distinguish failed enumeration from an authoritative empty registry, + // and from a registered root whose filesystem directory was deleted. + let empty = BTreeMap::new(); + let unregistered = + profile_footprint_from_registered_roots(&profile_root, Some(&empty), &overrides); + assert_eq!( + state_for_store(&unregistered), + ProjectRootStateV1::Unregistered + ); + let registered = BTreeMap::from([( + "proj_growing".to_owned(), + temp.path().join("deleted-worktree").display().to_string(), + )]); + let missing = + profile_footprint_from_registered_roots(&profile_root, Some(®istered), &overrides); + assert_eq!(state_for_store(&missing), ProjectRootStateV1::Missing); + assert_eq!( + missing + .entries + .iter() + .find(|entry| entry.key == "projects/proj_growing") + .and_then(|entry| entry.project.as_ref()) + .and_then(|project| project.registered_root.as_ref()) + .map(String::as_str), + Some(temp.path().join("deleted-worktree").to_str().unwrap()) + ); + assert!(Path::new(®istered["proj_growing"]).exists() == false); + } +} + /// The generations the retention pass protects: the one the mounted scheduler /// serves, through either the sealed seat or the text projection, and every /// live native-preview candidate binding of the repository. This is the set From edb27a364af80090b21507ad3218ea03cc7d7ff4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:14:32 -0800 Subject: [PATCH 27/85] test(doctor): use stable missing-root fixture Refs #4099 --- .../tracedecay-daemon-service/src/doctor_kernel.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index ecd80c5c186..f043e3c80d4 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -676,11 +676,7 @@ async fn profile_footprint_read( #[cfg(test)] mod profile_footprint_tests { use std::collections::BTreeMap; - use std::path::Path; - - use tracedecay_contracts::doctor::{ - DoctorEvidenceStateV1, DoctorStorageFamilyReadV1, - }; + use tracedecay_contracts::doctor::{DoctorEvidenceStateV1, DoctorStorageFamilyReadV1}; use tracedecay_maintenance::retention::footprint::{ ProfileFamilyV1, ProjectRootStateV1, footprint_findings, }; @@ -760,9 +756,10 @@ mod profile_footprint_tests { state_for_store(&unregistered), ProjectRootStateV1::Unregistered ); + let deleted_root = temp.path().join("deleted-worktree"); let registered = BTreeMap::from([( "proj_growing".to_owned(), - temp.path().join("deleted-worktree").display().to_string(), + deleted_root.display().to_string(), )]); let missing = profile_footprint_from_registered_roots(&profile_root, Some(®istered), &overrides); @@ -775,9 +772,9 @@ mod profile_footprint_tests { .and_then(|entry| entry.project.as_ref()) .and_then(|project| project.registered_root.as_ref()) .map(String::as_str), - Some(temp.path().join("deleted-worktree").to_str().unwrap()) + deleted_root.to_str() ); - assert!(Path::new(®istered["proj_growing"]).exists() == false); + assert!(!deleted_root.exists()); } } From 67a0121ea236798760bc3c8b58355026349e6b9b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:16:21 -0800 Subject: [PATCH 28/85] style(doctor): make footprint imports explicit Refs #4099 --- .../src/doctor_kernel.rs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index f043e3c80d4..205df53c5bd 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -8,7 +8,7 @@ //! `Unsupported`/`Absent`/`Denied`/`Unknown` read, never a fabricated healthy //! result, and partial coverage carries its real reason. //! -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; use tracedecay_application::advisory::github_runtime::{ @@ -44,6 +44,9 @@ use tracedecay_domain::CodeGenerationId; use tracedecay_global_db::{GlobalDbNativeIntegrationStore, RegisteredGlobalDb}; use crate::DaemonFeedbackRuntimeRegistrar; +use tracedecay_maintenance::retention::footprint::{ + ProfileFootprintV1, ProjectRootStateV1, census_profile_footprint, census_sqlite_temp_spill, +}; use tracedecay_maintenance::telemetry::GuardedStoreTelemetryPort; use tracedecay_session_temporal_store::SessionTemporalAccess; @@ -619,14 +622,10 @@ fn soft_budget_findings( /// proven registration state, but file sizes and their bounds do not. fn profile_footprint_from_registered_roots( profile_root: &Path, - registered_roots: Option<&std::collections::BTreeMap>, - overrides: &std::collections::BTreeMap, -) -> tracedecay_maintenance::retention::footprint::ProfileFootprintV1 { - use tracedecay_maintenance::retention::footprint::{ - ProjectRootStateV1, census_profile_footprint, - }; - - let empty_roots = std::collections::BTreeMap::new(); + registered_roots: Option<&BTreeMap>, + overrides: &BTreeMap, +) -> ProfileFootprintV1 { + let empty_roots = BTreeMap::new(); let mut footprint = census_profile_footprint(profile_root, registered_roots.unwrap_or(&empty_roots), overrides); if registered_roots.is_none() { @@ -646,7 +645,7 @@ async fn profile_footprint_read( registry: &RegisteredGlobalDb, profile_root: &Path, retention: &tracedecay_configuration::RetentionConfig, -) -> Option { +) -> Option { let registered = registry .list_code_projects(usize::MAX) .await @@ -655,12 +654,11 @@ async fn profile_footprint_read( projects .into_iter() .map(|project| (project.project_id, project.canonical_root)) - .collect::>() + .collect::>() }); let overrides = retention.store_soft_budgets_bytes.clone(); let profile_root = profile_root.to_path_buf(); tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - use tracedecay_maintenance::retention::footprint::census_sqlite_temp_spill; let mut footprint = profile_footprint_from_registered_roots(&profile_root, registered.as_ref(), &overrides); footprint.entries.extend(census_sqlite_temp_spill( From abd995624c53c9438d66c4dccb2198b0c5f7da86 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:14:56 -0800 Subject: [PATCH 29/85] style(doctor): format registered footprint handling --- .../tracedecay-daemon-service/src/doctor_kernel.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 205df53c5bd..1faed0953a1 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -626,8 +626,11 @@ fn profile_footprint_from_registered_roots( overrides: &BTreeMap, ) -> ProfileFootprintV1 { let empty_roots = BTreeMap::new(); - let mut footprint = - census_profile_footprint(profile_root, registered_roots.unwrap_or(&empty_roots), overrides); + let mut footprint = census_profile_footprint( + profile_root, + registered_roots.unwrap_or(&empty_roots), + overrides, + ); if registered_roots.is_none() { for entry in &mut footprint.entries { if let Some(project) = entry.project.as_mut() { @@ -740,10 +743,7 @@ mod profile_footprint_tests { named.get("projects/proj_growing"), Some(&DoctorEvidenceStateV1::Degraded) ); - assert_eq!( - named.get("profile"), - Some(&DoctorEvidenceStateV1::Degraded) - ); + assert_eq!(named.get("profile"), Some(&DoctorEvidenceStateV1::Degraded)); // Distinguish failed enumeration from an authoritative empty registry, // and from a registered root whose filesystem directory was deleted. From 0bb47ea1aa94be33a83d619f48c40f6a338370c9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:39:25 -0800 Subject: [PATCH 30/85] fix(storage): refuse malformed registry project paths Refs #4099. Validate all offline and daemon report project IDs at the common store-sampling boundary, before constructing or walking their paths. Keep direct project reports and persisted registry rows on one canonical path-segment policy. Regression proves path escape is rejected before any scan and a normal store is still measured. --- .../src/retention/storage_report.rs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 24985d2ec25..53354a02732 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1296,6 +1296,14 @@ fn append_project_report( remaining_entries: &mut usize, walk_control: &StorageReportWalkControl, ) -> tracedecay_domain::errors::Result<()> { + // Offline registry snapshots and live registry rows must obey the same + // path-segment authority as explicit project-scoped reports. Otherwise a + // corrupt project ID can escape projects/ and measure an unrelated tree. + tracedecay_runtime_core::storage::validate_project_id(project_id).map_err(|message| { + TraceDecayError::Config { + message: format!("invalid registered project id {project_id:?}: {message}"), + } + })?; let data_root = profile_root.join("projects").join(project_id); let code_index_root = data_root.join("code-index-v1"); let code_index_store_root = @@ -1975,6 +1983,58 @@ mod tests { use crate::retention::footprint::census_profile_footprint; use tracedecay_runtime_core::db::engine::TestConnection; + #[test] + fn registered_report_refuses_project_ids_that_escape_the_profile() { + let temp = tempfile::tempdir().unwrap(); + let profile = temp.path().join("profile"); + let outside = temp.path().join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::create_dir_all(profile.join("projects/proj_valid")).unwrap(); + std::fs::write(outside.join("foreign"), [7u8; 16]).unwrap(); + std::fs::write(profile.join("projects/proj_valid/local"), [3u8; 9]).unwrap(); + + let mut stores = Vec::new(); + let mut retention = Vec::new(); + let mut availability = Vec::new(); + let mut budget = MAX_STORAGE_REPORT_WALK_ENTRIES; + for project_id in ["../../outside", "../outside", "nested/store", "..", ".", "", "bad\\name"] { + let error = append_project_report( + &profile, + project_id, + "/work/valid", + Err(RETENTION_PROTECTION_UNRESOLVED), + &mut stores, + &mut retention, + &mut availability, + false, + &mut budget, + &StorageReportWalkControl::default(), + ) + .unwrap_err(); + assert!( + error.to_string().contains("invalid registered project id"), + "{project_id:?}: {error}" + ); + assert!(stores.is_empty(), "foreign storage was adopted"); + assert_eq!(budget, MAX_STORAGE_REPORT_WALK_ENTRIES); + } + assert_eq!(std::fs::read(outside.join("foreign")).unwrap(), [7u8; 16]); + + let valid = build_project_storage_report( + &profile, + "proj_valid", + Path::new("/work/valid"), + Err(RETENTION_PROTECTION_UNRESOLVED), + ) + .unwrap(); + assert_eq!(valid.stores.len(), 1); + assert_eq!(valid.stores[0].total_bytes, 9); + assert_eq!( + valid.stores[0].size_coverage.state, + StorageReportSizeStateV1::Complete + ); + } + fn store(project_id: &str, total_bytes: u64) -> StoreSizeReportEntry { StoreSizeReportEntry { project_id: project_id.to_owned(), From aa507c45c1b67cba37bf73b0859d68116b1fa234 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:40:13 -0800 Subject: [PATCH 31/85] style(storage): format malformed registry regression Refs #4099. Keep the invalid-identity fixture list compliant with the pinned formatter. --- .../src/retention/storage_report.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 53354a02732..146efb59fa9 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -1997,7 +1997,15 @@ mod tests { let mut retention = Vec::new(); let mut availability = Vec::new(); let mut budget = MAX_STORAGE_REPORT_WALK_ENTRIES; - for project_id in ["../../outside", "../outside", "nested/store", "..", ".", "", "bad\\name"] { + for project_id in [ + "../../outside", + "../outside", + "nested/store", + "..", + ".", + "", + "bad\\name", + ] { let error = append_project_report( &profile, project_id, From cec41066825dae986f61df401b3fb4e1d3c0db72 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:26:24 -0800 Subject: [PATCH 32/85] fix(storage): re-probe table growth after daemon restart Refs #4099. Retain measured baselines and latest evidence across restarts, but keep deadline-based skip markers process-local so a new reader can retry after improvements. Add a real-store restart regression that reproduces the previous 24-hour suppression. --- .../tracedecay-maintenance/src/telemetry.rs | 102 +++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index c83ff5efb69..01451afa647 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -51,7 +51,7 @@ struct TableWatermark { /// size: a multi-gigabyte session store needs minutes. Once a walk has run /// out of time, repeating it on a store at least as large only spends the /// deadline again. A smaller store, or a day-old mark, is walked again. -#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] struct OverBudgetTableSizeWalk { page_count: u64, observed_at: UtcMicros, @@ -99,6 +99,10 @@ fn settle_table_size_walk( struct TableGrowthStateV1 { watermarks: Option>, latest: Option, + // A timed-out census describes the work cost of this executable, not a + // durable store limit. Restart must re-probe after reader improvements. + // Persist only actual samples and their watermarks. + #[serde(skip)] over_budget: Option, } @@ -1331,10 +1335,15 @@ mod table_growth_tests { mod table_growth_restart_tests { use std::collections::BTreeSet; - use tracedecay_contracts::storage::TableGrowthTelemetryReadV1; + use tracedecay_contracts::now_micros; + use tracedecay_contracts::storage::{ + TableGrowthTelemetryReadV1, TableGrowthUnavailableReasonV1, + }; use tracedecay_runtime_core::cancellation::CancellationToken; - use super::{StoreTelemetrySamplingRegistry, storage_telemetry_request_context}; + use super::{ + OverBudgetTableSizeWalk, StoreTelemetrySamplingRegistry, storage_telemetry_request_context, + }; #[asupersync::test] async fn a_restarted_registry_reports_the_persisted_measurement() { @@ -1396,6 +1405,93 @@ mod table_growth_restart_tests { .all(|sample| sample.previous_observed_at == observed_at) ); } + #[asupersync::test] + async fn restart_reprobes_a_timed_out_census_without_losing_the_baseline() { + let temporary = tempfile::tempdir().expect("profile root"); + let profile_root = temporary.path().join("profile"); + let database_path = temporary.path().join("project.db"); + let (database, scope) = super::table_growth_tests::growth_fixture(&database_path).await; + let paths = BTreeSet::from([database_path.clone()]); + let before = StoreTelemetrySamplingRegistry::persisted_under(&profile_root); + assert!(before.register_port(&database_path, &scope, || { + database.storage_telemetry_handle() + })); + before + .advance_registered(&paths, &paths, &CancellationToken::new()) + .await; + let context = storage_telemetry_request_context(scope.clone()).expect("reader context"); + let (store, port) = before + .registered_port(&database_path, &scope) + .expect("registered port"); + let TableGrowthTelemetryReadV1::BaselineEstablished { observed_at, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("first maintenance pass must establish a baseline"); + }; + + database + .execute_write_batch( + "grow retained table after baseline", + "INSERT INTO growth_payload VALUES (zeroblob(32768));", + ) + .await + .expect("insert a nonzero measured payload"); + port.table_growth + .lock() + .expect("table growth state") + .over_budget = Some(OverBudgetTableSizeWalk { + page_count: 1, + observed_at: now_micros(), + }); + before + .advance_registered(&paths, &paths, &CancellationToken::new()) + .await; + let skipped = port.latest_table_growth(&context, &store); + assert!(matches!( + skipped, + TableGrowthTelemetryReadV1::Unknown { + reason: TableGrowthUnavailableReasonV1::ExceedsMeasurementBudget, + .. + } + )); + assert!(port.table_growth_state().over_budget.is_some()); + drop(port); + drop(before); + + let after = StoreTelemetrySamplingRegistry::persisted_under(&profile_root); + assert!(after.register_port(&database_path, &scope, || { + database.storage_telemetry_handle() + })); + let (store, port) = after + .registered_port(&database_path, &scope) + .expect("registered port after restart"); + assert_eq!( + port.latest_table_growth(&context, &store), + skipped, + "the last unavailability is retained until a real new measurement" + ); + assert_eq!( + port.table_growth_state().over_budget, + None, + "a previous executable's timed-out census must not suppress the new one" + ); + + after + .advance_registered(&paths, &paths, &CancellationToken::new()) + .await; + let TableGrowthTelemetryReadV1::Observed { samples, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("restart must retry and compare to the retained baseline"); + }; + let grown = samples + .iter() + .find(|sample| sample.table.as_str() == "growth_payload") + .expect("the retried census reports real table bytes"); + assert_eq!(grown.previous_observed_at, observed_at); + assert!(grown.growth_bytes().get() >= 32_768); + assert_eq!(port.table_growth_state().over_budget, None); + } } #[cfg(test)] From bcd66553de290a0ae0d32d77db871d72d43f5977 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:52:10 -0800 Subject: [PATCH 33/85] fix(storage): attribute graph sidecars to their container Refs #4099 --- .../src/retention/storage_report.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 146efb59fa9..c4b9c9e1e42 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -210,6 +210,10 @@ impl StoreKindBytesV1 { } else if first == GRAPH_REPLAY_POOL_DIRECTORY { &mut self.graph_replay_pool } else if first.ends_with(GRAPH_CONTAINER_WAL_SUFFIX) + // The container's verified and spill sidecars consume the same + // graph-container budget; they are not miscellaneous store data. + || first.ends_with(".grafeo.verified") + || first.ends_with(".grafeo.spill") || (first.ends_with(GRAPH_CONTAINER_SUFFIX) && components.next().is_none()) { &mut self.graph_container @@ -3564,6 +3568,8 @@ mod tests { ("tracedecay.graph-replay/generation.json", 40), ("tracedecay.grafeo", 500), ("tracedecay.grafeo.wal/wal_00000001.log", 60), + ("tracedecay.grafeo.verified", 80), + ("tracedecay.grafeo.spill/inflight", 120), ("sessions.db", 900), ("sessions.db-wal", 10), (".sessions.db.host-admission/claim", 5), @@ -3602,13 +3608,13 @@ mod tests { sealed_graph: 3_000, text_artifacts: 2_050, generation_artifacts: 1_000, - graph_container: 560, + graph_container: 760, graph_replay_pool: 40, sessions: 915, other: 7, } ); - assert_eq!(store.total_bytes, 11_768); + assert_eq!(store.total_bytes, 11_968); assert_eq!(store.total_bytes, walked_bytes); assert_eq!(store.unavailable_entry_count, 0); } From ad8d08521fcb9f6e3978fbf5b84dbe207ebfa107 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:53:01 -0800 Subject: [PATCH 34/85] style(storage): keep graph sidecar classification precise Refs #4099 --- crates/tracedecay-maintenance/src/retention/storage_report.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index c4b9c9e1e42..f83b5e295e0 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -210,8 +210,6 @@ impl StoreKindBytesV1 { } else if first == GRAPH_REPLAY_POOL_DIRECTORY { &mut self.graph_replay_pool } else if first.ends_with(GRAPH_CONTAINER_WAL_SUFFIX) - // The container's verified and spill sidecars consume the same - // graph-container budget; they are not miscellaneous store data. || first.ends_with(".grafeo.verified") || first.ends_with(".grafeo.spill") || (first.ends_with(GRAPH_CONTAINER_SUFFIX) && components.next().is_none()) From d992d04a40ca88351fedcb3ac95b4ea4ad674c3b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:26:50 -0800 Subject: [PATCH 35/85] fix(doctor): preserve long-path footprint findings Refs #4099 --- .../src/retention/footprint.rs | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 7c6d4ead0c0..2684ccfdee9 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -6,10 +6,12 @@ //! That is why the ceiling is an entry count: a byte budget small enough to be //! cheap is smaller than one real graph container. +use std::borrow::Cow; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tracedecay_contracts::doctor::{ DoctorCoverageCompletenessV1, DoctorCoverageStatementV1, DoctorEvidenceRefV1, DoctorEvidenceReferenceV1, DoctorEvidenceStateV1, DoctorFindingFamilyV1, DoctorFindingV1, @@ -530,6 +532,16 @@ fn footprint_keys(relative: &Path) -> Option { }) } +/// Keep a stable evidence reference even for deeply nested filesystem names. +/// The report retains the complete key; Doctor references have a smaller bound. +fn doctor_footprint_reference_key(key: &str) -> Cow<'_, str> { + if DoctorEvidenceReferenceV1::new(key.to_owned()).is_ok() { + return Cow::Borrowed(key); + } + let digest = Sha256::digest(key.as_bytes()); + Cow::Owned(format!("footprint/sha256:{digest:x}")) +} + /// One `OverBudgetStore` finding per footprint entry plus one for the profile /// total. A floor above its bound proves the overage, so an over-bound entry /// is degraded with complete coverage even when bytes went unread. @@ -537,9 +549,15 @@ fn footprint_keys(relative: &Path) -> Option { pub fn footprint_findings(footprint: &ProfileFootprintV1) -> DoctorStorageFamilyReadV1 { let mut findings = Vec::with_capacity(footprint.entries.len() + 1); for entry in &footprint.entries { + let evidence_key = doctor_footprint_reference_key(&entry.key); + let subject = if evidence_key.as_ref() == entry.key.as_str() { + entry_statement(entry) + } else { + format!("footprint entry {evidence_key} (full path in storage report)") + }; let Some(finding) = bounded_finding( - &entry.key, - &entry_statement(entry), + evidence_key.as_ref(), + &subject, entry.bytes, entry.bound_bytes, entry.unavailable_entry_count > 0 || footprint.truncated, @@ -1117,6 +1135,55 @@ mod tests { assert_eq!(remaining, 9); } + #[cfg(unix)] + #[test] + fn deeply_nested_wal_cannot_hide_large_store_or_profile_findings() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + let mut directory = profile.join("projects/proj_deep"); + for index in 0..6 { + directory.push(format!("shard_{index}_{}", "x".repeat(192))); + } + sized(&directory.join("sessions.wal"), 2 * GIB); + let overrides = BTreeMap::from([ + ("projects/proj_deep".to_owned(), 1), + (PROFILE_TOTAL_KEY.to_owned(), 1), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 2 * GIB); + let wal = footprint + .entries + .iter() + .find(|entry| entry.family == ProfileFamilyV1::Wal) + .expect("named WAL footprint"); + assert!(wal.key.len() > 1024, "fixture must exceed Doctor identity bound"); + assert_eq!(wal.bytes, 2 * GIB); + + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + let digest = Sha256::digest(wal.key.as_bytes()); + let reference = format!("footprint/sha256:{digest:x}"); + let wal_finding = &observed[&reference]; + assert_eq!( + wal_finding.finding().state(), + DoctorEvidenceStateV1::Degraded + ); + let statement = wal_finding.finding().coverage().statement(); + assert!(statement.contains("full path in storage report"), "{statement}"); + assert!(statement.contains(&format_bytes(2 * GIB)), "{statement}"); + assert!( + statement.contains(&format_bytes(default_bound_bytes(ProfileFamilyV1::Wal))), + "{statement}" + ); + for reference in ["projects/proj_deep", PROFILE_TOTAL_KEY] { + assert_eq!( + observed[reference].finding().state(), + DoctorEvidenceStateV1::Degraded, + "a long WAL identity must not suppress the other overages" + ); + } + } + #[test] fn an_unreadable_walk_root_is_accounted_as_unavailable() { let temporary = tempfile::tempdir().expect("walk fixture"); From 44bea7e79809f223e24f435eeac9c1979b28cd22 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:27:59 -0800 Subject: [PATCH 36/85] simplify(doctor): reuse canonical footprint digest Refs #4099 --- crates/tracedecay-maintenance/src/retention/footprint.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 2684ccfdee9..466fa96b063 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -11,13 +11,13 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; use tracedecay_contracts::doctor::{ DoctorCoverageCompletenessV1, DoctorCoverageStatementV1, DoctorEvidenceRefV1, DoctorEvidenceReferenceV1, DoctorEvidenceStateV1, DoctorFindingFamilyV1, DoctorFindingV1, DoctorStorageFamilyReadV1, DoctorStorageFindingKindV1, DoctorStorageFindingV1, storage_family_read, }; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_global_db::registry_maintenance::{RootLivenessV1, probe_root}; use tracedecay_runtime_core::text::format_bytes; @@ -538,8 +538,7 @@ fn doctor_footprint_reference_key(key: &str) -> Cow<'_, str> { if DoctorEvidenceReferenceV1::new(key.to_owned()).is_ok() { return Cow::Borrowed(key); } - let digest = Sha256::digest(key.as_bytes()); - Cow::Owned(format!("footprint/sha256:{digest:x}")) + Cow::Owned(format!("footprint/sha256:{}", sha256_hex(key.as_bytes()))) } /// One `OverBudgetStore` finding per footprint entry plus one for the profile @@ -1161,8 +1160,7 @@ mod tests { let observed = findings(&footprint); assert_eq!(observed.len(), footprint.entries.len() + 1); - let digest = Sha256::digest(wal.key.as_bytes()); - let reference = format!("footprint/sha256:{digest:x}"); + let reference = format!("footprint/sha256:{}", sha256_hex(wal.key.as_bytes())); let wal_finding = &observed[&reference]; assert_eq!( wal_finding.finding().state(), From 55ef65ea443612a9fd1f9aeeeabf2f649886edcf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:50:37 -0800 Subject: [PATCH 37/85] fix(doctor): retain the cause of partial hashed findings --- .../src/retention/footprint.rs | 54 +++++++++++++------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 466fa96b063..8a42b939dd7 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -549,11 +549,7 @@ pub fn footprint_findings(footprint: &ProfileFootprintV1) -> DoctorStorageFamily let mut findings = Vec::with_capacity(footprint.entries.len() + 1); for entry in &footprint.entries { let evidence_key = doctor_footprint_reference_key(&entry.key); - let subject = if evidence_key.as_ref() == entry.key.as_str() { - entry_statement(entry) - } else { - format!("footprint entry {evidence_key} (full path in storage report)") - }; + let subject = entry_statement(entry, &evidence_key); let Some(finding) = bounded_finding( evidence_key.as_ref(), &subject, @@ -593,7 +589,7 @@ fn floor_note(unavailable_entry_count: usize, truncated: bool) -> String { } } -fn entry_statement(entry: &ProfileFootprintEntryV1) -> String { +fn entry_statement(entry: &ProfileFootprintEntryV1, evidence_key: &str) -> String { let label = match entry.family { ProfileFamilyV1::ProjectStore => "project store", ProfileFamilyV1::GraphContainer => "graph container", @@ -608,6 +604,10 @@ fn entry_statement(entry: &ProfileFootprintEntryV1) -> String { ProfileFamilyV1::SqliteTempSpill => "unlinked SQLite temp files", ProfileFamilyV1::Other => "profile entry", }; + let floor = floor_note(entry.unavailable_entry_count, false); + if evidence_key != entry.key { + return format!("{label} {evidence_key} (full path in storage report){floor}"); + } let root = entry .project @@ -626,11 +626,7 @@ fn entry_statement(entry: &ProfileFootprintEntryV1) -> String { format_bytes(kinds.graph_replay_pool) ) }); - format!( - "{label} {}{root}{breakdown}{}", - entry.key, - floor_note(entry.unavailable_entry_count, false) - ) + format!("{label} {evidence_key}{root}{breakdown}{floor}") } fn bounded_finding( @@ -1136,15 +1132,24 @@ mod tests { #[cfg(unix)] #[test] - fn deeply_nested_wal_cannot_hide_large_store_or_profile_findings() { + fn hashed_wal_reference_keeps_overages_and_partial_coverage() { let temp = tempfile::tempdir().expect("footprint fixture"); let profile = temp.path().join("profile"); let mut directory = profile.join("projects/proj_deep"); + // Linux can create a key longer than Doctor's 1,024-byte reference. + // On Unix filesystems with a smaller path limit, a control character + // reaches the same reference fallback through a real directory walk. + #[cfg(target_os = "linux")] for index in 0..6 { directory.push(format!("shard_{index}_{}", "x".repeat(192))); } - sized(&directory.join("sessions.wal"), 2 * GIB); - let overrides = BTreeMap::from([ + #[cfg(not(target_os = "linux"))] + directory.push("shard\narchive"); + let wal_directory = directory.join("sessions.wal"); + sized(&wal_directory.join("segment-1"), 2 * GIB); + std::os::unix::fs::symlink("missing", wal_directory.join("segment-2")) + .expect("unreadable WAL segment"); + let mut overrides = BTreeMap::from([ ("projects/proj_deep".to_owned(), 1), (PROFILE_TOTAL_KEY.to_owned(), 1), ]); @@ -1155,7 +1160,12 @@ mod tests { .iter() .find(|entry| entry.family == ProfileFamilyV1::Wal) .expect("named WAL footprint"); - assert!(wal.key.len() > 1024, "fixture must exceed Doctor identity bound"); + #[cfg(target_os = "linux")] + assert!( + wal.key.len() > 1024, + "fixture must exceed Doctor identity bound" + ); + assert!(DoctorEvidenceReferenceV1::new(wal.key.clone()).is_err()); assert_eq!(wal.bytes, 2 * GIB); let observed = findings(&footprint); @@ -1167,7 +1177,10 @@ mod tests { DoctorEvidenceStateV1::Degraded ); let statement = wal_finding.finding().coverage().statement(); - assert!(statement.contains("full path in storage report"), "{statement}"); + assert!( + statement.contains("full path in storage report"), + "{statement}" + ); assert!(statement.contains(&format_bytes(2 * GIB)), "{statement}"); assert!( statement.contains(&format_bytes(default_bound_bytes(ProfileFamilyV1::Wal))), @@ -1180,6 +1193,15 @@ mod tests { "a long WAL identity must not suppress the other overages" ); } + + overrides.insert(wal.key.clone(), 3 * GIB); + let partial = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + let observed = findings(&partial); + let finding = observed[&reference].finding(); + assert_eq!(finding.state(), DoctorEvidenceStateV1::Partial); + let statement = finding.coverage().statement(); + assert!(statement.contains("WAL "), "{statement}"); + assert!(statement.contains("1 unreadable entries"), "{statement}"); } #[test] From 385fa23124710f2a8478256c6b90a3a74ef055df Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:51:28 -0800 Subject: [PATCH 38/85] fix(storage): include graph sidecars in Doctor container bounds Refs #4099 --- .../src/retention/footprint.rs | 83 +++++++++++++++++-- 1 file changed, 76 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 8a42b939dd7..e066b36037d 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -487,13 +487,22 @@ fn footprint_keys(relative: &Path) -> Option { ProfileFamilyV1::Wal, ) }) - .or_else(|| { - (inner.len() == 1 && inner[0].ends_with(GRAPH_CONTAINER_SUFFIX)).then(|| { - ( - format!("{store_key}/{}", inner[0]), - ProfileFamilyV1::GraphContainer, - ) - }) + .or_else(|| match inner { + [name] => { + // Verified and spill sidecars consume the same container + // footprint as the primary .grafeo file. + let stem = name + .strip_suffix(".verified") + .or_else(|| name.strip_suffix(".spill")) + .unwrap_or(name); + stem.ends_with(GRAPH_CONTAINER_SUFFIX).then(|| { + ( + format!("{store_key}/{stem}"), + ProfileFamilyV1::GraphContainer, + ) + }) + } + _ => None, }); return Some(FootprintKeys { primary: (store_key, ProfileFamilyV1::ProjectStore), @@ -1204,6 +1213,66 @@ mod tests { assert!(statement.contains("1 unreadable entries"), "{statement}"); } + #[test] + fn project_container_bound_includes_verified_and_spill_sidecars() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + let project = "projects/proj_sidecars"; + for (name, bytes) in [ + ("graph.grafeo", 5), + ("graph.grafeo.verified", 7), + ("graph.grafeo.spill", 11), + ("graph.grafeo.wal/segment", 13), + ("other.grafeo.spill", 17), + ("not-a-graph.grafeo.tmp", 19), + ] { + sized(&profile.join(project).join(name), bytes); + } + let container_key = format!("{project}/graph.grafeo"); + let sibling_key = format!("{project}/other.grafeo"); + let wal_key = format!("{project}/graph.grafeo.wal"); + let overrides = BTreeMap::from([(container_key.clone(), 10), (sibling_key.clone(), 16)]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 72); + assert_eq!(entry(&footprint, project).bytes, 72); + assert_eq!( + entry(&footprint, project) + .kinds + .expect("physical project-kind census") + .graph_container, + 53 + ); + assert_eq!( + entry(&footprint, &container_key).family, + ProfileFamilyV1::GraphContainer + ); + assert_eq!(entry(&footprint, &container_key).bytes, 23); + assert_eq!(entry(&footprint, &sibling_key).bytes, 17); + assert_eq!(entry(&footprint, &wal_key).family, ProfileFamilyV1::Wal); + assert_eq!(entry(&footprint, &wal_key).bytes, 13); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + + let observed = findings(&footprint); + assert_eq!( + observed[&container_key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "the former main-file-only overlay would falsely report healthy" + ); + assert_eq!( + observed[&sibling_key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "an orphaned spill still consumes a named container bound" + ); + assert_eq!( + observed[&wal_key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "WAL pressure remains its own bound, not double-charged to containers" + ); + } + #[test] fn an_unreadable_walk_root_is_accounted_as_unavailable() { let temporary = tempfile::tempdir().expect("walk fixture"); From d9faf827343c702e6260030f136c3d3a0f6b2c0b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:34:43 -0800 Subject: [PATCH 39/85] fix(storage): exclude unowned graph suffixes from container bounds Refs #4099. Match only graph containers and their verified/spill sidecars in the shared physical profile census, preserving unrelated files as distinct entries and accurate container overage findings. --- .../src/retention/footprint.rs | 77 ++++++++++++++++--- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index e066b36037d..0c7f519ec3c 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -467,6 +467,16 @@ fn is_wal(name: &str) -> bool { name.ends_with("-wal") || name.ends_with(".wal") } +/// Only canonical graph containers and their recognized sidecars share a +/// container bound. Unrelated suffixes must remain separate profile entries. +fn graph_container_stem(name: &str) -> Option<&str> { + let stem = name + .strip_suffix(".verified") + .or_else(|| name.strip_suffix(".spill")) + .unwrap_or(name); + stem.ends_with(GRAPH_CONTAINER_SUFFIX).then_some(stem) +} + /// The footprint keys one profile-relative path belongs to. A WAL directory /// is one entry keyed by the directory; its segments sum into it. fn footprint_keys(relative: &Path) -> Option { @@ -491,11 +501,7 @@ fn footprint_keys(relative: &Path) -> Option { [name] => { // Verified and spill sidecars consume the same container // footprint as the primary .grafeo file. - let stem = name - .strip_suffix(".verified") - .or_else(|| name.strip_suffix(".spill")) - .unwrap_or(name); - stem.ends_with(GRAPH_CONTAINER_SUFFIX).then(|| { + graph_container_stem(name).map(|stem| { ( format!("{store_key}/{stem}"), ProfileFamilyV1::GraphContainer, @@ -512,11 +518,8 @@ fn footprint_keys(relative: &Path) -> Option { } let primary = if let Some(end) = components.iter().position(|name| is_wal(name)) { (components[..=end].join("/"), ProfileFamilyV1::Wal) - } else if let Some(stem) = first - .find(GRAPH_CONTAINER_SUFFIX) - .map(|at| &first[..at + GRAPH_CONTAINER_SUFFIX.len()]) - { - // A top-level container and its sidecars (`.verified`, `.spill`). + } else if let Some(stem) = graph_container_stem(first) { + // A top-level container and only its recognized sidecars. (stem.to_owned(), ProfileFamilyV1::GraphContainer) } else if first == "global.db" || first.starts_with("global.db-") { ("global.db".to_owned(), ProfileFamilyV1::ProfileRegistry) @@ -1213,6 +1216,60 @@ mod tests { assert!(statement.contains("1 unreadable entries"), "{statement}"); } + #[test] + fn top_level_container_bound_excludes_unrelated_grafeo_suffixes() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (name, bytes) in [ + ("user-sessions.grafeo", 5), + ("user-sessions.grafeo.verified", 7), + ("user-sessions.grafeo.spill", 11), + ("user-sessions.grafeo.notes", 30), + ("other.grafeo.metrics", 40), + ] { + sized(&profile.join(name), bytes); + } + let container_key = "user-sessions.grafeo"; + let overrides = BTreeMap::from([(container_key.to_owned(), 24)]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + + assert_eq!(footprint.total_bytes, 93); + assert_eq!(entry(&footprint, container_key).bytes, 23); + assert_eq!( + entry(&footprint, container_key).family, + ProfileFamilyV1::GraphContainer + ); + for (name, bytes) in [ + ("user-sessions.grafeo.notes", 30), + ("other.grafeo.metrics", 40), + ] { + let unrelated = entry(&footprint, name); + assert_eq!(unrelated.family, ProfileFamilyV1::Other); + assert_eq!(unrelated.bytes, bytes); + } + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + let observed = findings(&footprint); + assert_eq!( + observed[container_key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "unowned files must not falsely trip the graph-container bound" + ); + + let over = census_profile_footprint( + &profile, + &BTreeMap::new(), + &BTreeMap::from([(container_key.to_owned(), 22)]), + ); + assert_eq!( + findings(&over)[container_key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "actual graph sidecar bytes must still trip their bound" + ); + } + #[test] fn project_container_bound_includes_verified_and_spill_sidecars() { let temp = tempfile::tempdir().expect("footprint fixture"); From 4b542edc22f625d23d4929ea244ef11ed1261c7d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:08:01 -0800 Subject: [PATCH 40/85] fix(storage): include spill directories in graph container bounds --- .../src/retention/footprint.rs | 42 +++++++------------ .../src/retention/storage_report.rs | 19 ++++++--- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 0c7f519ec3c..8445b5eac72 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -22,7 +22,7 @@ use tracedecay_global_db::registry_maintenance::{RootLivenessV1, probe_root}; use tracedecay_runtime_core::text::format_bytes; use super::storage_report::{ - FullProfileSizeV1, GRAPH_CONTAINER_SUFFIX, ProfileTotalCoverageStateV1, StoreKindBytesV1, + FullProfileSizeV1, ProfileTotalCoverageStateV1, StoreKindBytesV1, graph_container_stem, }; /// Entries a footprint walk visits before it stops and reports a floor. @@ -467,16 +467,6 @@ fn is_wal(name: &str) -> bool { name.ends_with("-wal") || name.ends_with(".wal") } -/// Only canonical graph containers and their recognized sidecars share a -/// container bound. Unrelated suffixes must remain separate profile entries. -fn graph_container_stem(name: &str) -> Option<&str> { - let stem = name - .strip_suffix(".verified") - .or_else(|| name.strip_suffix(".spill")) - .unwrap_or(name); - stem.ends_with(GRAPH_CONTAINER_SUFFIX).then_some(stem) -} - /// The footprint keys one profile-relative path belongs to. A WAL directory /// is one entry keyed by the directory; its segments sum into it. fn footprint_keys(relative: &Path) -> Option { @@ -497,18 +487,14 @@ fn footprint_keys(relative: &Path) -> Option { ProfileFamilyV1::Wal, ) }) - .or_else(|| match inner { - [name] => { - // Verified and spill sidecars consume the same container - // footprint as the primary .grafeo file. - graph_container_stem(name).map(|stem| { - ( - format!("{store_key}/{stem}"), - ProfileFamilyV1::GraphContainer, - ) - }) - } - _ => None, + .or_else(|| { + let (first, rest) = inner.split_first()?; + graph_container_stem(first, rest.is_empty()).map(|stem| { + ( + format!("{store_key}/{stem}"), + ProfileFamilyV1::GraphContainer, + ) + }) }); return Some(FootprintKeys { primary: (store_key, ProfileFamilyV1::ProjectStore), @@ -518,7 +504,7 @@ fn footprint_keys(relative: &Path) -> Option { } let primary = if let Some(end) = components.iter().position(|name| is_wal(name)) { (components[..=end].join("/"), ProfileFamilyV1::Wal) - } else if let Some(stem) = graph_container_stem(first) { + } else if let Some(stem) = graph_container_stem(first, components.len() == 1) { // A top-level container and only its recognized sidecars. (stem.to_owned(), ProfileFamilyV1::GraphContainer) } else if first == "global.db" || first.starts_with("global.db-") { @@ -1223,9 +1209,10 @@ mod tests { for (name, bytes) in [ ("user-sessions.grafeo", 5), ("user-sessions.grafeo.verified", 7), - ("user-sessions.grafeo.spill", 11), + ("user-sessions.grafeo.spill/scratch/inflight", 11), ("user-sessions.grafeo.notes", 30), ("other.grafeo.metrics", 40), + ("scratch.grafeo/unrelated", 13), ] { sized(&profile.join(name), bytes); } @@ -1233,7 +1220,7 @@ mod tests { let overrides = BTreeMap::from([(container_key.to_owned(), 24)]); let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); - assert_eq!(footprint.total_bytes, 93); + assert_eq!(footprint.total_bytes, 106); assert_eq!(entry(&footprint, container_key).bytes, 23); assert_eq!( entry(&footprint, container_key).family, @@ -1242,6 +1229,7 @@ mod tests { for (name, bytes) in [ ("user-sessions.grafeo.notes", 30), ("other.grafeo.metrics", 40), + ("scratch.grafeo", 13), ] { let unrelated = entry(&footprint, name); assert_eq!(unrelated.family, ProfileFamilyV1::Other); @@ -1280,7 +1268,7 @@ mod tests { ("graph.grafeo.verified", 7), ("graph.grafeo.spill", 11), ("graph.grafeo.wal/segment", 13), - ("other.grafeo.spill", 17), + ("other.grafeo.spill/scratch/inflight", 17), ("not-a-graph.grafeo.tmp", 19), ] { sized(&profile.join(project).join(name), bytes); diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index f83b5e295e0..92b463e89ca 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -65,7 +65,7 @@ const CODE_GENERATIONS_DIRECTORY: &str = "code-generations-v1"; const CODE_INDEX_DIRECTORY: &str = "code-index-v1"; const SEALED_GRAPH_DIRECTORY: &str = "tracedecay.sealed"; const GRAPH_REPLAY_POOL_DIRECTORY: &str = "tracedecay.graph-replay"; -pub(crate) const GRAPH_CONTAINER_SUFFIX: &str = ".grafeo"; +const GRAPH_CONTAINER_SUFFIX: &str = ".grafeo"; pub(crate) const GRAPH_CONTAINER_WAL_SUFFIX: &str = ".grafeo.wal"; /// Why a report without the daemon's scheduler authority cannot plan a /// retention dry run. @@ -192,11 +192,22 @@ pub struct StoreKindBytesV1 { pub other: u64, } +/// A container is a regular file at the store root. Its recognized sidecars +/// can also be directories, whose descendants belong to the same container. +pub(super) fn graph_container_stem(name: &str, is_leaf: bool) -> Option<&str> { + let stem = name + .strip_suffix(".verified") + .or_else(|| name.strip_suffix(".spill")) + .or_else(|| is_leaf.then_some(name))?; + stem.ends_with(GRAPH_CONTAINER_SUFFIX).then_some(stem) +} + impl StoreKindBytesV1 { pub(crate) fn add(&mut self, relative: &Path, bytes: u64) { let mut components = relative .components() - .map(|component| component.as_os_str().to_string_lossy()); + .map(|component| component.as_os_str().to_string_lossy()) + .peekable(); let first = components.next().unwrap_or_default(); let db_filename = tracedecay_runtime_core::config::DB_FILENAME; let family = if first == db_filename @@ -210,9 +221,7 @@ impl StoreKindBytesV1 { } else if first == GRAPH_REPLAY_POOL_DIRECTORY { &mut self.graph_replay_pool } else if first.ends_with(GRAPH_CONTAINER_WAL_SUFFIX) - || first.ends_with(".grafeo.verified") - || first.ends_with(".grafeo.spill") - || (first.ends_with(GRAPH_CONTAINER_SUFFIX) && components.next().is_none()) + || graph_container_stem(&first, components.peek().is_none()).is_some() { &mut self.graph_container } else if first == CODE_INDEX_DIRECTORY { From 4125cc9bdfefc18f27bcf6ed7ea49bdb013c02b6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:20:19 -0800 Subject: [PATCH 41/85] fix(storage): deduplicate unlinked SQLite TEMP descriptors Refs #4099. --- .../src/retention/footprint.rs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 8445b5eac72..e10203bbf0a 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -8,6 +8,10 @@ use std::borrow::Cow; use std::collections::BTreeMap; +#[cfg(unix)] +use std::collections::BTreeSet; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -410,6 +414,9 @@ pub fn census_sqlite_temp_spill( ) -> Option { let mut bytes = 0u64; let mut unavailable_entry_count = 0usize; + // Multiple descriptors may refer to one unlinked SQLite allocation. + #[cfg(unix)] + let mut observed_files = BTreeSet::new(); for entry in std::fs::read_dir(fd_directory).ok()? { let fd = match entry { Ok(fd) => fd, @@ -433,7 +440,13 @@ pub fn census_sqlite_temp_spill( .is_some_and(|name| name.starts_with("etilqs_") && name.ends_with(" (deleted)")); if unlinked_temp { match std::fs::metadata(fd.path()) { - Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + Ok(metadata) => { + #[cfg(unix)] + if !observed_files.insert((metadata.dev(), metadata.ino())) { + continue; + } + bytes = bytes.saturating_add(metadata.len()); + } // A closed descriptor no longer consumes space. Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => unavailable_entry_count = unavailable_entry_count.saturating_add(1), @@ -728,12 +741,38 @@ mod tests { std::os::unix::fs::symlink(&target, fd_directory.join(fd)).expect("fd link"); } + // Duplicated file descriptors must not double-count a physical inode. + std::os::unix::fs::symlink( + temporary.path().join("etilqs_5e598e639aa4c638 (deleted)"), + fd_directory.join("7"), + ) + .expect("duplicated temp descriptor"); + let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) .expect("a readable fd directory"); assert_eq!(entry.key, SQLITE_TEMP_SPILL_KEY); assert_eq!(entry.bytes, 1_924 * MIB); assert_eq!(entry.bound_bytes, GIB); + let healthy_budget = BTreeMap::from([(SQLITE_TEMP_SPILL_KEY.to_owned(), 2 * GIB)]); + let healthy = census_sqlite_temp_spill(&fd_directory, &healthy_budget) + .expect("same unlinked SQLite allocations"); + assert_eq!(healthy.bytes, 1_924 * MIB); + assert_eq!(healthy.bound_bytes, 2 * GIB); + let healthy_footprint = ProfileFootprintV1 { + entries: vec![healthy], + total_bytes: 0, + unavailable_entry_count: 0, + truncated: false, + total_bound_bytes: PROFILE_TOTAL_BOUND_BYTES, + }; + assert_eq!( + findings(&healthy_footprint)[SQLITE_TEMP_SPILL_KEY] + .finding() + .state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "duplicate handles must not cause a false storage-pressure finding" + ); let footprint = ProfileFootprintV1 { entries: vec![entry], total_bytes: 0, From acfe68e55fb66b29fc880b29c5c5bec5c381ebf1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:48:14 -0800 Subject: [PATCH 42/85] fix(storage): separate unrelated SQLite suffix footprint bytes Refs #4099 --- .../src/retention/footprint.rs | 75 ++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index e10203bbf0a..be363004053 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -520,9 +520,15 @@ fn footprint_keys(relative: &Path) -> Option { } else if let Some(stem) = graph_container_stem(first, components.len() == 1) { // A top-level container and only its recognized sidecars. (stem.to_owned(), ProfileFamilyV1::GraphContainer) - } else if first == "global.db" || first.starts_with("global.db-") { + } else if matches!( + first.as_str(), + "global.db" | "global.db-shm" | "global.db-journal" + ) { ("global.db".to_owned(), ProfileFamilyV1::ProfileRegistry) - } else if first.starts_with("user-sessions.db") { + } else if matches!( + first.as_str(), + "user-sessions.db" | "user-sessions.db-shm" | "user-sessions.db-journal" + ) { ("user-sessions.db".to_owned(), ProfileFamilyV1::UserSessions) } else if first.starts_with("user-memory.") { ("user-memory".to_owned(), ProfileFamilyV1::UserMemory) @@ -1357,6 +1363,71 @@ mod tests { ); } + #[test] + fn database_family_bounds_do_not_swallow_unrelated_suffix_files() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (name, bytes) in [ + ("global.db", 5), + ("global.db-shm", 7), + ("global.db-journal", 11), + ("global.db-wal", 13), + ("global.db-mystery", 17), + ("user-sessions.db", 19), + ("user-sessions.db-shm", 23), + ("user-sessions.db-journal", 29), + ("user-sessions.db-wal", 31), + ("user-sessions.db-backup", 37), + ] { + sized(&profile.join(name), bytes); + } + let overrides = BTreeMap::from([ + ("global.db".to_owned(), 23), + ("user-sessions.db".to_owned(), 71), + ("global.db-mystery".to_owned(), 16), + ("user-sessions.db-backup".to_owned(), 36), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 192); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + + for (key, family, bytes) in [ + ("global.db", ProfileFamilyV1::ProfileRegistry, 23), + ("user-sessions.db", ProfileFamilyV1::UserSessions, 71), + ("global.db-wal", ProfileFamilyV1::Wal, 13), + ("user-sessions.db-wal", ProfileFamilyV1::Wal, 31), + ("global.db-mystery", ProfileFamilyV1::Other, 17), + ("user-sessions.db-backup", ProfileFamilyV1::Other, 37), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + let observed = findings(&footprint); + for key in [ + "global.db", + "user-sessions.db", + "global.db-wal", + "user-sessions.db-wal", + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must not inherit unrelated suffix bytes" + ); + } + for key in ["global.db-mystery", "user-sessions.db-backup"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must retain its own named bound" + ); + } + } + #[test] fn an_unreadable_walk_root_is_accounted_as_unavailable() { let temporary = tempfile::tempdir().expect("walk fixture"); From b60cee7321112305e44af28f2b50bcc413f20cbf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:36:12 -0800 Subject: [PATCH 43/85] fix(storage): classify only owned project database sidecars Refs #4099 --- .../src/retention/storage_report.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index bab6ca56c6a..1c6bf856752 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -202,6 +202,15 @@ pub(super) fn graph_container_stem(name: &str, is_leaf: bool) -> Option<&str> { stem.ends_with(GRAPH_CONTAINER_SUFFIX).then_some(stem) } +// Only SQLite-owned sidecars belong to the database family. Incident debris, +// backups and unrelated adjacent files must remain visible as "other" bytes. +fn sqlite_family_member(name: &str, main: &str) -> bool { + name == main + || name + .strip_prefix(main) + .is_some_and(|suffix| matches!(suffix, "-wal" | "-shm" | "-journal")) +} + impl StoreKindBytesV1 { pub(crate) fn add(&mut self, relative: &Path, bytes: u64) { let mut components = relative @@ -210,11 +219,7 @@ impl StoreKindBytesV1 { .peekable(); let first = components.next().unwrap_or_default(); let db_filename = tracedecay_runtime_core::config::DB_FILENAME; - let family = if first == db_filename - || first - .strip_prefix(db_filename) - .is_some_and(|suffix| suffix.starts_with('-')) - { + let family = if sqlite_family_member(&first, db_filename) { &mut self.graph_database } else if first == SEALED_GRAPH_DIRECTORY { &mut self.sealed_graph @@ -234,10 +239,13 @@ impl StoreKindBytesV1 { } else { &mut self.generation_artifacts } - } else if first.starts_with(SESSIONS_DB_FILENAME) + } else if sqlite_family_member(&first, SESSIONS_DB_FILENAME) + || first.strip_prefix(SESSIONS_DB_FILENAME) + == Some(".delivery-settlement-spool-v1") || first .strip_prefix('.') - .is_some_and(|name| name.starts_with(SESSIONS_DB_FILENAME)) + .and_then(|name| name.strip_prefix(SESSIONS_DB_FILENAME)) + == Some(".host-admission") { &mut self.sessions } else { @@ -3580,6 +3588,11 @@ mod tests { ("sessions.db", 900), ("sessions.db-wal", 10), (".sessions.db.host-admission/claim", 5), + ("sessions.db.delivery-settlement-spool-v1/claim", 11), + ("tracedecay.db-archive", 17), + ("sessions.db-backup", 19), + ("sessions.db.corrupt-incident", 23), + (".sessions.db.host-admission-backup/claim", 13), ("hook-delivery-spool/claude/record", 7), ] { let path = data_root.join(relative); @@ -3617,11 +3630,11 @@ mod tests { generation_artifacts: 1_000, graph_container: 760, graph_replay_pool: 40, - sessions: 915, - other: 7, + sessions: 926, + other: 79, } ); - assert_eq!(store.total_bytes, 11_968); + assert_eq!(store.total_bytes, 12_051); assert_eq!(store.total_bytes, walked_bytes); assert_eq!(store.unavailable_entry_count, 0); } From 68e8021b3268d9c648d91b661b0b6b585ad10ffb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:20:17 -0800 Subject: [PATCH 44/85] fix(storage): isolate unrelated profile memory suffix files Refs #4099 --- .../src/retention/footprint.rs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index be363004053..c3a82b16af6 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -530,7 +530,10 @@ fn footprint_keys(relative: &Path) -> Option { "user-sessions.db" | "user-sessions.db-shm" | "user-sessions.db-journal" ) { ("user-sessions.db".to_owned(), ProfileFamilyV1::UserSessions) - } else if first.starts_with("user-memory.") { + } else if matches!( + first.as_str(), + "user-memory.db" | "user-memory.db-shm" | "user-memory.db-journal" + ) { ("user-memory".to_owned(), ProfileFamilyV1::UserMemory) } else { let family = match first.as_str() { @@ -1378,6 +1381,12 @@ mod tests { ("user-sessions.db-journal", 29), ("user-sessions.db-wal", 31), ("user-sessions.db-backup", 37), + ("user-memory.db", 41), + ("user-memory.db-shm", 43), + ("user-memory.db-journal", 47), + ("user-memory.db-wal", 53), + ("user-memory.db-backup", 59), + ("user-memory.grafeo.notes", 61), ] { sized(&profile.join(name), bytes); } @@ -1386,9 +1395,13 @@ mod tests { ("user-sessions.db".to_owned(), 71), ("global.db-mystery".to_owned(), 16), ("user-sessions.db-backup".to_owned(), 36), + ("user-memory".to_owned(), 132), + ("user-memory.db-wal".to_owned(), 54), + ("user-memory.db-backup".to_owned(), 58), + ("user-memory.grafeo.notes".to_owned(), 60), ]); let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); - assert_eq!(footprint.total_bytes, 192); + assert_eq!(footprint.total_bytes, 496); assert_eq!( footprint.full_profile_size().state, ProfileTotalCoverageStateV1::Complete @@ -1401,17 +1414,24 @@ mod tests { ("user-sessions.db-wal", ProfileFamilyV1::Wal, 31), ("global.db-mystery", ProfileFamilyV1::Other, 17), ("user-sessions.db-backup", ProfileFamilyV1::Other, 37), + ("user-memory", ProfileFamilyV1::UserMemory, 131), + ("user-memory.db-wal", ProfileFamilyV1::Wal, 53), + ("user-memory.db-backup", ProfileFamilyV1::Other, 59), + ("user-memory.grafeo.notes", ProfileFamilyV1::Other, 61), ] { let measured = entry(&footprint, key); assert_eq!(measured.family, family, "{key}"); assert_eq!(measured.bytes, bytes, "{key}"); } let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); for key in [ "global.db", "user-sessions.db", "global.db-wal", "user-sessions.db-wal", + "user-memory", + "user-memory.db-wal", ] { assert_eq!( observed[key].finding().state(), @@ -1419,7 +1439,12 @@ mod tests { "{key} must not inherit unrelated suffix bytes" ); } - for key in ["global.db-mystery", "user-sessions.db-backup"] { + for key in [ + "global.db-mystery", + "user-sessions.db-backup", + "user-memory.db-backup", + "user-memory.grafeo.notes", + ] { assert_eq!( observed[key].finding().state(), DoctorEvidenceStateV1::Degraded, From 7756fa749bc45285493eb3ef8368c799a9db082a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:36:09 -0800 Subject: [PATCH 45/85] fix(storage): exclude SQLite sidecar lookalike directories Refs #4099 Only regular SQLite family members may take a database size bound. Incident directories named as SQLite sidecars keep independent Doctor findings and project-report attribution, without changing profile totals. --- .../src/retention/footprint.rs | 89 ++++++++++++++++--- .../src/retention/storage_report.rs | 11 ++- 2 files changed, 84 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index c3a82b16af6..acb01ea983b 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -520,20 +520,26 @@ fn footprint_keys(relative: &Path) -> Option { } else if let Some(stem) = graph_container_stem(first, components.len() == 1) { // A top-level container and only its recognized sidecars. (stem.to_owned(), ProfileFamilyV1::GraphContainer) - } else if matches!( - first.as_str(), - "global.db" | "global.db-shm" | "global.db-journal" - ) { + } else if components.len() == 1 + && matches!( + first.as_str(), + "global.db" | "global.db-shm" | "global.db-journal" + ) + { ("global.db".to_owned(), ProfileFamilyV1::ProfileRegistry) - } else if matches!( - first.as_str(), - "user-sessions.db" | "user-sessions.db-shm" | "user-sessions.db-journal" - ) { + } else if components.len() == 1 + && matches!( + first.as_str(), + "user-sessions.db" | "user-sessions.db-shm" | "user-sessions.db-journal" + ) + { ("user-sessions.db".to_owned(), ProfileFamilyV1::UserSessions) - } else if matches!( - first.as_str(), - "user-memory.db" | "user-memory.db-shm" | "user-memory.db-journal" - ) { + } else if components.len() == 1 + && matches!( + first.as_str(), + "user-memory.db" | "user-memory.db-shm" | "user-memory.db-journal" + ) + { ("user-memory".to_owned(), ProfileFamilyV1::UserMemory) } else { let family = match first.as_str() { @@ -1453,6 +1459,65 @@ mod tests { } } + #[test] + fn sqlite_sidecar_named_directories_have_independent_doctor_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + // A directory bearing a SQLite sidecar name is not a SQLite file. + for (relative, bytes) in [ + ("global.db", 5), + ("user-sessions.db", 7), + ("user-memory.db", 9), + ("global.db-journal/incident.bin", 11), + ("user-memory.db-shm/incident.bin", 13), + ("user-sessions.db-shm/incident.bin", 17), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("global.db".to_owned(), 5), + ("user-sessions.db".to_owned(), 7), + ("user-memory".to_owned(), 9), + ("global.db-journal".to_owned(), 10), + ("user-memory.db-shm".to_owned(), 12), + ("user-sessions.db-shm".to_owned(), 16), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 62); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + for (key, family, bytes) in [ + ("global.db", ProfileFamilyV1::ProfileRegistry, 5), + ("user-sessions.db", ProfileFamilyV1::UserSessions, 7), + ("user-memory", ProfileFamilyV1::UserMemory, 9), + ("global.db-journal", ProfileFamilyV1::Other, 11), + ("user-memory.db-shm", ProfileFamilyV1::Other, 13), + ("user-sessions.db-shm", ProfileFamilyV1::Other, 17), + ] { + assert_eq!(entry(&footprint, key).family, family, "{key}"); + assert_eq!(entry(&footprint, key).bytes, bytes, "{key}"); + } + for key in ["global.db", "user-sessions.db", "user-memory", "profile"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key}" + ); + } + for key in ["global.db-journal", "user-memory.db-shm", "user-sessions.db-shm"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must have an independent named over-budget finding" + ); + } + } + #[test] fn an_unreadable_walk_root_is_accounted_as_unavailable() { let temporary = tempfile::tempdir().expect("walk fixture"); diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 1c6bf856752..6a5ca2e1196 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -218,8 +218,9 @@ impl StoreKindBytesV1 { .map(|component| component.as_os_str().to_string_lossy()) .peekable(); let first = components.next().unwrap_or_default(); + let root_file = components.peek().is_none(); let db_filename = tracedecay_runtime_core::config::DB_FILENAME; - let family = if sqlite_family_member(&first, db_filename) { + let family = if root_file && sqlite_family_member(&first, db_filename) { &mut self.graph_database } else if first == SEALED_GRAPH_DIRECTORY { &mut self.sealed_graph @@ -239,7 +240,7 @@ impl StoreKindBytesV1 { } else { &mut self.generation_artifacts } - } else if sqlite_family_member(&first, SESSIONS_DB_FILENAME) + } else if (root_file && sqlite_family_member(&first, SESSIONS_DB_FILENAME)) || first.strip_prefix(SESSIONS_DB_FILENAME) == Some(".delivery-settlement-spool-v1") || first @@ -3590,6 +3591,8 @@ mod tests { (".sessions.db.host-admission/claim", 5), ("sessions.db.delivery-settlement-spool-v1/claim", 11), ("tracedecay.db-archive", 17), + ("tracedecay.db-journal/incident.bin", 37), + ("sessions.db-shm/incident.bin", 41), ("sessions.db-backup", 19), ("sessions.db.corrupt-incident", 23), (".sessions.db.host-admission-backup/claim", 13), @@ -3631,10 +3634,10 @@ mod tests { graph_container: 760, graph_replay_pool: 40, sessions: 926, - other: 79, + other: 157, } ); - assert_eq!(store.total_bytes, 12_051); + assert_eq!(store.total_bytes, 12_129); assert_eq!(store.total_bytes, walked_bytes); assert_eq!(store.unavailable_entry_count, 0); } From 82384ba94f1841fe86c67b1038cf65dfc1e30e5a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:54:15 -0800 Subject: [PATCH 46/85] fix(storage): keep WAL-named payloads in their owner bounds Refs #4099 The profile census previously attributed nested LCM/handle/maintenance payloads named *.wal to standalone WAL buckets. That could suppress the owning family's aggregate over-budget finding. Preserve managed subtree ownership and cover exact accounting and named degraded Doctor states. --- .../src/retention/footprint.rs | 65 ++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index acb01ea983b..f018a87872d 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -515,7 +515,15 @@ fn footprint_keys(relative: &Path) -> Option { project: Some((components[1].clone(), inner.iter().collect())), }); } - let primary = if let Some(end) = components.iter().position(|name| is_wal(name)) { + // Managed profile families own their entire directory trees. An LCM payload, + // response handle, or maintenance cursor named *.wal is not a database + // journal; classifying it as one hides growth from its owning bound. + let owned_subtree = matches!( + first.as_str(), + "lcm-payloads" | "response-handles" | "maintenance" + ); + let wal_component = components.iter().position(|name| is_wal(name)); + let primary = if let Some(end) = wal_component.filter(|_| !owned_subtree) { (components[..=end].join("/"), ProfileFamilyV1::Wal) } else if let Some(stem) = graph_container_stem(first, components.len() == 1) { // A top-level container and only its recognized sidecars. @@ -1459,6 +1467,61 @@ mod tests { } } + #[test] + fn managed_profile_families_bound_wal_named_payloads() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (name, bytes) in [ + ("lcm-payloads/payload-wal/part", 25), + ("lcm-payloads/archive.wal", 34), + ("lcm-payloads/plain", 3), + ("response-handles/handle-wal/part", 11), + ("maintenance/cursor.wal", 13), + ("global.db-wal", 7), + ] { + sized(&profile.join(name), bytes); + } + let overrides = BTreeMap::from([ + ("lcm-payloads".to_owned(), 61), + ("response-handles".to_owned(), 10), + ("maintenance".to_owned(), 12), + ("global.db-wal".to_owned(), 7), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 93); + assert_eq!(footprint.entries.len(), 4, "owner aggregates must not be split"); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + for (key, family, bytes) in [ + ("lcm-payloads", ProfileFamilyV1::LcmPayloads, 62), + ("response-handles", ProfileFamilyV1::ResponseHandles, 11), + ("maintenance", ProfileFamilyV1::Maintenance, 13), + ("global.db-wal", ProfileFamilyV1::Wal, 7), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + for key in ["lcm-payloads", "response-handles", "maintenance"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must retain a named over-budget finding" + ); + } + for key in ["global.db-wal", "profile"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key}" + ); + } + } + #[test] fn sqlite_sidecar_named_directories_have_independent_doctor_bounds() { let temp = tempfile::tempdir().expect("footprint fixture"); From 70f5596b4f3bfa93eb63e20c05307951029cde3e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:23:29 -0800 Subject: [PATCH 47/85] fix(storage): charge WAL-named spill files to graph containers Refs #4099 Preserve the owning Grafeo container's Doctor bound when scratch names inside .grafeo.spill end with -wal or .wal. Keep the real container WAL on its independent bound, and cover both project and profile stores with a filesystem-backed behavioral regression. --- .../src/retention/footprint.rs | 101 +++++++++++++++--- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index f018a87872d..b0723e4fad3 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -491,23 +491,26 @@ fn footprint_keys(relative: &Path) -> Option { if first == PROJECTS_DIRECTORY && components.len() >= 3 { let store_key = format!("{PROJECTS_DIRECTORY}/{}", components[1]); let inner = &components[2..]; + // A container's verified/spill tree owns its descendants, even when + // a scratch file is named *.wal. Only the actual container WAL + // directory (or a WAL outside a container) gets a WAL bound. let overlay = inner - .iter() - .position(|name| is_wal(name)) - .map(|end| { - ( - format!("{store_key}/{}", inner[..=end].join("/")), - ProfileFamilyV1::Wal, - ) - }) - .or_else(|| { - let (first, rest) = inner.split_first()?; + .split_first() + .and_then(|(first, rest)| { graph_container_stem(first, rest.is_empty()).map(|stem| { ( format!("{store_key}/{stem}"), ProfileFamilyV1::GraphContainer, ) }) + }) + .or_else(|| { + inner.iter().position(|name| is_wal(name)).map(|end| { + ( + format!("{store_key}/{}", inner[..=end].join("/")), + ProfileFamilyV1::Wal, + ) + }) }); return Some(FootprintKeys { primary: (store_key, ProfileFamilyV1::ProjectStore), @@ -523,11 +526,11 @@ fn footprint_keys(relative: &Path) -> Option { "lcm-payloads" | "response-handles" | "maintenance" ); let wal_component = components.iter().position(|name| is_wal(name)); - let primary = if let Some(end) = wal_component.filter(|_| !owned_subtree) { - (components[..=end].join("/"), ProfileFamilyV1::Wal) - } else if let Some(stem) = graph_container_stem(first, components.len() == 1) { - // A top-level container and only its recognized sidecars. + let primary = if let Some(stem) = graph_container_stem(first, components.len() == 1) { + // Sidecar trees take precedence over WAL-looking scratch filenames. (stem.to_owned(), ProfileFamilyV1::GraphContainer) + } else if let Some(end) = wal_component.filter(|_| !owned_subtree) { + (components[..=end].join("/"), ProfileFamilyV1::Wal) } else if components.len() == 1 && matches!( first.as_str(), @@ -1380,6 +1383,76 @@ mod tests { ); } + #[test] + fn nested_wal_named_graph_spill_files_remain_under_container_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("user-sessions.grafeo", 5), + ("user-sessions.grafeo.spill/scratch/chunk.wal", 17), + ("user-sessions.grafeo.spill/scratch/segment-wal/part", 19), + ("user-sessions.grafeo.wal/segment", 7), + ("projects/proj_spill/graph.grafeo", 3), + ("projects/proj_spill/graph.grafeo.spill/scratch/chunk.wal", 23), + ("projects/proj_spill/graph.grafeo.spill/segment-wal/part", 29), + ("projects/proj_spill/graph.grafeo.wal/segment", 11), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("user-sessions.grafeo".to_owned(), 40), + ("projects/proj_spill/graph.grafeo".to_owned(), 54), + ("user-sessions.grafeo.wal".to_owned(), 7), + ("projects/proj_spill/graph.grafeo.wal".to_owned(), 11), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 114); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 5); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + + for (key, family, bytes) in [ + ("user-sessions.grafeo", ProfileFamilyV1::GraphContainer, 41), + ( + "projects/proj_spill/graph.grafeo", + ProfileFamilyV1::GraphContainer, + 55, + ), + ("user-sessions.grafeo.wal", ProfileFamilyV1::Wal, 7), + ("projects/proj_spill/graph.grafeo.wal", ProfileFamilyV1::Wal, 11), + ("projects/proj_spill", ProfileFamilyV1::ProjectStore, 66), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + + let observed = findings(&footprint); + assert_eq!(observed.len(), 6); + for key in ["user-sessions.grafeo", "projects/proj_spill/graph.grafeo"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must detect spill bytes despite WAL-looking filenames" + ); + } + for key in [ + "user-sessions.grafeo.wal", + "projects/proj_spill/graph.grafeo.wal", + "projects/proj_spill", + PROFILE_TOTAL_KEY, + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must retain its actual bound" + ); + } + } + #[test] fn database_family_bounds_do_not_swallow_unrelated_suffix_files() { let temp = tempfile::tempdir().expect("footprint fixture"); From ff8011df323207271de694576ea3e48988061f54 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:35:51 -0800 Subject: [PATCH 48/85] fix(storage): keep WAL-named host files under owner bounds Refs #4099 Charge nested WAL-looking host bundle and component filenames to their top-level directory bound, not an unrelated WAL key. Check exact physical bytes and named Doctor overages in a real filesystem regression, keeping genuine top-level WALs independent. Apply the pinned rustfmt corrections reported by exact-head CI. --- .../src/retention/footprint.rs | 95 ++++++++++++++++--- .../src/retention/storage_report.rs | 3 +- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index b0723e4fad3..2194f50bb1c 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -518,19 +518,13 @@ fn footprint_keys(relative: &Path) -> Option { project: Some((components[1].clone(), inner.iter().collect())), }); } - // Managed profile families own their entire directory trees. An LCM payload, - // response handle, or maintenance cursor named *.wal is not a database - // journal; classifying it as one hides growth from its owning bound. - let owned_subtree = matches!( - first.as_str(), - "lcm-payloads" | "response-handles" | "maintenance" - ); - let wal_component = components.iter().position(|name| is_wal(name)); + // A top-level family owns all descendants. WAL-looking nested scratch + // filenames in an otherwise unrelated tree must not escape that bound. let primary = if let Some(stem) = graph_container_stem(first, components.len() == 1) { // Sidecar trees take precedence over WAL-looking scratch filenames. (stem.to_owned(), ProfileFamilyV1::GraphContainer) - } else if let Some(end) = wal_component.filter(|_| !owned_subtree) { - (components[..=end].join("/"), ProfileFamilyV1::Wal) + } else if is_wal(first) { + (first.clone(), ProfileFamilyV1::Wal) } else if components.len() == 1 && matches!( first.as_str(), @@ -1393,8 +1387,14 @@ mod tests { ("user-sessions.grafeo.spill/scratch/segment-wal/part", 19), ("user-sessions.grafeo.wal/segment", 7), ("projects/proj_spill/graph.grafeo", 3), - ("projects/proj_spill/graph.grafeo.spill/scratch/chunk.wal", 23), - ("projects/proj_spill/graph.grafeo.spill/segment-wal/part", 29), + ( + "projects/proj_spill/graph.grafeo.spill/scratch/chunk.wal", + 23, + ), + ( + "projects/proj_spill/graph.grafeo.spill/segment-wal/part", + 29, + ), ("projects/proj_spill/graph.grafeo.wal/segment", 11), ] { sized(&profile.join(relative), bytes); @@ -1422,7 +1422,11 @@ mod tests { 55, ), ("user-sessions.grafeo.wal", ProfileFamilyV1::Wal, 7), - ("projects/proj_spill/graph.grafeo.wal", ProfileFamilyV1::Wal, 11), + ( + "projects/proj_spill/graph.grafeo.wal", + ProfileFamilyV1::Wal, + 11, + ), ("projects/proj_spill", ProfileFamilyV1::ProjectStore, 66), ] { let measured = entry(&footprint, key); @@ -1562,7 +1566,11 @@ mod tests { ]); let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); assert_eq!(footprint.total_bytes, 93); - assert_eq!(footprint.entries.len(), 4, "owner aggregates must not be split"); + assert_eq!( + footprint.entries.len(), + 4, + "owner aggregates must not be split" + ); assert_eq!( footprint.full_profile_size().state, ProfileTotalCoverageStateV1::Complete @@ -1595,6 +1603,59 @@ mod tests { } } + #[test] + fn unrelated_profile_trees_keep_wal_named_descendants_under_their_own_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("host-bundle-stage/artifact-wal/chunk", 40), + ("host-bundle-stage/cache/trace.wal", 30), + ("host-components/bundle.wal", 17), + ("host-components/plain", 3), + ("global.db-wal", 7), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("host-bundle-stage".to_owned(), 69), + ("host-components".to_owned(), 19), + ("global.db-wal".to_owned(), 7), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 97); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 3); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + for (key, family, bytes) in [ + ("host-bundle-stage", ProfileFamilyV1::Other, 70), + ("host-components", ProfileFamilyV1::Other, 20), + ("global.db-wal", ProfileFamilyV1::Wal, 7), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + let observed = findings(&footprint); + assert_eq!(observed.len(), 4); + for key in ["host-bundle-stage", "host-components"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must retain its named over-budget finding" + ); + } + for key in ["global.db-wal", PROFILE_TOTAL_KEY] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must keep its actual bound" + ); + } + } + #[test] fn sqlite_sidecar_named_directories_have_independent_doctor_bounds() { let temp = tempfile::tempdir().expect("footprint fixture"); @@ -1645,7 +1706,11 @@ mod tests { "{key}" ); } - for key in ["global.db-journal", "user-memory.db-shm", "user-sessions.db-shm"] { + for key in [ + "global.db-journal", + "user-memory.db-shm", + "user-sessions.db-shm", + ] { assert_eq!( observed[key].finding().state(), DoctorEvidenceStateV1::Degraded, diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 6a5ca2e1196..89645e31264 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -241,8 +241,7 @@ impl StoreKindBytesV1 { &mut self.generation_artifacts } } else if (root_file && sqlite_family_member(&first, SESSIONS_DB_FILENAME)) - || first.strip_prefix(SESSIONS_DB_FILENAME) - == Some(".delivery-settlement-spool-v1") + || first.strip_prefix(SESSIONS_DB_FILENAME) == Some(".delivery-settlement-spool-v1") || first .strip_prefix('.') .and_then(|name| name.strip_prefix(SESSIONS_DB_FILENAME)) From a7fc901415c4062277e9ffef38a20a0c07ac3ca6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:33:42 -0800 Subject: [PATCH 49/85] fix(storage): resolve SQLite family helper collision Rename the predicate used by project kind accounting so it cannot shadow the SQLite path constructor used by offline registry snapshots. Restore Bazel Rust compilation without changing family classification, snapshot handling, or the existing filesystem-backed regressions. Refs #4099 --- .../tracedecay-maintenance/src/retention/storage_report.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 89645e31264..222f819eb47 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -204,7 +204,7 @@ pub(super) fn graph_container_stem(name: &str, is_leaf: bool) -> Option<&str> { // Only SQLite-owned sidecars belong to the database family. Incident debris, // backups and unrelated adjacent files must remain visible as "other" bytes. -fn sqlite_family_member(name: &str, main: &str) -> bool { +fn is_sqlite_family_member(name: &str, main: &str) -> bool { name == main || name .strip_prefix(main) @@ -220,7 +220,7 @@ impl StoreKindBytesV1 { let first = components.next().unwrap_or_default(); let root_file = components.peek().is_none(); let db_filename = tracedecay_runtime_core::config::DB_FILENAME; - let family = if root_file && sqlite_family_member(&first, db_filename) { + let family = if root_file && is_sqlite_family_member(&first, db_filename) { &mut self.graph_database } else if first == SEALED_GRAPH_DIRECTORY { &mut self.sealed_graph @@ -240,7 +240,7 @@ impl StoreKindBytesV1 { } else { &mut self.generation_artifacts } - } else if (root_file && sqlite_family_member(&first, SESSIONS_DB_FILENAME)) + } else if (root_file && is_sqlite_family_member(&first, SESSIONS_DB_FILENAME)) || first.strip_prefix(SESSIONS_DB_FILENAME) == Some(".delivery-settlement-spool-v1") || first .strip_prefix('.') From 9c8ed6c2f849de63086b88403d447079f9a2ab38 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:35:04 -0800 Subject: [PATCH 50/85] fix(storage): preserve project-owned nested WAL-looking payloads The Doctor project overlay must only promote WALs rooted in the project store. Scratch files inside indexed artifacts, LCM payloads and response handles remain charged to the project, not unrelated WAL findings. Add a real profile filesystem census regression for exact bytes, project-kind attribution, and over-budget vs healthy Doctor evidence. Refs #4099 --- .../src/retention/footprint.rs | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 2194f50bb1c..97e46a9f376 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -505,11 +505,8 @@ fn footprint_keys(relative: &Path) -> Option { }) }) .or_else(|| { - inner.iter().position(|name| is_wal(name)).map(|end| { - ( - format!("{store_key}/{}", inner[..=end].join("/")), - ProfileFamilyV1::Wal, - ) + inner.first().filter(|name| is_wal(name)).map(|first| { + (format!("{store_key}/{first}"), ProfileFamilyV1::Wal) }) }); return Some(FootprintKeys { @@ -1603,6 +1600,57 @@ mod tests { } } + #[test] + fn project_payloads_with_wal_names_retain_only_real_root_wal_overlays() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("projects/proj_payload/code-index-v1/objects/archive.wal", 40), + ("projects/proj_payload/lcm-payloads/payload-wal/chunk", 30), + ("projects/proj_payload/response-handles/result.wal", 17), + ("projects/proj_payload/plain", 3), + ("projects/proj_payload/sessions.db-wal", 7), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("projects/proj_payload".to_owned(), 96), + ("projects/proj_payload/sessions.db-wal".to_owned(), 7), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 97); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 2, "nested payloads cannot be false WALs"); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + let project = entry(&footprint, "projects/proj_payload"); + assert_eq!(project.family, ProfileFamilyV1::ProjectStore); + assert_eq!(project.bytes, 97); + let kinds = project.kinds.expect("project breakdown"); + assert_eq!(kinds.generation_artifacts, 40); + assert_eq!(kinds.sessions, 7); + assert_eq!(kinds.other, 50); + assert_eq!(kinds.total(), 97); + let wal = entry(&footprint, "projects/proj_payload/sessions.db-wal"); + assert_eq!(wal.family, ProfileFamilyV1::Wal); + assert_eq!(wal.bytes, 7); + let observed = findings(&footprint); + assert_eq!(observed.len(), 3); + assert_eq!( + observed["projects/proj_payload"].finding().state(), + DoctorEvidenceStateV1::Degraded + ); + for key in ["projects/proj_payload/sessions.db-wal", PROFILE_TOTAL_KEY] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key}" + ); + } + } + #[test] fn unrelated_profile_trees_keep_wal_named_descendants_under_their_own_bounds() { let temp = tempfile::tempdir().expect("footprint fixture"); From b7bfa9d84e392067ac7c0debc57606c98e5cf91c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:30:47 -0800 Subject: [PATCH 51/85] fix(storage): distinguish SQLite WALs from lookalike directories Only regular root-level WAL files and actual Grafeo WAL directories receive WAL bounds. Keep similarly named incident trees separately bounded and preserve genuine graph WAL pressure findings. Add real filesystem-backed Doctor and storage-kind regressions. Refs #4099 --- .../src/retention/footprint.rs | 93 ++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 97e46a9f376..dac56f1a220 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -480,6 +480,13 @@ fn is_wal(name: &str) -> bool { name.ends_with("-wal") || name.ends_with(".wal") } +// SQLite WALs are regular root-level files. Only Grafeo WALs are +// directories, so a similarly named incident tree must not inherit a WAL +// budget or hide its own independently bounded footprint. +fn is_wal_family_root(name: &str, is_file: bool) -> bool { + is_wal(name) && (is_file || name.ends_with(".grafeo.wal")) +} + /// The footprint keys one profile-relative path belongs to. A WAL directory /// is one entry keyed by the directory; its segments sum into it. fn footprint_keys(relative: &Path) -> Option { @@ -505,7 +512,10 @@ fn footprint_keys(relative: &Path) -> Option { }) }) .or_else(|| { - inner.first().filter(|name| is_wal(name)).map(|first| { + inner + .first() + .filter(|name| is_wal_family_root(name, inner.len() == 1)) + .map(|first| { (format!("{store_key}/{first}"), ProfileFamilyV1::Wal) }) }); @@ -520,7 +530,7 @@ fn footprint_keys(relative: &Path) -> Option { let primary = if let Some(stem) = graph_container_stem(first, components.len() == 1) { // Sidecar trees take precedence over WAL-looking scratch filenames. (stem.to_owned(), ProfileFamilyV1::GraphContainer) - } else if is_wal(first) { + } else if is_wal_family_root(first, components.len() == 1) { (first.clone(), ProfileFamilyV1::Wal) } else if components.len() == 1 && matches!( @@ -1790,4 +1800,83 @@ mod tests { assert_eq!(unavailable, 1); assert_eq!(remaining, 10); } + #[test] + fn sqlite_wal_lookalike_directories_keep_independent_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("global.db", 5), + ("global.db-wal/incident.bin", 11), + ("scratch.wal/chunk", 13), + ("user-sessions.db-wal", 7), + ("projects/proj_wal/sessions.db-wal", 17), + ("projects/proj_wal/tracedecay.db-wal/incident.bin", 19), + ("projects/proj_wal/tracedecay.grafeo.wal/segment", 23), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("global.db-wal".to_owned(), 10), + ("scratch.wal".to_owned(), 12), + ("projects/proj_wal".to_owned(), 58), + ("user-sessions.db-wal".to_owned(), 7), + ("projects/proj_wal/sessions.db-wal".to_owned(), 17), + ("projects/proj_wal/tracedecay.grafeo.wal".to_owned(), 23), + (PROFILE_TOTAL_KEY.to_owned(), 95), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 95); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 7); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + for (key, family, bytes) in [ + ("global.db", ProfileFamilyV1::ProfileRegistry, 5), + ("global.db-wal", ProfileFamilyV1::Other, 11), + ("scratch.wal", ProfileFamilyV1::Other, 13), + ("user-sessions.db-wal", ProfileFamilyV1::Wal, 7), + ("projects/proj_wal", ProfileFamilyV1::ProjectStore, 59), + ("projects/proj_wal/sessions.db-wal", ProfileFamilyV1::Wal, 17), + ( + "projects/proj_wal/tracedecay.grafeo.wal", + ProfileFamilyV1::Wal, + 23, + ), + ] { + assert_eq!(entry(&footprint, key).family, family, "{key}"); + assert_eq!(entry(&footprint, key).bytes, bytes, "{key}"); + } + let kinds = entry(&footprint, "projects/proj_wal") + .kinds + .expect("production project-kind breakdown"); + assert_eq!(kinds.sessions, 17); + assert_eq!(kinds.graph_container, 23); + assert_eq!(kinds.other, 19); + assert_eq!(kinds.total(), 59); + + let observed = findings(&footprint); + assert_eq!(observed.len(), 8); + for key in ["global.db-wal", "scratch.wal", "projects/proj_wal"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must not evade its physical footprint bound" + ); + } + for key in [ + "global.db", + "user-sessions.db-wal", + "projects/proj_wal/sessions.db-wal", + "projects/proj_wal/tracedecay.grafeo.wal", + PROFILE_TOTAL_KEY, + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must retain its real, independent bound" + ); + } + } } From 092ba0e233181cadee7fe26b8221868a710ca8df Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:31:41 -0800 Subject: [PATCH 52/85] style(storage): align WAL classifier and regression formatting Refs #4099 --- crates/tracedecay-maintenance/src/retention/footprint.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index dac56f1a220..d06d6525ce9 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -516,8 +516,8 @@ fn footprint_keys(relative: &Path) -> Option { .first() .filter(|name| is_wal_family_root(name, inner.len() == 1)) .map(|first| { - (format!("{store_key}/{first}"), ProfileFamilyV1::Wal) - }) + (format!("{store_key}/{first}"), ProfileFamilyV1::Wal) + }) }); return Some(FootprintKeys { primary: (store_key, ProfileFamilyV1::ProjectStore), @@ -1800,6 +1800,7 @@ mod tests { assert_eq!(unavailable, 1); assert_eq!(remaining, 10); } + #[test] fn sqlite_wal_lookalike_directories_keep_independent_bounds() { let temp = tempfile::tempdir().expect("footprint fixture"); From cdf122861d580c49cf7573513ffe08b771abf0b0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:23:06 -0800 Subject: [PATCH 53/85] style(storage): satisfy pinned footprint rustfmt check Refs #4099 --- .../src/retention/footprint.rs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index d06d6525ce9..b4a35143fa3 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -515,9 +515,7 @@ fn footprint_keys(relative: &Path) -> Option { inner .first() .filter(|name| is_wal_family_root(name, inner.len() == 1)) - .map(|first| { - (format!("{store_key}/{first}"), ProfileFamilyV1::Wal) - }) + .map(|first| (format!("{store_key}/{first}"), ProfileFamilyV1::Wal)) }); return Some(FootprintKeys { primary: (store_key, ProfileFamilyV1::ProjectStore), @@ -1615,7 +1613,10 @@ mod tests { let temp = tempfile::tempdir().expect("footprint fixture"); let profile = temp.path().join("profile"); for (relative, bytes) in [ - ("projects/proj_payload/code-index-v1/objects/archive.wal", 40), + ( + "projects/proj_payload/code-index-v1/objects/archive.wal", + 40, + ), ("projects/proj_payload/lcm-payloads/payload-wal/chunk", 30), ("projects/proj_payload/response-handles/result.wal", 17), ("projects/proj_payload/plain", 3), @@ -1630,7 +1631,11 @@ mod tests { let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); assert_eq!(footprint.total_bytes, 97); assert_eq!(footprint.unavailable_entry_count, 0); - assert_eq!(footprint.entries.len(), 2, "nested payloads cannot be false WALs"); + assert_eq!( + footprint.entries.len(), + 2, + "nested payloads cannot be false WALs" + ); assert_eq!( footprint.full_profile_size().state, ProfileTotalCoverageStateV1::Complete @@ -1839,7 +1844,11 @@ mod tests { ("scratch.wal", ProfileFamilyV1::Other, 13), ("user-sessions.db-wal", ProfileFamilyV1::Wal, 7), ("projects/proj_wal", ProfileFamilyV1::ProjectStore, 59), - ("projects/proj_wal/sessions.db-wal", ProfileFamilyV1::Wal, 17), + ( + "projects/proj_wal/sessions.db-wal", + ProfileFamilyV1::Wal, + 17, + ), ( "projects/proj_wal/tracedecay.grafeo.wal", ProfileFamilyV1::Wal, From a1513ef01d9efa314367a49a49f2b9aedfa8955f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:54:09 -0800 Subject: [PATCH 54/85] fix(storage): retain project root partial findings Refs #4099 --- .../src/retention/footprint.rs | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index b4a35143fa3..5054fb98be2 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -495,7 +495,11 @@ fn footprint_keys(relative: &Path) -> Option { .map(|component| component.as_os_str().to_string_lossy().into_owned()) .collect::>(); let first = components.first()?; - if first == PROJECTS_DIRECTORY && components.len() >= 3 { + // A non-traversable project root (symlink, unreadable directory, or + // unexpected file) still belongs to that project's named finding. + // Otherwise its unavailable bytes appear under a generic `projects` key + // while the affected store disappears from Doctor altogether. + if first == PROJECTS_DIRECTORY && components.len() >= 2 { let store_key = format!("{PROJECTS_DIRECTORY}/{}", components[1]); let inner = &components[2..]; // A container's verified/spill tree owns its descendants, even when @@ -1145,6 +1149,66 @@ mod tests { assert_eq!(remaining, 100 - 1 - 64 - 1 - 1); } + + #[cfg(unix)] + #[test] + fn unreadable_project_root_has_its_own_partial_finding() { + let temporary = tempfile::tempdir().expect("footprint fixture"); + let profile = temporary.path().join("profile"); + let foreign = temporary.path().join("foreign"); + sized(&foreign.join("private"), 2 * GIB); + sized(&profile.join("projects/proj_live/tracedecay.db"), 11); + std::os::unix::fs::symlink(&foreign, profile.join("projects/proj_blocked")) + .expect("substituted project directory"); + + let registered_root = temporary.path().join("registered-checkout"); + std::fs::create_dir(®istered_root).expect("registered root"); + let registered = BTreeMap::from([( + "proj_blocked".to_owned(), + registered_root.display().to_string(), + )]); + let overrides = BTreeMap::from([ + ("projects/proj_blocked".to_owned(), 1), + ("projects/proj_live".to_owned(), 11), + (PROFILE_TOTAL_KEY.to_owned(), 11), + ]); + + let footprint = census_profile_footprint(&profile, ®istered, &overrides); + assert_eq!(footprint.total_bytes, 11, "foreign bytes must not be counted"); + assert_eq!(footprint.unavailable_entry_count, 1); + assert_eq!(footprint.entries.len(), 2, "no generic projects entry"); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Partial + ); + let blocked = entry(&footprint, "projects/proj_blocked"); + assert_eq!(blocked.family, ProfileFamilyV1::ProjectStore); + assert_eq!(blocked.bytes, 0); + assert_eq!(blocked.unavailable_entry_count, 1); + assert_eq!(blocked.bound_bytes, 1); + assert_eq!( + blocked.project.as_ref().expect("registered project").state, + ProjectRootStateV1::Present + ); + assert_eq!(entry(&footprint, "projects/proj_live").bytes, 11); + + let observed = findings(&footprint); + assert_eq!(observed.len(), 3); + assert_eq!( + observed["projects/proj_blocked"].finding().state(), + DoctorEvidenceStateV1::Partial, + "an untraversable project store cannot be reported healthy or omitted" + ); + assert_eq!( + observed["projects/proj_live"].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage + ); + assert_eq!( + observed[PROFILE_TOTAL_KEY].finding().state(), + DoctorEvidenceStateV1::Partial + ); + } + #[test] fn one_entry_budget_is_shared_across_walk_roots() { let temporary = tempfile::tempdir().expect("walk fixture"); From 22bfc698908e18700372f436931564dfb4165c06 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:54:44 -0800 Subject: [PATCH 55/85] style(storage): align project-root regression formatting Refs #4099 --- crates/tracedecay-maintenance/src/retention/footprint.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 5054fb98be2..e73c86d4b06 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -1149,7 +1149,6 @@ mod tests { assert_eq!(remaining, 100 - 1 - 64 - 1 - 1); } - #[cfg(unix)] #[test] fn unreadable_project_root_has_its_own_partial_finding() { From fb4b65fc185f7bfa938e96fe6b0c600b369201ef Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:51:05 -0800 Subject: [PATCH 56/85] style(storage): satisfy pinned footprint rustfmt (Refs #4099) --- crates/tracedecay-maintenance/src/retention/footprint.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index e73c86d4b06..e106386820a 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -1173,7 +1173,10 @@ mod tests { ]); let footprint = census_profile_footprint(&profile, ®istered, &overrides); - assert_eq!(footprint.total_bytes, 11, "foreign bytes must not be counted"); + assert_eq!( + footprint.total_bytes, 11, + "foreign bytes must not be counted" + ); assert_eq!(footprint.unavailable_entry_count, 1); assert_eq!(footprint.entries.len(), 2, "no generic projects entry"); assert_eq!( From 1825f4aadc75c09651393685ad10ba215880b271 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:14:27 -0800 Subject: [PATCH 57/85] test(storage): keep hashed WAL diagnostic test valid Refs #4099. A nested .wal payload is not a WAL family after root-only classification; test bounded evidence-key behavior directly. --- .../src/retention/footprint.rs | 76 +++++++++---------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index e106386820a..4c51d7fe06a 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -1258,47 +1258,43 @@ mod tests { assert_eq!(remaining, 9); } - #[cfg(unix)] #[test] fn hashed_wal_reference_keeps_overages_and_partial_coverage() { - let temp = tempfile::tempdir().expect("footprint fixture"); - let profile = temp.path().join("profile"); - let mut directory = profile.join("projects/proj_deep"); - // Linux can create a key longer than Doctor's 1,024-byte reference. - // On Unix filesystems with a smaller path limit, a control character - // reaches the same reference fallback through a real directory walk. - #[cfg(target_os = "linux")] - for index in 0..6 { - directory.push(format!("shard_{index}_{}", "x".repeat(192))); - } - #[cfg(not(target_os = "linux"))] - directory.push("shard\narchive"); - let wal_directory = directory.join("sessions.wal"); - sized(&wal_directory.join("segment-1"), 2 * GIB); - std::os::unix::fs::symlink("missing", wal_directory.join("segment-2")) - .expect("unreadable WAL segment"); - let mut overrides = BTreeMap::from([ - ("projects/proj_deep".to_owned(), 1), - (PROFILE_TOTAL_KEY.to_owned(), 1), - ]); - let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); - assert_eq!(footprint.total_bytes, 2 * GIB); - let wal = footprint - .entries - .iter() - .find(|entry| entry.family == ProfileFamilyV1::Wal) - .expect("named WAL footprint"); - #[cfg(target_os = "linux")] - assert!( - wal.key.len() > 1024, - "fixture must exceed Doctor identity bound" - ); - assert!(DoctorEvidenceReferenceV1::new(wal.key.clone()).is_err()); - assert_eq!(wal.bytes, 2 * GIB); + // A real WAL overlay is anchored at a store root; nested payloads do + // not create WAL families. Construct the oversized diagnostic key + // directly to exercise the bounded Doctor evidence contract. + let key = format!("projects/proj_deep/{}.wal", "x".repeat(1025)); + let mut footprint = ProfileFootprintV1 { + entries: vec![ + ProfileFootprintEntryV1 { + key: "projects/proj_deep".to_owned(), + family: ProfileFamilyV1::ProjectStore, + bytes: 2 * GIB, + unavailable_entry_count: 1, + bound_bytes: 1, + project: None, + kinds: None, + }, + ProfileFootprintEntryV1 { + key: key.clone(), + family: ProfileFamilyV1::Wal, + bytes: 2 * GIB, + unavailable_entry_count: 1, + bound_bytes: default_bound_bytes(ProfileFamilyV1::Wal), + project: None, + kinds: None, + }, + ], + total_bytes: 2 * GIB, + unavailable_entry_count: 1, + truncated: false, + total_bound_bytes: 1, + }; + assert!(DoctorEvidenceReferenceV1::new(key.clone()).is_err()); let observed = findings(&footprint); assert_eq!(observed.len(), footprint.entries.len() + 1); - let reference = format!("footprint/sha256:{}", sha256_hex(wal.key.as_bytes())); + let reference = format!("footprint/sha256:{}", sha256_hex(key.as_bytes())); let wal_finding = &observed[&reference]; assert_eq!( wal_finding.finding().state(), @@ -1314,17 +1310,17 @@ mod tests { statement.contains(&format_bytes(default_bound_bytes(ProfileFamilyV1::Wal))), "{statement}" ); + assert!(statement.contains("1 unreadable entries"), "{statement}"); for reference in ["projects/proj_deep", PROFILE_TOTAL_KEY] { assert_eq!( observed[reference].finding().state(), DoctorEvidenceStateV1::Degraded, - "a long WAL identity must not suppress the other overages" + "an oversized identity must not suppress other overages" ); } - overrides.insert(wal.key.clone(), 3 * GIB); - let partial = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); - let observed = findings(&partial); + footprint.entries[1].bound_bytes = 3 * GIB; + let observed = findings(&footprint); let finding = observed[&reference].finding(); assert_eq!(finding.state(), DoctorEvidenceStateV1::Partial); let statement = finding.coverage().statement(); From 7e021897660a750a9eb767c9d50ba4a28ada7ce6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:15:16 -0800 Subject: [PATCH 58/85] fix(storage): expose replaced project-root files in doctor Refs #4099. Preserve measured bytes but mark the missing store traversal unavailable and apply the unknown-file bound. Exercise normal and overbound replacements through the real footprint census. --- .../src/retention/footprint.rs | 103 +++++++++++++++++- 1 file changed, 99 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 4c51d7fe06a..41752ce4986 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -311,6 +311,8 @@ struct EntryTally { bytes: u64, unavailable_entry_count: usize, kinds: Option, + /// An expected project directory was replaced by a regular file. + unexpected_project_file: bool, } struct FootprintKeys { @@ -348,10 +350,28 @@ pub(crate) fn census_profile_footprint_with_budget( else { return; }; + // The file itself is measurable, but it cannot be a usable + // project-store directory. Keep its bytes in the physical total, + // and report the missing store traversal as unavailable rather + // than calling the replaced project healthy. + let unexpected_project_file = matches!(walked, WalkedEntryV1::File { .. }) + && keys + .project + .as_ref() + .is_some_and(|(_, relative)| relative.as_os_str().is_empty()); + if unexpected_project_file { + unavailable_entry_count = unavailable_entry_count.saturating_add(1); + } let primary = tallies.entry(keys.primary).or_default(); tally(primary, walked); + if unexpected_project_file { + primary.unavailable_entry_count = + primary.unavailable_entry_count.saturating_add(1); + primary.unexpected_project_file = true; + } if let (Some((_, store_relative)), WalkedEntryV1::File { bytes }) = (&keys.project, walked) + && !unexpected_project_file { primary .kinds @@ -375,10 +395,15 @@ pub(crate) fn census_profile_footprint_with_budget( } }); ProfileFootprintEntryV1 { - bound_bytes: bound_overrides - .get(&key) - .copied() - .unwrap_or_else(|| default_bound_bytes(family)), + bound_bytes: bound_overrides.get(&key).copied().unwrap_or_else(|| { + if tally.unexpected_project_file { + // An unrelated file must not inherit a project's 10 GiB + // allowance. An explicit owner override still wins. + default_bound_bytes(ProfileFamilyV1::Other) + } else { + default_bound_bytes(family) + } + }), key, family, bytes: tally.bytes, @@ -1211,6 +1236,76 @@ mod tests { ); } + #[test] + fn regular_file_replacing_project_directory_is_not_healthy_or_project_budgeted() { + let temporary = tempfile::tempdir().expect("footprint fixture"); + let profile = temporary.path().join("profile"); + sized(&profile.join("projects/proj_replaced"), 11); + sized(&profile.join("projects/proj_healthy/tracedecay.db"), 7); + let registered_root = temporary.path().join("registered-checkout"); + std::fs::create_dir(®istered_root).expect("registered root"); + let registered = BTreeMap::from([( + "proj_replaced".to_owned(), + registered_root.display().to_string(), + )]); + + let footprint = census_profile_footprint(&profile, ®istered, &BTreeMap::new()); + assert_eq!(footprint.total_bytes, 18, "count all regular-file bytes"); + assert_eq!(footprint.unavailable_entry_count, 1); + assert_eq!(footprint.entries.len(), 2, "name the replaced store"); + let replaced = entry(&footprint, "projects/proj_replaced"); + assert_eq!(replaced.family, ProfileFamilyV1::ProjectStore); + assert_eq!(replaced.bytes, 11, "do not hide the replacement file"); + assert_eq!(replaced.unavailable_entry_count, 1); + assert_eq!( + replaced.bound_bytes, + default_bound_bytes(ProfileFamilyV1::Other), + "a replacement file must not inherit the 10 GiB project allowance" + ); + assert!(replaced.kinds.is_none(), "not a valid project-store file"); + assert_eq!( + replaced.project.as_ref().expect("registration").state, + ProjectRootStateV1::Present + ); + let healthy = entry(&footprint, "projects/proj_healthy"); + assert_eq!(healthy.bytes, 7); + assert_eq!(healthy.unavailable_entry_count, 0); + assert_eq!( + healthy.bound_bytes, + default_bound_bytes(ProfileFamilyV1::ProjectStore) + ); + + let observed = findings(&footprint); + assert_eq!(observed.len(), 3); + assert_eq!( + observed["projects/proj_replaced"].finding().state(), + DoctorEvidenceStateV1::Partial + ); + assert_eq!( + observed["projects/proj_healthy"].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage + ); + assert_eq!( + observed[PROFILE_TOTAL_KEY].finding().state(), + DoctorEvidenceStateV1::Partial + ); + + sized( + &profile.join("projects/proj_replaced"), + default_bound_bytes(ProfileFamilyV1::Other) + 1, + ); + let over = census_profile_footprint(&profile, ®istered, &BTreeMap::new()); + assert_eq!( + over.total_bytes, + default_bound_bytes(ProfileFamilyV1::Other) + 1 + 7 + ); + assert_eq!( + findings(&over)["projects/proj_replaced"].finding().state(), + DoctorEvidenceStateV1::Degraded, + "a large replacement file must trigger its tighter named bound" + ); + } + #[test] fn one_entry_budget_is_shared_across_walk_roots() { let temporary = tempfile::tempdir().expect("walk fixture"); From f0d9914b121dcab3f6a4c3d3b32f48cde6b48598 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:15:57 -0800 Subject: [PATCH 59/85] test(storage): exercise replacement bound without large disk allocation Refs #4099. Test the degraded threshold using an explicit small limit while retaining the independent default-bound assertion. --- .../src/retention/footprint.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 41752ce4986..3e6cb971d95 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -1290,15 +1290,12 @@ mod tests { DoctorEvidenceStateV1::Partial ); - sized( - &profile.join("projects/proj_replaced"), - default_bound_bytes(ProfileFamilyV1::Other) + 1, - ); - let over = census_profile_footprint(&profile, ®istered, &BTreeMap::new()); - assert_eq!( - over.total_bytes, - default_bound_bytes(ProfileFamilyV1::Other) + 1 + 7 - ); + // Exercise the same overage path without allocating a 1 GiB fixture + // on platforms that eagerly provision extended regular files. + let overrides = BTreeMap::from([("projects/proj_replaced".to_owned(), 10)]); + let over = census_profile_footprint(&profile, ®istered, &overrides); + assert_eq!(over.total_bytes, 18); + assert_eq!(entry(&over, "projects/proj_replaced").bound_bytes, 10); assert_eq!( findings(&over)["projects/proj_replaced"].finding().state(), DoctorEvidenceStateV1::Degraded, From 7de8bcd8c3884ef13431a989e3ccf709282d1d23 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:24:52 -0800 Subject: [PATCH 60/85] style(retention): apply rustfmt to footprint tally --- crates/tracedecay-maintenance/src/retention/footprint.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 3e6cb971d95..b1d97a5d50f 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -365,8 +365,7 @@ pub(crate) fn census_profile_footprint_with_budget( let primary = tallies.entry(keys.primary).or_default(); tally(primary, walked); if unexpected_project_file { - primary.unavailable_entry_count = - primary.unavailable_entry_count.saturating_add(1); + primary.unavailable_entry_count = primary.unavailable_entry_count.saturating_add(1); primary.unexpected_project_file = true; } if let (Some((_, store_relative)), WalkedEntryV1::File { bytes }) = From 95e7ad1da297972c5be7bd5e997c0f4237c9419f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:41:15 -0800 Subject: [PATCH 61/85] fix(storage): retry table census after freelist reclamation Refs #4099 --- .../tracedecay-maintenance/src/telemetry.rs | 87 +++++++++++++++++-- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 01451afa647..2334817611f 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -44,15 +44,15 @@ struct TableWatermark { observed_at: UtcMicros, } -/// The store size at which a per-table `dbstat` walk last ran out of its -/// deadline. +/// The live-page count at which a per-table `dbstat` walk timed out. /// -/// The walk reads every page of every table, so its cost tracks the store -/// size: a multi-gigabyte session store needs minutes. Once a walk has run -/// out of time, repeating it on a store at least as large only spends the -/// deadline again. A smaller store, or a day-old mark, is walked again. +/// Deleting data can reclaim table pages to SQLite's freelist without shrinking +/// the database file. `dbstat` no longer walks those pages, so the retry +/// decision uses allocated pages minus freelist pages instead of file size. +/// A store with fewer live pages, or a day-old mark, gets another measurement. #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct OverBudgetTableSizeWalk { + /// Allocated pages minus freelist pages when the walk timed out. page_count: u64, observed_at: UtcMicros, } @@ -209,10 +209,13 @@ impl GuardedStoreTelemetryPort { let over_budget = self.table_growth_state().over_budget; let walk = native_runtime::spawn_blocking(move || { let interrupted = || telemetry_interruption(&context); - let page_count = handle + let sample = handle .store_size_telemetry(reader_wait, interrupted) - .map_err(|error| error.table_growth_unavailable_reason())? - .page_count; + .map_err(|error| error.table_growth_unavailable_reason())?; + let page_count = sample + .page_count + .checked_sub(sample.freelist_pages) + .ok_or(TableGrowthUnavailableReasonV1::StorageOther)?; if over_budget.is_some_and(|walk| walk.still_applies(page_count, now_micros())) { return Ok(TableSizeWalk::Skipped); } @@ -1328,6 +1331,72 @@ mod table_growth_tests { ); assert_eq!(port.table_growth_state().over_budget, None); } + + #[asupersync::test] + async fn freed_pages_retry_a_timed_out_census_without_file_compaction() { + let temporary = tempfile::tempdir().expect("telemetry fixture root"); + let (database, store, port) = + fixture_port(&temporary.path().join("project.db"), Duration::from_secs(5)).await; + let context = storage_telemetry_request_context(port.scope.clone()).expect("context"); + + database + .execute_write_batch( + "grow a disposable table before retention", + "WITH RECURSIVE n(i) AS ( + SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 8 + ) INSERT INTO growth_payload SELECT zeroblob(1048576) FROM n;", + ) + .await + .expect("fill the table with real overflow pages"); + let StorageTelemetryReadV1::Observed { sample: before } = + port.store_size(&context, &store).await + else { + panic!("the grown store must expose its page and freelist counts"); + }; + let live_pages_before = before.page_count - before.freelist_pages; + assert!(live_pages_before > 1024, "the fixture must have many live pages"); + + port.table_growth + .lock() + .expect("table growth state") + .over_budget = Some(OverBudgetTableSizeWalk { + page_count: live_pages_before, + observed_at: now_micros(), + }); + assert!(matches!( + port.table_growth(&context, &store).await, + TableGrowthTelemetryReadV1::Unknown { + reason: TableGrowthUnavailableReasonV1::ExceedsMeasurementBudget, + .. + } + )); + + database + .execute_write_batch( + "reclaim the disposable table without vacuum", + "DELETE FROM growth_payload;", + ) + .await + .expect("reclaim the table's overflow pages"); + let StorageTelemetryReadV1::Observed { sample: after } = + port.store_size(&context, &store).await + else { + panic!("the reclaimed store must expose its page and freelist counts"); + }; + assert_eq!( + after.page_count, before.page_count, + "the database did not shrink, so allocated-page retry suppression would still apply" + ); + assert!(after.freelist_pages > before.freelist_pages); + assert!(after.page_count - after.freelist_pages < live_pages_before); + + let retried = port.table_growth(&context, &store).await; + assert!( + matches!(retried, TableGrowthTelemetryReadV1::BaselineEstablished { .. }), + "the cheaper post-reclamation census must run without a restart: {retried:?}" + ); + assert_eq!(port.table_growth_state().over_budget, None); + } } #[cfg(test)] From 6179c8bb711c2e3e4f8941fcb4398a7831e2244b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:46:40 -0800 Subject: [PATCH 62/85] test(storage): preserve growth baselines through reclamation Refs #4099. Preserve the concurrent freelist-aware retry fix and its regression. Add a real maintenance-registry journey proving unchanged work remains suppressed, logical deletion retries without compaction, old watermarks survive that retry, and subsequent growth is measured. Seed the existing deadline transition deterministically rather than relying on host speed. Keep production policy and measurement limits unchanged. --- .../tracedecay-maintenance/src/telemetry.rs | 6 +- .../src/telemetry/reclaim_tests.rs | 152 ++++++++++++++++++ 2 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 2334817611f..3d3084f2fe1 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -26,6 +26,8 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use crate::tick::MaintenanceTickOutcome; +#[cfg(test)] +mod reclaim_tests; mod text_artifact_scans; use text_artifact_scans::TextArtifactScansV1; @@ -1667,8 +1669,8 @@ mod registry_tests { assert!(registry.graph_replay_release_attempt_admitted(project)); assert!(registry.graph_replay_release_attempt_admitted(project)); - // Consecutive unhealthy attempts widen the skip window 1, 2, 4, and - // cap at GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS denied ticks. + // Consecutive unhealthy attempts open a bounded skip window: + // 1, 2, 4, and capped at GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS. for expected_skips in [1_usize, 2, 4, 8, 8] { registry.record_graph_replay_release_unhealthy(project); let mut denied = 0_usize; diff --git a/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs b/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs new file mode 100644 index 00000000000..68ec289be8d --- /dev/null +++ b/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs @@ -0,0 +1,152 @@ +//! Retry real table measurements after logical reclamation, without compaction. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::collections::BTreeSet; + +use tracedecay_contracts::now_micros; +use tracedecay_contracts::storage::{ + StorageTelemetryReadV1, StoreSizeTelemetryPort, TableGrowthTelemetryReadV1, + TableGrowthUnavailableReasonV1, +}; +use tracedecay_runtime_core::cancellation::CancellationToken; + +use super::table_growth_tests::growth_fixture; +use super::{ + StoreTelemetrySamplingRegistry, TableSizeWalk, settle_table_size_walk, + storage_telemetry_request_context, +}; + +#[asupersync::test] +async fn logical_reclamation_retries_a_timed_out_census_without_shrinking_the_file() { + let temporary = tempfile::tempdir().expect("telemetry reclamation fixture"); + let database_path = temporary.path().join("project.db"); + let (database, scope) = growth_fixture(&database_path).await; + database + .execute_write_batch( + "grow reclaimable payload", + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM n WHERE x < 32) \ + INSERT INTO growth_payload SELECT zeroblob(262144) FROM n;", + ) + .await + .expect("grow a real table before its baseline"); + + let registry = StoreTelemetrySamplingRegistry::default(); + assert!(registry.register_port(&database_path, &scope, || { + database.storage_telemetry_handle() + })); + let paths = BTreeSet::from([database_path.clone()]); + let cancellation = CancellationToken::new(); + let baseline = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(baseline.observed, 1); + assert_eq!(baseline.unavailable, 0); + let (store, port) = registry + .registered_port(&database_path, &scope) + .expect("registered measurement port"); + let context = storage_telemetry_request_context(scope.clone()).expect("reader context"); + let TableGrowthTelemetryReadV1::BaselineEstablished { observed_at, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("the first pass must retain a real baseline"); + }; + let StorageTelemetryReadV1::Observed { sample: before } = + port.store_size(&context, &store).await + else { + panic!("the pre-reclamation size must be measured"); + }; + let occupied_before = before.page_count - before.freelist_pages; + assert!(occupied_before > 0); + + // Seed the production timeout transition deterministically; reproducing a + // wall-clock timeout would make this correctness test depend on host speed. + let TableSizeWalk::OverBudget(mark) = settle_table_size_walk( + Err(TableGrowthUnavailableReasonV1::DeadlineExceeded), + occupied_before, + now_micros(), + ) + .unwrap() + else { + panic!("a deadline must produce a retry-suppression mark"); + }; + port.table_growth.lock().unwrap().over_budget = Some(mark); + let unchanged = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(unchanged.observed, 0); + assert_eq!(unchanged.unavailable, 1); + assert!(matches!( + port.latest_table_growth(&context, &store), + TableGrowthTelemetryReadV1::Unknown { + reason: TableGrowthUnavailableReasonV1::ExceedsMeasurementBudget, + .. + } + )); + + database + .execute_write_batch( + "reclaim table payload through its writer", + "DELETE FROM growth_payload WHERE rowid > 1;", + ) + .await + .expect("logical deletion without VACUUM or restart"); + let context = storage_telemetry_request_context(scope.clone()).expect("reader context"); + let StorageTelemetryReadV1::Observed { sample: after } = + port.store_size(&context, &store).await + else { + panic!("the post-reclamation size must be measured"); + }; + assert!( + after.page_count >= before.page_count, + "the old allocated-page guard must still suppress this fixture" + ); + assert!(after.freelist_pages > before.freelist_pages); + assert!(after.page_count - after.freelist_pages < occupied_before); + + let retried = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(retried.observed, 1); + assert_eq!(retried.unavailable, 0); + let TableGrowthTelemetryReadV1::Observed { samples, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("retry must compare with the original baseline, not replace it"); + }; + let retained = samples + .iter() + .find(|sample| sample.table.as_str() == "growth_payload") + .expect("the retained table remains measured"); + assert_eq!(retained.previous_observed_at, observed_at); + assert!(retained.previous_bytes.get() >= 8 * 1024 * 1024); + assert!(retained.current_bytes.get() >= 262_144); + assert!(retained.current_bytes.get() < retained.previous_bytes.get()); + let retained_bytes = retained.current_bytes; + assert_eq!(port.table_growth_state().over_budget, None); + + database + .execute_write_batch( + "grow the retained table again", + "INSERT INTO growth_payload VALUES (zeroblob(262144));", + ) + .await + .expect("growth after reclamation"); + let resumed = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(resumed.observed, 1); + assert_eq!(resumed.unavailable, 0); + let context = storage_telemetry_request_context(scope).expect("reader context"); + let TableGrowthTelemetryReadV1::Observed { samples, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("subsequent growth must remain observable"); + }; + let grown = samples + .iter() + .find(|sample| sample.table.as_str() == "growth_payload") + .expect("the regrown table is measured"); + assert_eq!(grown.previous_bytes, retained_bytes); + assert!(grown.growth_bytes().get() >= 262_144); +} From aaf3e6fa2349e2a9afee5e38d6157f7ecf2213da Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:17:05 -0800 Subject: [PATCH 63/85] style(storage): apply Bazel-pinned Rust formatting Refs #4099. Match the three exact rustfmt hunks reported by Repository gates on the reclamation regression. No production behavior, test expectations, or CI policy changes. --- crates/tracedecay-maintenance/src/telemetry.rs | 10 ++++++++-- .../src/telemetry/reclaim_tests.rs | 3 +-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 3d3084f2fe1..6efec00c94e 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -1356,7 +1356,10 @@ mod table_growth_tests { panic!("the grown store must expose its page and freelist counts"); }; let live_pages_before = before.page_count - before.freelist_pages; - assert!(live_pages_before > 1024, "the fixture must have many live pages"); + assert!( + live_pages_before > 1024, + "the fixture must have many live pages" + ); port.table_growth .lock() @@ -1394,7 +1397,10 @@ mod table_growth_tests { let retried = port.table_growth(&context, &store).await; assert!( - matches!(retried, TableGrowthTelemetryReadV1::BaselineEstablished { .. }), + matches!( + retried, + TableGrowthTelemetryReadV1::BaselineEstablished { .. } + ), "the cheaper post-reclamation census must run without a restart: {retried:?}" ); assert_eq!(port.table_growth_state().over_budget, None); diff --git a/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs b/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs index 68ec289be8d..f7327dced63 100644 --- a/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs +++ b/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs @@ -66,8 +66,7 @@ async fn logical_reclamation_retries_a_timed_out_census_without_shrinking_the_fi occupied_before, now_micros(), ) - .unwrap() - else { + .unwrap() else { panic!("a deadline must produce a retry-suppression mark"); }; port.table_growth.lock().unwrap().over_budget = Some(mark); From 3610e38c4ce79d01bf2dce2f576c286f2d412791 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:25:56 -0800 Subject: [PATCH 64/85] fix(storage): refuse linked profile roots in footprint census Refs #4099. Directory enumeration follows symlinks when the walk root itself is linked, even though child entries use no-follow metadata. Check each pending directory before enumeration, keep its unreadable coverage explicit, and never attribute the linked target's bytes to the requested profile. Add a filesystem-backed regression: the linked profile reports a partial zero-byte floor without foreign evidence; the directly selected real root reports the exact 2048-byte overage. Preserve existing entry budgets and cancellation behavior. --- .../src/retention/footprint.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index b1d97a5d50f..069540813e0 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -237,6 +237,16 @@ pub fn walk_regular_files_with_budget( WalkInterruptionV1::DeadlineExceeded => WalkTerminationV1::DeadlineExceeded, }; } + // read_dir follows a symlink when the directory argument itself is + // a symlink. Check every pending directory, including the caller's + // root, instead of relying only on child DirEntry::file_type. + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + _ => { + visit(&directory, WalkedEntryV1::Unavailable); + continue; + } + } let Ok(entries) = std::fs::read_dir(&directory) else { visit(&directory, WalkedEntryV1::Unavailable); continue; @@ -1959,6 +1969,45 @@ mod tests { assert_eq!(remaining, 10); } + #[cfg(unix)] + #[test] + fn linked_profile_root_cannot_measure_foreign_storage_as_owned() { + let temporary = tempfile::tempdir().expect("footprint fixture"); + let outside = temporary.path().join("outside"); + let profile = temporary.path().join("profile"); + sized(&outside.join("lcm-payloads/foreign"), 2048); + std::os::unix::fs::symlink(&outside, &profile).expect("linked profile root"); + let overrides = BTreeMap::from([("lcm-payloads".to_owned(), 1)]); + + let linked = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(linked.total_bytes, 0, "foreign bytes must not be owned"); + assert_eq!(linked.unavailable_entry_count, 1); + assert!(linked.entries.is_empty()); + assert!(!linked.truncated); + assert_eq!( + linked.full_profile_size().state, + ProfileTotalCoverageStateV1::Partial + ); + let linked_findings = findings(&linked); + assert_eq!(linked_findings.len(), 1); + assert_eq!( + linked_findings["profile"].finding().state(), + DoctorEvidenceStateV1::Partial, + "unverifiable root must not be reported healthy" + ); + + // The same bytes must be measured and over-budget when the actual + // directory is explicitly selected instead of its symbolic link. + let owned = census_profile_footprint(&outside, &BTreeMap::new(), &overrides); + assert_eq!(owned.total_bytes, 2048); + assert_eq!(owned.unavailable_entry_count, 0); + assert_eq!(entry(&owned, "lcm-payloads").bytes, 2048); + assert_eq!( + findings(&owned)["lcm-payloads"].finding().state(), + DoctorEvidenceStateV1::Degraded + ); + } + #[test] fn sqlite_wal_lookalike_directories_keep_independent_bounds() { let temp = tempfile::tempdir().expect("footprint fixture"); From 607a49650cd98344bc2415d5d63c9220f4315a99 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:00:19 -0800 Subject: [PATCH 65/85] fix(storage): refuse linked roots before report sampling Refs #4099. The profile footprint census refuses a linked root, but the offline registry reader and per-project report still resolved child paths beneath that link and sampled foreign storage. Reject the linked profile authority before registry reads, paged reports or project sizing. Cover the actual offline and project report paths with a filesystem symlink and foreign payloads; explicitly selecting the real directory continues to report exact measured bytes. --- .../src/retention/storage_report.rs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index 222f819eb47..bcd19dfb49c 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -626,6 +626,7 @@ async fn build_storage_report_with_walk_budget( profile_root: &Path, walk_entry_budget: usize, ) -> tracedecay_domain::errors::Result { + reject_linked_profile_root(profile_root)?; let global_db_path = profile_root.join(GLOBAL_DB_FILENAME); // The registry rows come from an off-profile copy, so reading them first // leaves the profile untouched. @@ -914,6 +915,7 @@ async fn build_storage_report_page_with_walk_budget( cancellation: Option, deadline: Option, ) -> tracedecay_domain::errors::Result { + reject_linked_profile_root(profile_root)?; let limit = limit.clamp(1, MAX_STORAGE_REPORT_PAGE_LIMIT); let walk_control = StorageReportWalkControl { cancellation, @@ -1249,6 +1251,7 @@ fn build_project_storage_report_with_control( sample_free_page_header: bool, walk_control: &StorageReportWalkControl, ) -> tracedecay_domain::errors::Result { + reject_linked_profile_root(profile_root)?; tracedecay_runtime_core::storage::validate_project_id(project_id).map_err(|message| { tracedecay_domain::errors::TraceDecayError::Config { message: message.to_owned(), @@ -1286,6 +1289,21 @@ fn build_project_storage_report_with_control( }) } +/// A caller-supplied profile symlink is not the authority that owns its target. +/// The physical footprint walker refuses that link; report readers must also +/// refuse it before inspecting registry rows or sampling individual stores. +fn reject_linked_profile_root(profile_root: &Path) -> tracedecay_domain::errors::Result<()> { + match std::fs::symlink_metadata(profile_root) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(report_error( + "inspect storage report profile root", + "profile root is a symbolic link; refusing to read foreign storage", + )), + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(report_error("inspect storage report profile root", error)), + } +} + fn directory_chain_is_nofollow(root: &Path, directory: &Path) -> bool { if !std::fs::symlink_metadata(root).is_ok_and(|metadata| metadata.is_dir()) { return false; @@ -3272,6 +3290,44 @@ mod tests { assert_eq!(foreign.metadata().unwrap().len(), 512 * 1024 * 1024); } + #[cfg(unix)] + #[asupersync::test] + async fn a_linked_profile_cannot_read_foreign_registry_or_store_bytes() { + let temporary = tempfile::tempdir().unwrap(); + let foreign = temporary.path().join("foreign"); + let linked = temporary.path().join("profile-link"); + std::fs::create_dir_all(foreign.join("projects/proj_foreign")).unwrap(); + std::fs::write(foreign.join("projects/proj_foreign/payload"), [1u8; 16]).unwrap(); + std::fs::write(foreign.join(USER_SESSIONS_DB_FILENAME), [2u8; 4_096]).unwrap(); + std::os::unix::fs::symlink(&foreign, &linked).unwrap(); + + // Before the guard, the footprint was partial, but offline sampling + // still followed this profile link and attributed foreign store bytes. + let offline = build_storage_report(&linked) + .await + .expect_err("offline report must not read linked profile storage"); + assert!(offline.to_string().contains("symbolic link"), "{offline}"); + + // The explicit project route also reports shared profile DB sizes, + // so it must not silently measure a linked profile either. + let project = build_project_storage_report( + &linked, + "proj_foreign", + Path::new("/repos/foreign"), + Err(RETENTION_PROTECTION_UNRESOLVED), + ) + .expect_err("project-scoped report must refuse linked profile storage"); + assert!(project.to_string().contains("symbolic link"), "{project}"); + + // Explicitly selecting the real directory remains measurable. The + // refusal is for a linked authority, not for the fixture's contents. + let real = build_storage_report(&foreign).await.unwrap(); + assert_eq!(real.user_sessions_db_bytes, 4_096); + assert_eq!(real.unregistered_dir_count, 1); + assert_eq!(real.unregistered_bytes, 16); + assert_eq!(real.full_profile_size.unwrap().total_bytes, 4_112); + } + #[asupersync::test] async fn storage_report_names_user_sessions_db_family() { let tmp = tempfile::TempDir::new().unwrap(); From 830e9cb570288d75786ed8b6b85f51fd63bb7c8a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:01:30 -0800 Subject: [PATCH 66/85] test(storage): prove linked roots refuse real registry sampling Refs #4099. Seed a real foreign SQLite registry alongside the external project and sessions bytes in the linked-profile refusal journey. The explicit real-root control must report the registered project and exact payload bytes, proving the negative case is not vacuous. --- .../src/retention/storage_report.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index bcd19dfb49c..97cd4485383 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -3299,6 +3299,7 @@ mod tests { std::fs::create_dir_all(foreign.join("projects/proj_foreign")).unwrap(); std::fs::write(foreign.join("projects/proj_foreign/payload"), [1u8; 16]).unwrap(); std::fs::write(foreign.join(USER_SESSIONS_DB_FILENAME), [2u8; 4_096]).unwrap(); + seed_global_db(&foreign, &[("proj_foreign", "/repos/foreign")]).await; std::os::unix::fs::symlink(&foreign, &linked).unwrap(); // Before the guard, the footprint was partial, but offline sampling @@ -3323,9 +3324,15 @@ mod tests { // refusal is for a linked authority, not for the fixture's contents. let real = build_storage_report(&foreign).await.unwrap(); assert_eq!(real.user_sessions_db_bytes, 4_096); - assert_eq!(real.unregistered_dir_count, 1); - assert_eq!(real.unregistered_bytes, 16); - assert_eq!(real.full_profile_size.unwrap().total_bytes, 4_112); + assert_eq!(real.stores.len(), 1); + assert_eq!(real.stores[0].project_id, "proj_foreign"); + assert_eq!(real.stores[0].total_bytes, 16); + assert_eq!(real.unregistered_dir_count, 0); + assert_eq!(real.unregistered_bytes, 0); + let registry_bytes = std::fs::metadata(foreign.join(GLOBAL_DB_FILENAME)) + .unwrap() + .len(); + assert!(real.full_profile_size.unwrap().total_bytes >= registry_bytes + 4_112); } #[asupersync::test] From 1b1b93459c7ee13203419a87885af5ee684ffe0f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:18:55 -0800 Subject: [PATCH 67/85] test(storage): satisfy strict Clippy for linked-root census Refs #4099. Preserve an exact empty entry set assertion with typed value-level diagnostics, rather than the Clippy-denied assert_is_empty form. No production semantics or coverage assertions change. --- crates/tracedecay-maintenance/src/retention/footprint.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 069540813e0..a9d8df8aa35 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -1982,7 +1982,7 @@ mod tests { let linked = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); assert_eq!(linked.total_bytes, 0, "foreign bytes must not be owned"); assert_eq!(linked.unavailable_entry_count, 1); - assert!(linked.entries.is_empty()); + assert_eq!(linked.entries, Vec::::new()); assert!(!linked.truncated); assert_eq!( linked.full_profile_size().state, From 83c09f0b0ef47994d54bcceec31d00c743315108 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:32:56 -0800 Subject: [PATCH 68/85] test(mcp): route restart fixture initialization explicitly The stdio proxy reads its optional hint setting on a background connection. Keep the restart fixture listening when that settings RPC arrives before the host initialize request, returning method-not-found to unsupported RPCs instead of consuming the sentinel response. Preserve the restart deadline, authenticated preamble, response drain, and exact initialize sentinel assertion. --- .../tests/mcp_suite/mcp_cli_serve_test.rs | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs index a85046acbb1..39ffc996921 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs @@ -693,24 +693,39 @@ async fn serve_started_during_daemon_restart_window_proxies_to_restarted_daemon( } let request: Value = serde_json::from_str(request_line.trim()).expect("request json"); - let response = json!({ - "jsonrpc": "2.0", - "id": request["id"], - "result": { - "protocolVersion": "2024-11-05", - "capabilities": { "tools": {} }, - "serverInfo": { - "name": "sentinel-restarted-daemon", - "version": "0.0.1-sentinel" + // Serve also reads its optional hint setting on a background + // connection. That request can reach the restarted listener + // first; only the host's initialize completes this fixture. + let initialize = request["method"] == "initialize"; + let response = if initialize { + json!({ + "jsonrpc": "2.0", + "id": request["id"], + "result": { + "protocolVersion": "2024-11-05", + "capabilities": { "tools": {} }, + "serverInfo": { + "name": "sentinel-restarted-daemon", + "version": "0.0.1-sentinel" + } } - } - }); + }) + } else { + json!({ + "jsonrpc": "2.0", + "id": request["id"], + "error": { + "code": -32601, + "message": "fixture daemon only serves initialize" + } + }) + }; writeln!(stream, "{response}").expect("write fake daemon response"); // Drain until the proxy hangs up so the response is not lost to a // connection reset. let mut scratch = [0_u8; 64]; while matches!(reader.read(&mut scratch), Ok(n) if n > 0) {} - Some(()) + initialize.then_some(()) }, || "timed out waiting for serve to proxy a request to the restarted daemon".to_string(), ); From f7e1257f86856c0c22c38de0f5510b86ff50e583 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:48:56 -0800 Subject: [PATCH 69/85] fix(storage): retain size bounds in long Doctor findings Retain measured bytes and configured bound when registered-root details or valid long evidence identifiers exceed Doctor's 512-byte text limit. Only compact optional details and hash the evidence reference if needed; full identities remain in the storage report. Add a filesystem-backed census-to-Doctor regression for both truncation cases. Refs #4099 --- .../src/retention/footprint.rs | 100 +++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index a9d8df8aa35..3763aae2eaa 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -615,6 +615,39 @@ fn doctor_footprint_reference_key(key: &str) -> Cow<'_, str> { Cow::Owned(format!("footprint/sha256:{}", sha256_hex(key.as_bytes()))) } +/// Preserve the measured size and bound in Doctor even when an optional +/// registered root, kind breakdown, or long footprint identity exceeds its +/// 512-byte coverage statement. Full identities and breakdowns remain in the +/// storage report; a truncated number would make the finding unactionable. +fn bounded_entry_subject(entry: &ProfileFootprintEntryV1) -> (String, String) { + let mut evidence_key = doctor_footprint_reference_key(&entry.key).into_owned(); + let mut subject = entry_statement(entry, &evidence_key); + let fits = |subject: &str| { + let statement = format!( + "{subject}: {} of {} bound", + format_bytes(entry.bytes), + format_bytes(entry.bound_bytes) + ); + super::diagnostics::bounded_statement(&statement) == statement + }; + if !fits(&subject) { + let mut compact = entry.clone(); + compact.kinds = None; + if let Some(project) = compact.project.as_mut() { + // Keep the verified root state, but omit its optional long path. + project.registered_root = None; + } + subject = entry_statement(&compact, &evidence_key); + if !fits(&subject) { + // A valid Doctor reference can be longer than its statement budget. + // Retain stable evidence identity without losing the numeric bound. + evidence_key = format!("footprint/sha256:{}", sha256_hex(entry.key.as_bytes())); + subject = entry_statement(&compact, &evidence_key); + } + } + (evidence_key, subject) +} + /// One `OverBudgetStore` finding per footprint entry plus one for the profile /// total. A floor above its bound proves the overage, so an over-bound entry /// is degraded with complete coverage even when bytes went unread. @@ -622,10 +655,9 @@ fn doctor_footprint_reference_key(key: &str) -> Cow<'_, str> { pub fn footprint_findings(footprint: &ProfileFootprintV1) -> DoctorStorageFamilyReadV1 { let mut findings = Vec::with_capacity(footprint.entries.len() + 1); for entry in &footprint.entries { - let evidence_key = doctor_footprint_reference_key(&entry.key); - let subject = entry_statement(entry, &evidence_key); + let (evidence_key, subject) = bounded_entry_subject(entry); let Some(finding) = bounded_finding( - evidence_key.as_ref(), + evidence_key.as_str(), &subject, entry.bytes, entry.bound_bytes, @@ -2091,4 +2123,66 @@ mod tests { ); } } + + #[test] + fn long_footprint_details_keep_doctor_size_and_bound_visible() { + let temporary = tempfile::tempdir().unwrap(); + let profile = temporary.path().join("profile"); + let key = "projects/proj_long"; + let bytes = 3 * 1024; + sized(&profile.join(key).join("tracedecay.db"), bytes); + let registered = BTreeMap::from([( + "proj_long".to_owned(), + temporary.path().display().to_string(), + )]); + let overrides = BTreeMap::from([(key.to_owned(), 1)]); + let mut footprint = census_profile_footprint(&profile, ®istered, &overrides); + let project = footprint + .entries + .iter_mut() + .find(|entry| entry.key == key) + .expect("measured project"); + assert_eq!( + project.project.as_ref().unwrap().state, + ProjectRootStateV1::Present + ); + // The complete state remains available in the storage report; only + // the Doctor coverage text is bounded. + project.project.as_mut().unwrap().registered_root = Some("long-root/".repeat(120)); + let observed = findings(&footprint); + let finding = &observed[key]; + let statement = finding.finding().coverage().statement(); + assert_eq!(finding.finding().state(), DoctorEvidenceStateV1::Degraded); + assert!(statement.contains("root present"), "{statement}"); + assert!( + statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), + "the measured overage must not be clipped after a long root: {statement}" + ); + assert!(!statement.contains("long-root/long-root"), "{statement}"); + + // A valid long evidence key can also fill the statement budget before + // the numeric overage. Hash the reference, never the reported bytes. + let project = footprint + .entries + .iter_mut() + .find(|entry| entry.key == key) + .expect("measured project"); + project.key = format!("projects/proj_{}/graph.grafeo", "x".repeat(520)); + let hash_key = format!("footprint/sha256:{}", sha256_hex(project.key.as_bytes())); + let observed = findings(&footprint); + let finding = &observed[&hash_key]; + let statement = finding.finding().coverage().statement(); + assert_eq!(finding.finding().state(), DoctorEvidenceStateV1::Degraded); + assert!(statement.contains("full path in storage report"), "{statement}"); + assert!( + statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), + "the complete numeric bound must survive a long evidence key: {statement}" + ); + assert_eq!( + std::fs::metadata(profile.join(key).join("tracedecay.db")) + .unwrap() + .len(), + bytes + ); + } } From 1564021dc5dd1991ef5455d5f84efa60c3825afe Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:17:37 -0800 Subject: [PATCH 70/85] style(maintenance): format footprint coverage assertion --- crates/tracedecay-maintenance/src/retention/footprint.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 3763aae2eaa..6275c0684e5 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -2173,7 +2173,10 @@ mod tests { let finding = &observed[&hash_key]; let statement = finding.finding().coverage().statement(); assert_eq!(finding.finding().state(), DoctorEvidenceStateV1::Degraded); - assert!(statement.contains("full path in storage report"), "{statement}"); + assert!( + statement.contains("full path in storage report"), + "{statement}" + ); assert!( statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), "the complete numeric bound must survive a long evidence key: {statement}" From f5a1f47a16ae2781165874cdeb73d9cd69d5ab31 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:32:40 -0800 Subject: [PATCH 71/85] fix(doctor): retain sanitized registered roots in footprint findings Compare the bounded final statement against the same control-folded text Doctor publishes, so short checkout names with tabs or newlines do not lose their identity. Extend the real profile census regression to prove sanitized identity, intact measured overage, and existing long-root trimming. Refs #4099 --- .../src/retention/footprint.rs | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 6275c0684e5..44a0d74223f 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -22,6 +22,7 @@ use tracedecay_contracts::doctor::{ storage_family_read, }; use tracedecay_domain::canonical_text::sha256_hex; +use tracedecay_domain::fold_control_characters; use tracedecay_global_db::registry_maintenance::{RootLivenessV1, probe_root}; use tracedecay_runtime_core::text::format_bytes; @@ -628,7 +629,11 @@ fn bounded_entry_subject(entry: &ProfileFootprintEntryV1) -> (String, String) { format_bytes(entry.bytes), format_bytes(entry.bound_bytes) ); - super::diagnostics::bounded_statement(&statement) == statement + // Compare the *rendered* statement. Doctor folds control characters, + // so a valid short registered path containing a tab or newline must + // not be mistaken for an overlong detail and silently omitted. + super::diagnostics::bounded_statement(&statement) + == fold_control_characters(&statement).trim() }; if !fits(&subject) { let mut compact = entry.clone(); @@ -2146,8 +2151,31 @@ mod tests { project.project.as_ref().unwrap().state, ProjectRootStateV1::Present ); + // A short registered checkout path with control characters fits once + // folded by Doctor. Preserve its useful identity rather than dropping + // it just because the sanitized representation differs byte-for-byte. + project.project.as_mut().unwrap().registered_root = + Some("checkout\tbranch\nalpha".to_owned()); + let sanitized = findings(&footprint); + let sanitized_statement = sanitized[key].finding().coverage().statement(); + assert!( + sanitized_statement.contains("for checkout branch alpha (root present)"), + "sanitized registered checkout was omitted: {sanitized_statement}" + ); + assert!(!sanitized_statement.contains('\t'), "{sanitized_statement}"); + assert!(!sanitized_statement.contains('\n'), "{sanitized_statement}"); + assert!( + sanitized_statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), + "a sanitized identity must keep the exact measured overage: {sanitized_statement}" + ); + // The complete state remains available in the storage report; only // the Doctor coverage text is bounded. + let project = footprint + .entries + .iter_mut() + .find(|entry| entry.key == key) + .expect("measured project"); project.project.as_mut().unwrap().registered_root = Some("long-root/".repeat(120)); let observed = findings(&footprint); let finding = &observed[key]; From 7ad8c461fec5aab88dc43e471ad0e5c7e8bf87dd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:17:56 -0800 Subject: [PATCH 72/85] fix(storage): bound retained table-growth checkpoint reads Refs #4099 Read the maintenance-owned table-growth state through the existing no-follow, nonblocking bounded-file authority. Refuse sparse oversized state and foreign symlinks without importing their bytes; warn and re-baseline as for invalid state. Cap the corresponding writer output before publication, preserving the previous file on refusal. A real-filesystem regression proves valid persistence and readback, sparse oversize refusal without payload allocation, and that a valid foreign symlink is not accepted. No schema, budgets for live store measurement, daemon GC, or CI policies change. --- .../tracedecay-maintenance/src/telemetry.rs | 106 +++++++++++++++++- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 6efec00c94e..73c57a62dc1 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -37,6 +37,9 @@ const STORAGE_TELEMETRY_USE_CASE: &str = "use-case.application.storage.telemetry const TABLE_GROWTH_STATE_DIRECTORY: &str = "maintenance"; const TABLE_GROWTH_STATE_FILE: &str = "table-growth-v1.json"; +/// The durable baseline is metadata, never an unbounded profile payload. +/// Leave room below the maintenance-family soft bound for other cursors. +const MAX_TABLE_GROWTH_STATE_BYTES: usize = 16 * 1024 * 1024; /// How long a walk that ran out of its deadline suppresses the next one. const TABLE_SIZE_WALK_REPROBE_MICROS: i64 = 24 * 60 * 60 * 1_000_000; @@ -891,10 +894,35 @@ impl StoreTelemetrySamplingRegistry { /// Every persisted store state, keyed by database path. A missing or /// unreadable file is an empty state: the next pass re-baselines. fn load_table_growth_states(state_path: &Path) -> BTreeMap { - std::fs::read(state_path) - .ok() - .and_then(|bytes| serde_json::from_slice(&bytes).ok()) - .unwrap_or_default() + // A stale or substituted maintenance file need not respect the size of + // data this daemon normally writes. Do not read a sparse multi-GB file, + // follow a foreign symlink, or wait forever on a FIFO during startup. + let bytes = match tracedecay_private_fs::framed_log::read_bounded( + state_path, + MAX_TABLE_GROWTH_STATE_BYTES, + ) { + Ok(Some(bytes)) => bytes, + Ok(None) => return BTreeMap::new(), + Err(error) => { + tracing::warn!( + path = %state_path.display(), + %error, + "table growth checkpoint cannot be read; measurements will re-baseline" + ); + return BTreeMap::new(); + } + }; + match serde_json::from_slice(&bytes) { + Ok(states) => states, + Err(error) => { + tracing::warn!( + path = %state_path.display(), + %error, + "table growth checkpoint is invalid; measurements will re-baseline" + ); + BTreeMap::new() + } + } } /// Merge `updates` into the persisted state and drop stores whose database no @@ -912,6 +940,12 @@ fn persist_table_growth_states( states.extend(updates); states.retain(|database_path, _| Path::new(database_path).exists()); let bytes = serde_json::to_vec(&states).map_err(std::io::Error::other)?; + if bytes.len() > MAX_TABLE_GROWTH_STATE_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "table growth checkpoint exceeds the bounded maintenance-state size", + )); + } if let Some(parent) = state_path.parent() { std::fs::create_dir_all(parent)?; } @@ -971,6 +1005,70 @@ fn storage_telemetry_request_context( ) } +#[cfg(test)] +#[allow(clippy::expect_used, clippy::panic)] +mod table_growth_checkpoint_tests { + use super::{ + MAX_TABLE_GROWTH_STATE_BYTES, TableGrowthStateV1, load_table_growth_states, + persist_table_growth_states, + }; + + #[test] + fn checkpoint_reads_refuse_oversized_and_foreign_files_without_losing_valid_state() { + let temp = tempfile::tempdir().expect("isolated checkpoint fixture"); + let database = temp.path().join("project.db"); + std::fs::write(&database, b"owned").expect("existing database identity"); + let checkpoint = temp.path().join("maintenance/table-growth-v1.json"); + let key = database.to_str().expect("UTF-8 fixture path").to_owned(); + persist_table_growth_states( + &checkpoint, + vec![(key.clone(), TableGrowthStateV1::default())], + ) + .expect("persist genuine bounded state"); + let loaded = load_table_growth_states(&checkpoint); + assert_eq!(loaded.len(), 1); + assert!(loaded.contains_key(&key)); + + // A sparse file exercises the held-file length refusal, without + // allocating or reading an oversized JSON payload in the test. + std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(&checkpoint) + .expect("replace disposable checkpoint") + .set_len(MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1) + .expect("sparse oversized checkpoint"); + assert_eq!(load_table_growth_states(&checkpoint).len(), 0); + assert_eq!( + std::fs::metadata(&checkpoint).unwrap().len(), + MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1 + ); + + #[cfg(unix)] + { + let foreign = temp.path().join("foreign-state.json"); + std::fs::write( + &foreign, + serde_json::to_vec(&std::collections::BTreeMap::from([( + key.clone(), + TableGrowthStateV1::default(), + )])) + .expect("valid foreign state"), + ) + .expect("foreign contents"); + std::fs::remove_file(&checkpoint).expect("remove disposable oversized file"); + std::os::unix::fs::symlink(&foreign, &checkpoint) + .expect("substitute foreign checkpoint symlink"); + assert_eq!( + load_table_growth_states(&checkpoint).len(), + 0, + "the previous std::fs::read would import this valid foreign state" + ); + assert!(std::fs::read(&foreign).unwrap().len() > 2); + } + } +} + #[cfg(test)] mod retention_tick_log_tests { use super::StoreTelemetrySamplingRegistry; From 035f9bb74f8f6a7a1bdbd24063b00a6aa072ac2d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:20:25 -0800 Subject: [PATCH 73/85] fix(storage): preserve refused growth checkpoints on disk Refs #4099 Do not replace an existing oversized, foreign-linked, or otherwise unreadable table-growth checkpoint after the new bounded reader refuses it. Surface the failure to persistence so the previous bytes remain untouched, while live sampling can establish a new in-memory baseline. Strengthen the real-file regression to assert refusal preserves sparse file length and foreign symlink/target through a persistence attempt. --- .../tracedecay-maintenance/src/telemetry.rs | 65 +++++++++++++------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 73c57a62dc1..84c56721f0f 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -576,7 +576,17 @@ impl StoreTelemetrySamplingRegistry { .table_growth_state_path .as_deref() .zip(path.to_str()) - .and_then(|(state_path, key)| load_table_growth_states(state_path).remove(key)) + .and_then(|(state_path, key)| match load_table_growth_states(state_path) { + Ok(mut states) => states.remove(key), + Err(error) => { + tracing::warn!( + path = %state_path.display(), + %error, + "table growth checkpoint is unavailable; preserving it and re-baselining this store" + ); + None + } + }) .unwrap_or_default(); let port = GuardedStoreTelemetryPort::new( handle, @@ -893,34 +903,29 @@ impl StoreTelemetrySamplingRegistry { /// Every persisted store state, keyed by database path. A missing or /// unreadable file is an empty state: the next pass re-baselines. -fn load_table_growth_states(state_path: &Path) -> BTreeMap { +fn load_table_growth_states( + state_path: &Path, +) -> std::io::Result> { // A stale or substituted maintenance file need not respect the size of // data this daemon normally writes. Do not read a sparse multi-GB file, // follow a foreign symlink, or wait forever on a FIFO during startup. + // Refuse unsafe reads rather than replacing their sole existing bytes. let bytes = match tracedecay_private_fs::framed_log::read_bounded( state_path, MAX_TABLE_GROWTH_STATE_BYTES, - ) { - Ok(Some(bytes)) => bytes, - Ok(None) => return BTreeMap::new(), - Err(error) => { - tracing::warn!( - path = %state_path.display(), - %error, - "table growth checkpoint cannot be read; measurements will re-baseline" - ); - return BTreeMap::new(); - } + )? { + Some(bytes) => bytes, + None => return Ok(BTreeMap::new()), }; match serde_json::from_slice(&bytes) { - Ok(states) => states, + Ok(states) => Ok(states), Err(error) => { tracing::warn!( path = %state_path.display(), %error, "table growth checkpoint is invalid; measurements will re-baseline" ); - BTreeMap::new() + Ok(BTreeMap::new()) } } } @@ -936,7 +941,7 @@ fn persist_table_growth_states( state_path: &Path, updates: Vec<(String, TableGrowthStateV1)>, ) -> std::io::Result<()> { - let mut states = load_table_growth_states(state_path); + let mut states = load_table_growth_states(state_path)?; states.extend(updates); states.retain(|database_path, _| Path::new(database_path).exists()); let bytes = serde_json::to_vec(&states).map_err(std::io::Error::other)?; @@ -1025,7 +1030,7 @@ mod table_growth_checkpoint_tests { vec![(key.clone(), TableGrowthStateV1::default())], ) .expect("persist genuine bounded state"); - let loaded = load_table_growth_states(&checkpoint); + let loaded = load_table_growth_states(&checkpoint).expect("valid checkpoint readback"); assert_eq!(loaded.len(), 1); assert!(loaded.contains_key(&key)); @@ -1038,7 +1043,18 @@ mod table_growth_checkpoint_tests { .expect("replace disposable checkpoint") .set_len(MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1) .expect("sparse oversized checkpoint"); - assert_eq!(load_table_growth_states(&checkpoint).len(), 0); + assert_eq!( + load_table_growth_states(&checkpoint).unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + assert!( + persist_table_growth_states( + &checkpoint, + vec![(key.clone(), TableGrowthStateV1::default())], + ) + .is_err(), + "an oversized existing checkpoint must not be replaced" + ); assert_eq!( std::fs::metadata(&checkpoint).unwrap().len(), MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1 @@ -1060,10 +1076,19 @@ mod table_growth_checkpoint_tests { std::os::unix::fs::symlink(&foreign, &checkpoint) .expect("substitute foreign checkpoint symlink"); assert_eq!( - load_table_growth_states(&checkpoint).len(), - 0, + load_table_growth_states(&checkpoint).unwrap_err().kind(), + std::io::ErrorKind::InvalidInput, "the previous std::fs::read would import this valid foreign state" ); + assert!( + persist_table_growth_states( + &checkpoint, + vec![(key, TableGrowthStateV1::default())], + ) + .is_err(), + "a foreign symlink must never be replaced by persistence" + ); + assert!(checkpoint.symlink_metadata().unwrap().file_type().is_symlink()); assert!(std::fs::read(&foreign).unwrap().len() > 2); } } From 15e0869f260ddb29c11f8dca78681db3286bd183 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:20:51 -0800 Subject: [PATCH 74/85] test(storage): assert checkpoint refusal without Debug state Refs #4099 Match the actual I/O error variant without requiring the persisted TableGrowthStateV1 map to implement Debug. Preserve positive on-disk checkpoint and foreign-link identity assertions. --- crates/tracedecay-maintenance/src/telemetry.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 84c56721f0f..6f63146668d 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -1043,10 +1043,10 @@ mod table_growth_checkpoint_tests { .expect("replace disposable checkpoint") .set_len(MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1) .expect("sparse oversized checkpoint"); - assert_eq!( - load_table_growth_states(&checkpoint).unwrap_err().kind(), - std::io::ErrorKind::InvalidData - ); + assert!(matches!( + load_table_growth_states(&checkpoint), + Err(error) if error.kind() == std::io::ErrorKind::InvalidData + )); assert!( persist_table_growth_states( &checkpoint, @@ -1075,9 +1075,11 @@ mod table_growth_checkpoint_tests { std::fs::remove_file(&checkpoint).expect("remove disposable oversized file"); std::os::unix::fs::symlink(&foreign, &checkpoint) .expect("substitute foreign checkpoint symlink"); - assert_eq!( - load_table_growth_states(&checkpoint).unwrap_err().kind(), - std::io::ErrorKind::InvalidInput, + assert!( + matches!( + load_table_growth_states(&checkpoint), + Err(error) if error.kind() == std::io::ErrorKind::InvalidInput + ), "the previous std::fs::read would import this valid foreign state" ); assert!( From 51494793a9c974f20cd9fca1241b24b16d6fa5e5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:14:44 -0800 Subject: [PATCH 75/85] fix(storage): retry table growth after clock regression Refs #4099 A failed dbstat timeout is a process-local wall-clock mark. If the wall clock moves backward, negative elapsed time currently passes the 24-hour suppression predicate and incorrectly blocks retries. Reject a future mark and cover the actual guarded telemetry port with a regression that starts from an established baseline and demands another live measured result. --- crates/tracedecay-maintenance/src/telemetry.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 6f63146668d..894904b777c 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -64,7 +64,9 @@ struct OverBudgetTableSizeWalk { impl OverBudgetTableSizeWalk { fn still_applies(self, page_count: u64, now: UtcMicros) -> bool { + // A regressed wall clock cannot extend a failed-walk hold indefinitely. page_count >= self.page_count + && now.0 >= self.observed_at.0 && now.0.saturating_sub(self.observed_at.0) < TABLE_SIZE_WALK_REPROBE_MICROS } } @@ -1457,6 +1459,22 @@ mod table_growth_tests { "a day-old over-budget mark is probed again" ); assert_eq!(port.table_growth_state().over_budget, None); + + // A prior timed-out walk can have a future wall-clock timestamp after + // time synchronization moves backward. It must not hide table growth + // for the entire clock skew plus the normal retry interval. + mark( + 1, + UtcMicros(now_micros().0.saturating_add(super::TABLE_SIZE_WALK_REPROBE_MICROS)), + ); + assert!( + matches!( + port.table_growth(&context, &store).await, + TableGrowthTelemetryReadV1::Observed { .. } + ), + "a future timeout marker must not suppress a live measurement" + ); + assert_eq!(port.table_growth_state().over_budget, None); } #[asupersync::test] From ec3ef1d968d6bfd9b85a06299cd6fa53b9e72531 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:27:19 -0800 Subject: [PATCH 76/85] style(storage): format telemetry retry regressions Apply the pinned formatter layout reported by repository gates. Validation: owning Bazel formatter target passed; independent review completed. --- crates/tracedecay-maintenance/src/telemetry.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index 894904b777c..352d6e55161 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -1092,7 +1092,13 @@ mod table_growth_checkpoint_tests { .is_err(), "a foreign symlink must never be replaced by persistence" ); - assert!(checkpoint.symlink_metadata().unwrap().file_type().is_symlink()); + assert!( + checkpoint + .symlink_metadata() + .unwrap() + .file_type() + .is_symlink() + ); assert!(std::fs::read(&foreign).unwrap().len() > 2); } } @@ -1465,7 +1471,11 @@ mod table_growth_tests { // for the entire clock skew plus the normal retry interval. mark( 1, - UtcMicros(now_micros().0.saturating_add(super::TABLE_SIZE_WALK_REPROBE_MICROS)), + UtcMicros( + now_micros() + .0 + .saturating_add(super::TABLE_SIZE_WALK_REPROBE_MICROS), + ), ); assert!( matches!( From 0ef68227674b4221761969dab9360d4e54c4fe57 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:11:38 -0800 Subject: [PATCH 77/85] test(mcp): cover the dispatch-ceiling deadline terminal `rename_preview_deadline_stops_the_worker_before_the_next_file` carries one deadline through three timers: the pre-dispatch gate, the admitted read, and the outer dispatch ceiling. The ceiling can settle first on a loaded host, and the graph-tool adapter reports that retryable route refusal as an unavailable pre-admission state under `tool_dispatch_deadline_exceeded` rather than the admitted read's `timed_out`. The test only accepted the other two terminals, so the ceiling's legal settlement failed the assertion. Pin that record exactly instead of accepting any deadline envelope. --- .../mcp_handler_test/rename_preview_test.rs | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs index 0c6800d13ae..d053018da7d 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs @@ -589,9 +589,12 @@ async fn rename_preview_deadline_stops_the_worker_before_the_next_file() { .unwrap() }; let response = serde_json::to_value(response).unwrap(); - // The outer dispatch and admitted read carry the same deadline. Either - // timer can settle first; both must stop this worker before its next file. + // Three timers carry this one deadline and any of them may settle first, + // so the exact terminal names which one won. All three stop this worker + // before its next file; none may hang, and none may claim a commit. match (response.get("error"), response.get("result")) { + // The deadline had already elapsed at dispatch, so nothing was + // admitted and the boundary answers the JSON-RPC request itself. (Some(error), None) => { assert_eq!(error["code"], -32603, "{response}"); let problem = &error["data"]; @@ -600,12 +603,31 @@ async fn rename_preview_deadline_stops_the_worker_before_the_next_file() { assert_eq!(problem["tool"], "tracedecay_rename_preview"); assert_eq!(problem["retryable"], true); } + // The admitted read's own deadline won while the worker paused on + // the source read. (None, Some(result)) => { let problem = refusal_problem(result); - assert_eq!(problem["kind"], "timed_out", "{response}"); - assert_eq!(problem["code"], "timed_out", "{response}"); - assert_eq!(problem["cancellation_stage"], "during_read", "{response}"); - assert_eq!(problem["terminality"], "admitted_terminal", "{response}"); + match problem["code"].as_str() { + Some("timed_out") => { + assert_eq!(problem["kind"], "timed_out", "{response}"); + assert_eq!(problem["cancellation_stage"], "during_read", "{response}"); + assert_eq!(problem["terminality"], "admitted_terminal", "{response}"); + } + // The outer dispatch ceiling won after admission. It is a + // retryable route refusal, which the graph-tool adapter + // reports as an unavailable pre-admission state under the + // ceiling's own reason code. + Some("tool_dispatch_deadline_exceeded") => { + assert_eq!(problem["kind"], "unavailable", "{response}"); + assert_eq!(problem["retryable"], true, "{response}"); + assert_eq!(problem["retry"], "after_delay", "{response}"); + assert_eq!(problem["legal_actions"], json!(["retry"]), "{response}"); + assert_eq!(problem["terminality"], "pre_admission", "{response}"); + assert_eq!(problem["owning_layer"], "application", "{response}"); + assert_eq!(problem["cancellation_stage"], Value::Null, "{response}"); + } + _ => panic!("expected one canonical deadline terminal: {response}"), + } } _ => panic!("expected one canonical deadline terminal: {response}"), } From 80f51e3cf36e36b94711538a1d49ad95bc913c2f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:13:05 -0800 Subject: [PATCH 78/85] Revert "test(mcp): cover the dispatch-ceiling deadline terminal" This reverts commit 0ef68227674b4221761969dab9360d4e54c4fe57. Accepting `unavailable`/`pre_admission` for a deadline that fires after admission would codify a likely terminality misclassification and is outside this PR's scope. The rename-preview race is pre-existing on master and should be fixed at its owner. Co-Authored-By: Claude Opus 5.5 --- .../mcp_handler_test/rename_preview_test.rs | 34 ++++--------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs index d053018da7d..0c6800d13ae 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_handler_test/rename_preview_test.rs @@ -589,12 +589,9 @@ async fn rename_preview_deadline_stops_the_worker_before_the_next_file() { .unwrap() }; let response = serde_json::to_value(response).unwrap(); - // Three timers carry this one deadline and any of them may settle first, - // so the exact terminal names which one won. All three stop this worker - // before its next file; none may hang, and none may claim a commit. + // The outer dispatch and admitted read carry the same deadline. Either + // timer can settle first; both must stop this worker before its next file. match (response.get("error"), response.get("result")) { - // The deadline had already elapsed at dispatch, so nothing was - // admitted and the boundary answers the JSON-RPC request itself. (Some(error), None) => { assert_eq!(error["code"], -32603, "{response}"); let problem = &error["data"]; @@ -603,31 +600,12 @@ async fn rename_preview_deadline_stops_the_worker_before_the_next_file() { assert_eq!(problem["tool"], "tracedecay_rename_preview"); assert_eq!(problem["retryable"], true); } - // The admitted read's own deadline won while the worker paused on - // the source read. (None, Some(result)) => { let problem = refusal_problem(result); - match problem["code"].as_str() { - Some("timed_out") => { - assert_eq!(problem["kind"], "timed_out", "{response}"); - assert_eq!(problem["cancellation_stage"], "during_read", "{response}"); - assert_eq!(problem["terminality"], "admitted_terminal", "{response}"); - } - // The outer dispatch ceiling won after admission. It is a - // retryable route refusal, which the graph-tool adapter - // reports as an unavailable pre-admission state under the - // ceiling's own reason code. - Some("tool_dispatch_deadline_exceeded") => { - assert_eq!(problem["kind"], "unavailable", "{response}"); - assert_eq!(problem["retryable"], true, "{response}"); - assert_eq!(problem["retry"], "after_delay", "{response}"); - assert_eq!(problem["legal_actions"], json!(["retry"]), "{response}"); - assert_eq!(problem["terminality"], "pre_admission", "{response}"); - assert_eq!(problem["owning_layer"], "application", "{response}"); - assert_eq!(problem["cancellation_stage"], Value::Null, "{response}"); - } - _ => panic!("expected one canonical deadline terminal: {response}"), - } + assert_eq!(problem["kind"], "timed_out", "{response}"); + assert_eq!(problem["code"], "timed_out", "{response}"); + assert_eq!(problem["cancellation_stage"], "during_read", "{response}"); + assert_eq!(problem["terminality"], "admitted_terminal", "{response}"); } _ => panic!("expected one canonical deadline terminal: {response}"), } From 870d2b5f0d22d84f7d4a79ae14f9abc7bb85debd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:38:51 -0800 Subject: [PATCH 79/85] fix(doctor): scope sealed census to projection identities --- .../src/doctor_kernel.rs | 45 ++++--- .../src/doctor_kernel/tests.rs | 118 ++++++++++++++++++ .../tracedecay-graph-db/src/sealed_store.rs | 86 +++++++++++-- .../sealed_store.rs | 2 +- 4 files changed, 228 insertions(+), 23 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 1faed0953a1..f7893b1ce01 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -42,6 +42,9 @@ use tracedecay_contracts::{ }; use tracedecay_domain::CodeGenerationId; use tracedecay_global_db::{GlobalDbNativeIntegrationStore, RegisteredGlobalDb}; +use tracedecay_graph_db::{ + GraphGenerationId, GraphNamespace, GraphProjectionId, GraphProjectionIdentity, +}; use crate::DaemonFeedbackRuntimeRegistrar; use tracedecay_maintenance::retention::footprint::{ @@ -947,35 +950,45 @@ pub async fn collect_code_generation_retention_findings( storage_family_read(vec![finding]) } -/// The journaled generation ids whose sealed artifacts are still live in the -/// project graph store: each projection's verified head, every publication +/// The journaled projection/generation identities whose sealed artifacts remain +/// live in the project graph store: each projection's verified head, every publication /// newer than its head (pending), and every generation an active replay /// depends on. A sealed directory outside this set is one the ordinary /// retirement pass reclaims; `None` when the journal cannot be read (an /// absent table on a store that never published, a lock, a corrupt row). async fn live_sealed_generations( graph: &tracedecay_runtime_core::db::Database, -) -> Option> { +) -> Option> { + let shard = serde_json::to_string(&graph.registered_binding().shard_id).ok()?; let mut rows = graph .read_connection() .query( - "SELECT replay.generation + "SELECT replay.namespace, replay.projection, replay.generation FROM graph_publication_replay_v1 AS replay LEFT JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id AND head.namespace = replay.namespace AND head.projection = replay.projection - WHERE head.replay_sequence IS NULL - OR replay.sequence >= head.replay_sequence + WHERE replay.shard_id = ?1 + AND (head.replay_sequence IS NULL + OR replay.sequence >= head.replay_sequence) UNION - SELECT generation FROM graph_publication_replay_dependencies_v1", - (), + SELECT namespace, projection, generation + FROM graph_publication_replay_dependencies_v1 + WHERE shard_id = ?1", + (shard,), ) .await .ok()?; let mut heads = BTreeSet::new(); while let Some(row) = rows.next().await.ok()? { - heads.insert(row.get::(0).ok()?); + heads.insert(( + GraphProjectionIdentity::new( + GraphNamespace::new(row.get::(0).ok()?).ok()?, + GraphProjectionId::new(row.get::(1).ok()?).ok()?, + ), + GraphGenerationId::new(row.get::(2).ok()?).ok()?, + )); } Some(heads) } @@ -986,21 +999,25 @@ async fn live_sealed_generations( /// that retirement has not yet reached. `None` when the journal cannot be /// read. async fn deferred_native_retirements(graph: &tracedecay_runtime_core::db::Database) -> Option { + let shard = serde_json::to_string(&graph.registered_binding().shard_id).ok()?; let mut rows = graph .read_connection() .query( - "SELECT (SELECT COUNT(*) FROM graph_publication_replay_tombstones_v1) + "SELECT (SELECT COUNT(*) FROM graph_publication_replay_tombstones_v1 + WHERE shard_id = ?1) + (SELECT COUNT(*) FROM graph_publication_replay_v1 AS replay JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id AND head.namespace = replay.namespace AND head.projection = replay.projection - WHERE replay.sequence < head.replay_sequence - AND replay.generation NOT IN ( - SELECT generation FROM graph_publication_replay_dependencies_v1 + WHERE replay.shard_id = ?1 + AND replay.sequence < head.replay_sequence + AND NOT EXISTS ( + SELECT 1 FROM graph_publication_replay_dependencies_v1 AS dependency + WHERE dependency.dependency_replay_sequence = replay.sequence ))", - (), + (shard,), ) .await .ok()?; diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs index b25e2bf0df4..c340b696dd1 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs @@ -15,9 +15,127 @@ use tracedecay_contracts::doctor::{ ObservabilityReadV1, ObservabilityStateV1, host_integration_finding, }; use tracedecay_graph_db::GraphDbCorruptionCensusV1; +use tracedecay_runtime_core::db::{Database, DatabaseAuthority, TestDatabaseRuntimeMode}; +use tracedecay_rusqlite_runtime::repository::GRAPH_PUBLICATION_SCHEMA_V1; use super::*; +#[tokio::test] +async fn sealed_journal_liveness_and_retirement_preserve_projection_identity() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("graph.db"); + let authority = + DatabaseAuthority::acquire_test(&path, "doctor sealed journal fixture").unwrap(); + let (database, _) = + Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Initialize) + .await + .unwrap(); + database + .execute_write_batch( + "install graph publication schema", + GRAPH_PUBLICATION_SCHEMA_V1, + ) + .await + .unwrap(); + let shard = serde_json::to_string(&database.registered_binding().shard_id).unwrap(); + let mut foreign = database.registered_binding().shard_id.clone(); + foreign.profile_id = tracedecay_domain::UserProfileId::new("profile.foreign").unwrap(); + let foreign = serde_json::to_string(&foreign).unwrap(); + let digest = format!("sha256:{}", "0".repeat(64)); + for (sequence, owner, namespace, projection, generation) in [ + (1_i64, &shard, "project", "code", "generation:shared"), + (2, &shard, "project", "sessions", "generation:shared"), + (3, &shard, "project", "sessions", "generation:current"), + (4, &shard, "other-project", "code", "generation:shared"), + (5, &shard, "other-project", "code", "generation:current"), + (6, &foreign, "project", "sessions", "generation:shared"), + ] { + database + .execute_write( + "publish graph journal fixture", + "INSERT INTO graph_publication_replay_v1 + (sequence, shard_id, namespace, projection, generation, idempotency_key, + input_digest, dependency_generation_closure_digest, direct_dependency_bytes, + expected_recovered_digest, canonical_replay_source_digest, canonical_replay_source) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7, 2, ?7, ?7, X'7b7d')", + (sequence, owner.as_str(), namespace, projection, generation, + format!("publication:{sequence}"), digest.as_str()), + ) + .await + .unwrap(); + } + for sequence in [1_i64, 3, 5, 6] { + database + .execute_write( + "install graph journal head", + "INSERT INTO graph_verified_heads_v1 + (shard_id, namespace, projection, replay_sequence, recovered_digest) + SELECT shard_id, namespace, projection, sequence, expected_recovered_digest + FROM graph_publication_replay_v1 WHERE sequence = ?1", + (sequence,), + ) + .await + .unwrap(); + } + database + .execute_write( + "retain exact graph dependency", + "INSERT INTO graph_publication_replay_dependencies_v1 + (owner_replay_sequence, ordinal, dependency_replay_sequence, + shard_id, namespace, projection, generation) + SELECT 3, 0, sequence, shard_id, namespace, projection, generation + FROM graph_publication_replay_v1 WHERE sequence = 1", + (), + ) + .await + .unwrap(); + let identity = |namespace: &str, projection: &str, generation: &str| { + ( + GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new(projection).unwrap(), + ), + GraphGenerationId::new(generation).unwrap(), + ) + }; + let mut expected = BTreeSet::from([ + identity("project", "code", "generation:shared"), + identity("project", "sessions", "generation:current"), + identity("other-project", "code", "generation:current"), + ]); + assert_eq!( + live_sealed_generations(&database).await, + Some(expected.clone()) + ); + assert_eq!(deferred_native_retirements(&database).await, Some(2)); + + database + .execute_write( + "retain one superseded projection as a dependency", + "INSERT INTO graph_publication_replay_dependencies_v1 + (owner_replay_sequence, ordinal, dependency_replay_sequence, + shard_id, namespace, projection, generation) + SELECT 3, 1, sequence, shard_id, namespace, projection, generation + FROM graph_publication_replay_v1 WHERE sequence = 4", + (), + ) + .await + .unwrap(); + expected.insert(identity("other-project", "code", "generation:shared")); + assert_eq!(live_sealed_generations(&database).await, Some(expected)); + assert_eq!(deferred_native_retirements(&database).await, Some(1)); + assert_eq!( + database + .query_scalar_i64( + "preserved replays", + "SELECT COUNT(*) FROM graph_publication_replay_v1" + ) + .await + .unwrap(), + 6, + ); +} + #[test] fn configuration_read_from_pin_absent_on_cold_cache() { let missing: Result = diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..39fc95f638c 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -78,10 +78,10 @@ use crate::state::{ }; use crate::{ GraphCommit, GraphDb, GraphDbError, GraphDbLocation, GraphDbOpenOptions, GraphDurability, - GraphEntity, GraphFormatVersion, GraphGenerationManifest, GraphGenerationManifestIdentity, - GraphGenerationRowSpill, GraphGenerationRows, GraphNamespace, GraphProjectionId, - GraphProjectionIdentity, GraphRelation, GraphWatermark, GraphWriteBatch, NeverCancelled, - SourceGeneration, SpilledGraphGeneration, + GraphEntity, GraphFormatVersion, GraphGenerationId, GraphGenerationManifest, + GraphGenerationManifestIdentity, GraphGenerationRowSpill, GraphGenerationRows, GraphNamespace, + GraphProjectionId, GraphProjectionIdentity, GraphRelation, GraphWatermark, GraphWriteBatch, + NeverCancelled, SourceGeneration, SpilledGraphGeneration, }; /// A sealed generation opened without the shared staging database. @@ -529,15 +529,15 @@ pub struct SealedStoreCensusV1 { pub unrecognized_count: u64, } -/// Measure the sealed root beside `database_path` against the generation ids -/// of the store's current verified heads (as journaled: `:`). +/// Measure the sealed root beside `database_path` against the exact projection +/// and generation identities that remain live in that store's journal. /// /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the /// census stays truthful when a receipt is mid-write or corrupt. pub fn census_sealed_store( database_path: &Path, - head_generations: &std::collections::BTreeSet, + head_generations: &std::collections::BTreeSet<(GraphProjectionIdentity, GraphGenerationId)>, ) -> std::io::Result { let root = sealed_store_root(database_path); let entries = match std::fs::read_dir(&root) { @@ -573,7 +573,18 @@ pub fn census_sealed_store( census.unrecognized_count += 1; continue; }; - if head_generations.contains(&receipt.generation) { + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace), + GraphProjectionId::new(receipt.projection), + GraphGenerationId::new(receipt.generation), + ) else { + census.unrecognized_count += 1; + continue; + }; + if head_generations.contains(&( + GraphProjectionIdentity::new(namespace, projection), + generation, + )) { census.head_count += 1; census.head_bytes += bytes; } else { @@ -2809,6 +2820,65 @@ mod build_tests { .unwrap() } + #[test] + fn sealed_census_scopes_equal_generation_names_to_their_projection() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let mut heads = BTreeSet::new(); + let mut artifacts = Vec::new(); + let mut expected_head_bytes = 0; + let mut expected_superseded_bytes = 0; + for (namespace, projection, generation, live) in [ + ("project", "code", "generation:shared", true), + ("project", "sessions", "generation:shared", false), + ("project", "sessions", "generation:current", true), + ("other-project", "code", "generation:shared", false), + ("other-project", "code", "generation:current", true), + ] { + let mut manifest = manifest(2, 0); + manifest.projection = GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new(projection).unwrap(), + ); + manifest.generation = GraphGenerationId::new(generation).unwrap(); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + let directory = store.directory.clone(); + drop(store); + if live { + heads.insert((identity.projection, identity.generation)); + } + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + let bytes = std::fs::read(&path).unwrap(); + if live { + expected_head_bytes += bytes.len() as u64; + } else { + expected_superseded_bytes += bytes.len() as u64; + } + artifacts.push((path, bytes)); + } + } + + let census = census_sealed_store(&database_path, &heads).unwrap(); + assert_eq!(census.head_count, 3); + assert_eq!(census.superseded_count, 2); + assert_eq!(census.unrecognized_count, 0); + assert_eq!(census.head_bytes, expected_head_bytes); + assert_eq!(census.superseded_bytes, expected_superseded_bytes); + for (path, bytes) in artifacts { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + } + fn open_source(database_path: &std::path::Path) -> crate::GraphDbLeaseV1 { let owner = GraphDbOwner::open(GraphDbOpenOptions { location: GraphDbLocation::Persistent(database_path.to_path_buf()), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 3bb165ee0f6..7906c8c836f 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1354,7 +1354,7 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { std::fs::create_dir_all(&unreadable).unwrap(); std::fs::write(unreadable.join("generation.grafeo"), b"no receipt").unwrap(); - let heads = std::collections::BTreeSet::from(["census-g2".to_owned()]); + let heads = BTreeSet::from([(identity, GraphGenerationId::new("census-g2").unwrap())]); let census = tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) .unwrap(); From b935a0595ff506abb1ff6c3b59d97496941b95a9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:49:12 -0800 Subject: [PATCH 80/85] test(graph-db): import census_sealed_store in build tests The projection-scoped census regression in `build_tests` calls `census_sealed_store` but the module's `use super::{...}` list did not import it, so the graph-db unit test binary failed to compile (E0425) under Clippy and the Linux fleet. Co-Authored-By: Claude Opus 5.5 --- crates/tracedecay-graph-db/src/sealed_store.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index f84e3d8afed..5be0cb4f394 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2896,8 +2896,9 @@ mod build_tests { use super::{ NativeCompactEncodeBoundaryV1, SEALED_STORE_DATABASE_FILE, SealedCompactRows, SealedRowSource, build_or_open_sealed_store, build_or_open_sealed_store_with_work, - build_sealed_container_with_work, push_manifest_rows, sealed_artifact_database_options, - sealed_copy_proof, sealed_generation_directory, sealed_store_root, + build_sealed_container_with_work, census_sealed_store, push_manifest_rows, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, }; use tracedecay_store::runtime::{GraphNativeWorkKindV1, GraphNativeWorkV1}; From 5d9410150f6baeec186d3760cb2e0b3a150a16d9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:12:49 -0800 Subject: [PATCH 81/85] test(doctor): initialize isolated sealed journal fixture Register the canonical global/session schema installer before publishing the isolated profile runtime. The projection-identity regression now executes independently instead of relying on test order; no production admission or assertion is relaxed. --- crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs index c340b696dd1..e407ca50581 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs @@ -22,6 +22,7 @@ use super::*; #[tokio::test] async fn sealed_journal_liveness_and_retirement_preserve_projection_identity() { + tracedecay_global_db::register_registered_schema_installer(); let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("graph.db"); let authority = From 5aa879f3569b359dbb744551c5396a9265c49195 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:40:50 -0800 Subject: [PATCH 82/85] test(storage): repair batched and native-clock fixtures Seed the same 10,001 reconciliation rows in one canonical transaction. Make the Codex unsettled-clock regression independent of scheduler delay. --- .../src/observation_projection/rebuild.rs | 14 +++---- .../src/runtime/hosts/codex.rs | 10 ++++- .../src/runtime/hosts/codex/tests.rs | 41 ++++++++++++++----- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index beb861a9d9a..af96c31baf2 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2693,18 +2693,16 @@ mod activation_tests { let harness = RegisteredGlobalDbHarness::open("session-overlap-paging").await; let mut staged_rows = Vec::with_capacity(OVERLAPS_PAST_TRANSPORT_ROW_CAP as usize); for index in 0..OVERLAPS_PAST_TRANSPORT_ROW_CAP { - let session_id = format!("session.{index:05}"); - let active = SessionRecord { - session_id: session_id.clone(), - ..session(None, None) - }; - assert!(harness.registered.upsert_session(&active).await); staged_rows.push(SessionRecord { - session_id, - title: Some(format!("composer {index:05}")), + session_id: format!("session.{index:05}"), ..session(None, None) }); } + // Fixture setup needs one durable batch, not a commit per overlap. + assert!(harness.registered.upsert_sessions(&staged_rows).await); + for (index, staged) in staged_rows.iter_mut().enumerate() { + staged.title = Some(format!("composer {index:05}")); + } let transaction = harness.registered.begin_write_transaction().await.unwrap(); transaction .execute( diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 2ab2ff738ac..6dffe68dc47 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2578,7 +2578,15 @@ fn codex_directory_changed( path: &Path, stored: Option, ) -> TranscriptIngestResult { - let Some(current) = directory_stat_witness(path)? else { + codex_directory_changed_since(path, directory_stat_witness(path)?, stored) +} + +fn codex_directory_changed_since( + path: &Path, + current: Option, + stored: Option, +) -> TranscriptIngestResult { + let Some(current) = current else { return Ok(stored.is_some()); }; let Some(stored) = stored else { diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 771f91636db..0e48723c8bf 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -1237,6 +1237,8 @@ mod recent_first_discovery_tests { directory_entry_fingerprint, directory_stat_witness, indexed_replay_pass, replay_index_entries_visited_for_test, reset_replay_index_entries_visited_for_test, }; + #[cfg(unix)] + use crate::runtime::hosts::codex::{DirectoryStatWitness, codex_directory_changed_since}; use crate::runtime::ingest::quarantine::SourceSweep; use crate::runtime::jsonl_observation_admission::{ SHARED_JSONL_PIN_GATES, install_test_shared_jsonl_preparation_authority, @@ -2716,23 +2718,40 @@ mod recent_first_discovery_tests { std::fs::create_dir_all(&bucket).unwrap(); std::fs::write(bucket.join("rollout-existing.jsonl"), b"{}\n").unwrap(); - // A recently changed bucket uses an entry-name witness when the - // clock is still coarse. Model a different previously observed stat - // (such as a directory metadata update) with identical entry names. + // Supply the native coarse-clock sample directly: scheduler delay + // after the write must not decide which clock branch this exercises. + let metadata = std::fs::metadata(&bucket).unwrap(); + let stat = + codex_directory_stat_identity(&bucket, &metadata, ChangeStamp::Unvouched(0)).unwrap(); let mut stored = codex_directory_witness(&bucket).unwrap().unwrap(); + stored.stat = stat; + stored.entries = Some(directory_entry_fingerprint(&bucket).unwrap()); assert!( - !directory_stat_witness(&bucket).unwrap().unwrap().settled, - "this regression requires the unsettled-clock branch" - ); - assert!(stored.entries.is_some()); - assert_eq!( - stored.entries, - Some(directory_entry_fingerprint(&bucket).unwrap()) + !codex_directory_changed_since( + &bucket, + Some(DirectoryStatWitness { + stat, + settled: false + }), + Some(stored), + ) + .unwrap(), + "an unchanged unsettled directory retains its name witness" ); + // Model a different previously observed stat (such as a directory + // metadata update) while the eligible entry names remain identical. stored.stat[0] ^= 1; assert!( - codex_directory_changed(&bucket, Some(stored)).unwrap(), + codex_directory_changed_since( + &bucket, + Some(DirectoryStatWitness { + stat, + settled: false + }), + Some(stored), + ) + .unwrap(), "a changed native identity must invalidate even when the entry-name fingerprint is unchanged" ); } From 5a2b67cd7aa1c6a4f23e59849fbfd3c25dd15fac Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:24:18 -0800 Subject: [PATCH 83/85] test(hooks): await Hermes transcript publication --- .../tests/core_cli_suite/tool_daemon_test.rs | 87 +++++++++++++------ 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index 429644ba3c3..2712a67e197 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -60,6 +60,41 @@ fn run_command_with_timeout(command: Command, timeout: Duration) -> Output { common::output_with_timeout(command, timeout) } +/// Transcript admission and its temporal projection publish separately. The +/// fixture follows only the typed same-request publication retry; its stable +/// message identities make a replay preserve the already-admitted transcript. +fn ingest_transcript_until_published(command: &mut Command) -> Output { + let deadline = Instant::now() + CLI_ROUNDTRIP_TIMEOUT; + loop { + let output = command.output().expect("transcript ingest should run"); + let response = serde_json::from_slice::(&output.stdout).ok(); + let problem = response + .as_ref() + .map(|value| &value["structuredContent"]["problem"]); + let publication_pending = problem.is_some_and(|problem| { + problem["code"] == "temporal_refresh_unavailable" + && problem["retryable"] == true + && problem["terminality"] == "pre_admission" + && problem["retry_scope"] == "same_request" + && problem["retry"] == "after_delay" + }); + if output.status.success() || !publication_pending { + return output; + } + let delay = Duration::from_millis( + problem.unwrap()["retry_after_millis"] + .as_u64() + .expect("publication retry must name its delay"), + ); + assert!( + Instant::now() + delay < deadline, + "transcript projection did not publish before the fixture deadline: {}", + String::from_utf8_lossy(&output.stdout) + ); + std::thread::sleep(delay); + } +} + enum FakeDaemonResponse { Complete { text: String }, HoldOpen, @@ -3676,18 +3711,18 @@ fn hermes_read_only_preflight_keeps_project_lcm_grep_available() { ], }) .to_string(); - let user_output = tracedecay_command_with_home(&home_path) - .current_dir(std::path::Path::new("/")) - .env("TRACEDECAY_DAEMON_SOCKET", &socket) - .args([ - "tool", - "tracedecay_hook_runtime", - "--json", - "--args", - user_args.as_str(), - ]) - .output() - .expect("user-scoped transcript ingest should run"); + let user_output = ingest_transcript_until_published( + tracedecay_command_with_home(&home_path) + .current_dir(std::path::Path::new("/")) + .env("TRACEDECAY_DAEMON_SOCKET", &socket) + .args([ + "tool", + "tracedecay_hook_runtime", + "--json", + "--args", + user_args.as_str(), + ]), + ); assert!( user_output.status.success(), "user-scoped Hermes transcript ingest must succeed projectless\nstdout:\n{}\nstderr:\n{}", @@ -3718,20 +3753,20 @@ fn hermes_read_only_preflight_keeps_project_lcm_grep_available() { ], }) .to_string(); - let project_output = tracedecay_command_with_home(&home_path) - .current_dir(&project_path) - .env("TRACEDECAY_DAEMON_SOCKET", &socket) - .args([ - "tool", - "--project", - &project_arg, - "tracedecay_hook_runtime", - "--json", - "--args", - project_args.as_str(), - ]) - .output() - .expect("project-scoped transcript ingest should run"); + let project_output = ingest_transcript_until_published( + tracedecay_command_with_home(&home_path) + .current_dir(&project_path) + .env("TRACEDECAY_DAEMON_SOCKET", &socket) + .args([ + "tool", + "--project", + &project_arg, + "tracedecay_hook_runtime", + "--json", + "--args", + project_args.as_str(), + ]), + ); assert!( project_output.status.success(), "project-scoped Hermes transcript ingest must succeed\nstdout:\n{}\nstderr:\n{}", From a677d29b056bdb59d4e8cdafb6f35d345e231667 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:34:49 -0800 Subject: [PATCH 84/85] fix(runtime): finish settled owners after their runtime stops --- .../src/operation_task_owner.rs | 67 ++++++++++++++----- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-runtime-core/src/operation_task_owner.rs b/crates/tracedecay-runtime-core/src/operation_task_owner.rs index 23ae5090f23..df63c41ba29 100644 --- a/crates/tracedecay-runtime-core/src/operation_task_owner.rs +++ b/crates/tracedecay-runtime-core/src/operation_task_owner.rs @@ -189,23 +189,31 @@ impl RuntimeOperationTaskOwnerV1 { state.lifecycle = RuntimeOperationTaskLifecycleV1::Fenced; let (publish_completion, completion) = watch::channel(None); let tasks = std::mem::take(&mut state.tasks); - self.runtime.spawn(async move { - let mut failures = Vec::new(); - for (task_id, task) in tasks { - if let Err(error) = task.await { - failures.push(RuntimeOperationTaskJoinFailure { - task_id, - error: Arc::new(error), - }); + if tasks.is_empty() { + // Completed owners need no runtime to drain. Cleanup can + // arrive after their admission runtime has stopped; an + // empty reaper spawned there would be cancelled without + // publishing the already-settled result. + publish_completion.send_replace(Some(Ok(()))); + } else { + self.runtime.spawn(async move { + let mut failures = Vec::new(); + for (task_id, task) in tasks { + if let Err(error) = task.await { + failures.push(RuntimeOperationTaskJoinFailure { + task_id, + error: Arc::new(error), + }); + } } - } - let result = if failures.is_empty() { - Ok(()) - } else { - Err(RuntimeOperationShutdownError::JoinFailures(failures)) - }; - publish_completion.send_replace(Some(result)); - }); + let result = if failures.is_empty() { + Ok(()) + } else { + Err(RuntimeOperationShutdownError::JoinFailures(failures)) + }; + publish_completion.send_replace(Some(result)); + }); + } state.lifecycle = RuntimeOperationTaskLifecycleV1::Draining { completion: completion.clone(), }; @@ -248,6 +256,33 @@ mod tests { use std::sync::atomic::AtomicBool; use tokio::sync::oneshot; + #[test] + fn settled_owner_shuts_down_after_its_admission_runtime_stops() { + let admission_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let owner = RuntimeOperationTaskOwnerV1::new(admission_runtime.handle().clone()); + admission_runtime.block_on(async { + owner.retain(|_| async {}).unwrap(); + owner.settled().await; + }); + drop(admission_runtime); + + let cleanup_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + cleanup_runtime.block_on(async { + owner.shutdown().await.unwrap(); + owner.shutdown().await.unwrap(); + assert!(matches!( + owner.retain(|_| async {}), + Err(RuntimeOperationAdmissionError::Closed) + )); + }); + } + #[tokio::test] async fn accepted_settlement_survives_request_and_shutdown_waiter_cancellation() { let owner = Arc::new(RuntimeOperationTaskOwnerV1::new(Handle::current())); From 1b24d78a5bf34f13f5cd6db0e585dd5b09428436 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:37:31 -0800 Subject: [PATCH 85/85] test(db): reuse published schema for cancellation sweep --- .../src/tests/schema_install_cancellation.rs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 64b90ebeb1e..46e055a7a75 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -96,7 +96,7 @@ impl RegisteredStore { /// Inspect a completed family with an ordinary independent read-only SQLite /// connection. Reopening the registered runtime for every observation would /// perform another full attachment lifecycle and obscure cancellation timing. - fn external_schema_objects(&self) -> Vec { + fn external_schema_objects(&self) -> Vec<(String, Option)> { let connection = rusqlite::Connection::open_with_flags( &self.path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, @@ -104,12 +104,12 @@ impl RegisteredStore { .unwrap(); let mut statement = connection .prepare( - "SELECT type || ':' || name FROM sqlite_schema + "SELECT type || ':' || name, sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", ) .unwrap(); statement - .query_map([], |row| row.get::<_, String>(0)) + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() .map(Result::unwrap) .collect() @@ -185,12 +185,14 @@ async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { #[tokio::test(flavor = "multi_thread")] async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { - let reference = RegisteredStore::create(); - let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; + let store = RegisteredStore::create(); + let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); - let installed = reference.external_schema_objects(); + let installed = store.external_schema_objects(); assert!( - installed.contains(&"table:code_projects".to_owned()), + installed + .iter() + .any(|(object, _)| object == "table:code_projects"), "a completed attach installs the registered schema: {installed:?}" ); @@ -201,12 +203,11 @@ async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { .chain((2..total_polls).step_by(stride)) .chain([total_polls + 10]) { - let store = RegisteredStore::create(); - // Cancellation cannot modify the already-published schema or - // prevent the next admission from reopening the same family. - let (owner, runtime, retirement) = store.publish().await.into_parts(); - drop((owner, runtime)); - retire_runtime(retirement).await; + // The reference admission, then each successful reopen below, leave + // this family closed with the exact baseline schema. Creating another + // family would repeat initialization without exercising another + // cancellation point. Keep real runtime retirement and reopening + // around every sampled admission. let before_admission = store.external_schema_objects(); assert_eq!(before_admission, installed); let (outcome, _) = store.attach_cancelled_at(cancel_at).await;