diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs index 429644ba3c3..2712a67e197 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_daemon_test.rs @@ -60,6 +60,41 @@ fn run_command_with_timeout(command: Command, timeout: Duration) -> Output { common::output_with_timeout(command, timeout) } +/// Transcript admission and its temporal projection publish separately. The +/// fixture follows only the typed same-request publication retry; its stable +/// message identities make a replay preserve the already-admitted transcript. +fn ingest_transcript_until_published(command: &mut Command) -> Output { + let deadline = Instant::now() + CLI_ROUNDTRIP_TIMEOUT; + loop { + let output = command.output().expect("transcript ingest should run"); + let response = serde_json::from_slice::(&output.stdout).ok(); + let problem = response + .as_ref() + .map(|value| &value["structuredContent"]["problem"]); + let publication_pending = problem.is_some_and(|problem| { + problem["code"] == "temporal_refresh_unavailable" + && problem["retryable"] == true + && problem["terminality"] == "pre_admission" + && problem["retry_scope"] == "same_request" + && problem["retry"] == "after_delay" + }); + if output.status.success() || !publication_pending { + return output; + } + let delay = Duration::from_millis( + problem.unwrap()["retry_after_millis"] + .as_u64() + .expect("publication retry must name its delay"), + ); + assert!( + Instant::now() + delay < deadline, + "transcript projection did not publish before the fixture deadline: {}", + String::from_utf8_lossy(&output.stdout) + ); + std::thread::sleep(delay); + } +} + enum FakeDaemonResponse { Complete { text: String }, HoldOpen, @@ -3676,18 +3711,18 @@ fn hermes_read_only_preflight_keeps_project_lcm_grep_available() { ], }) .to_string(); - let user_output = tracedecay_command_with_home(&home_path) - .current_dir(std::path::Path::new("/")) - .env("TRACEDECAY_DAEMON_SOCKET", &socket) - .args([ - "tool", - "tracedecay_hook_runtime", - "--json", - "--args", - user_args.as_str(), - ]) - .output() - .expect("user-scoped transcript ingest should run"); + let user_output = ingest_transcript_until_published( + tracedecay_command_with_home(&home_path) + .current_dir(std::path::Path::new("/")) + .env("TRACEDECAY_DAEMON_SOCKET", &socket) + .args([ + "tool", + "tracedecay_hook_runtime", + "--json", + "--args", + user_args.as_str(), + ]), + ); assert!( user_output.status.success(), "user-scoped Hermes transcript ingest must succeed projectless\nstdout:\n{}\nstderr:\n{}", @@ -3718,20 +3753,20 @@ fn hermes_read_only_preflight_keeps_project_lcm_grep_available() { ], }) .to_string(); - let project_output = tracedecay_command_with_home(&home_path) - .current_dir(&project_path) - .env("TRACEDECAY_DAEMON_SOCKET", &socket) - .args([ - "tool", - "--project", - &project_arg, - "tracedecay_hook_runtime", - "--json", - "--args", - project_args.as_str(), - ]) - .output() - .expect("project-scoped transcript ingest should run"); + let project_output = ingest_transcript_until_published( + tracedecay_command_with_home(&home_path) + .current_dir(&project_path) + .env("TRACEDECAY_DAEMON_SOCKET", &socket) + .args([ + "tool", + "--project", + &project_arg, + "tracedecay_hook_runtime", + "--json", + "--args", + project_args.as_str(), + ]), + ); assert!( project_output.status.success(), "project-scoped Hermes transcript ingest must succeed\nstdout:\n{}\nstderr:\n{}", diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 167442e0a0f..f7893b1ce01 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -8,7 +8,7 @@ //! `Unsupported`/`Absent`/`Denied`/`Unknown` read, never a fabricated healthy //! result, and partial coverage carries its real reason. //! -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; use tracedecay_application::advisory::github_runtime::{ @@ -42,8 +42,14 @@ use tracedecay_contracts::{ }; use tracedecay_domain::CodeGenerationId; use tracedecay_global_db::{GlobalDbNativeIntegrationStore, RegisteredGlobalDb}; +use tracedecay_graph_db::{ + GraphGenerationId, GraphNamespace, GraphProjectionId, GraphProjectionIdentity, +}; use crate::DaemonFeedbackRuntimeRegistrar; +use tracedecay_maintenance::retention::footprint::{ + ProfileFootprintV1, ProjectRootStateV1, census_profile_footprint, census_sqlite_temp_spill, +}; use tracedecay_maintenance::telemetry::GuardedStoreTelemetryPort; use tracedecay_session_temporal_store::SessionTemporalAccess; @@ -614,6 +620,30 @@ fn soft_budget_findings( storage_family_read(findings) } +/// A failed registry listing must not hide the physical profile footprint. +/// An unknown registry is not an empty registry: orphan classifications require +/// proven registration state, but file sizes and their bounds do not. +fn profile_footprint_from_registered_roots( + profile_root: &Path, + registered_roots: Option<&BTreeMap>, + overrides: &BTreeMap, +) -> ProfileFootprintV1 { + let empty_roots = BTreeMap::new(); + let mut footprint = census_profile_footprint( + profile_root, + registered_roots.unwrap_or(&empty_roots), + overrides, + ); + if registered_roots.is_none() { + for entry in &mut footprint.entries { + if let Some(project) = entry.project.as_mut() { + project.state = ProjectRootStateV1::Unverifiable; + } + } + } + footprint +} + /// Size every store family under the profile against its bound. Registered /// project roots name each project store's root and whether it still exists. #[tracing::instrument(name = "daemon.doctor.profile_footprint", level = "trace", skip_all)] @@ -621,21 +651,22 @@ async fn profile_footprint_read( registry: &RegisteredGlobalDb, profile_root: &Path, retention: &tracedecay_configuration::RetentionConfig, -) -> Option { +) -> Option { let registered = registry .list_code_projects(usize::MAX) .await - .ok()? - .into_iter() - .map(|project| (project.project_id, project.canonical_root)) - .collect::>(); + .ok() + .map(|projects| { + projects + .into_iter() + .map(|project| (project.project_id, project.canonical_root)) + .collect::>() + }); let overrides = retention.store_soft_budgets_bytes.clone(); let profile_root = profile_root.to_path_buf(); tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - use tracedecay_maintenance::retention::footprint::{ - census_profile_footprint, census_sqlite_temp_spill, - }; - let mut footprint = census_profile_footprint(&profile_root, ®istered, &overrides); + let mut footprint = + profile_footprint_from_registered_roots(&profile_root, registered.as_ref(), &overrides); footprint.entries.extend(census_sqlite_temp_spill( Path::new("/proc/self/fd"), &overrides, @@ -646,6 +677,108 @@ async fn profile_footprint_read( .ok() } +#[cfg(test)] +mod profile_footprint_tests { + use std::collections::BTreeMap; + use tracedecay_contracts::doctor::{DoctorEvidenceStateV1, DoctorStorageFamilyReadV1}; + use tracedecay_maintenance::retention::footprint::{ + ProfileFamilyV1, ProjectRootStateV1, footprint_findings, + }; + + use super::profile_footprint_from_registered_roots; + + fn state_for_store( + profile: &tracedecay_maintenance::retention::footprint::ProfileFootprintV1, + ) -> ProjectRootStateV1 { + profile + .entries + .iter() + .find(|entry| entry.key == "projects/proj_growing") + .and_then(|entry| entry.project.as_ref()) + .expect("measured project store") + .state + } + + #[test] + fn unreadable_registry_cannot_hide_large_profile_or_mislabel_orphans() { + let temp = tempfile::tempdir().unwrap(); + let profile_root = temp.path().join("profile"); + let store_root = profile_root.join("projects/proj_growing"); + std::fs::create_dir_all(&store_root).unwrap(); + std::fs::create_dir_all(profile_root.join("lcm-payloads")).unwrap(); + std::fs::write(store_root.join("tracedecay.db-wal"), vec![0; 4096]).unwrap(); + std::fs::write(profile_root.join("lcm-payloads/blob"), vec![0; 2048]).unwrap(); + + let overrides = BTreeMap::from([ + ("projects/proj_growing".to_owned(), 1024), + ("profile".to_owned(), 1024), + ]); + // None is a failed registry read, not a successful empty listing. + let unknown = profile_footprint_from_registered_roots(&profile_root, None, &overrides); + assert_eq!(unknown.total_bytes, 6144); + assert_eq!(state_for_store(&unknown), ProjectRootStateV1::Unverifiable); + let project = unknown + .entries + .iter() + .find(|entry| entry.key == "projects/proj_growing") + .unwrap(); + assert_eq!(project.family, ProfileFamilyV1::ProjectStore); + assert_eq!(project.bytes, 4096); + assert_eq!(project.bound_bytes, 1024); + assert_eq!(project.project.as_ref().unwrap().registered_root, None); + + let DoctorStorageFamilyReadV1::Observed { findings } = footprint_findings(&unknown) else { + panic!("physical footprint must remain observable without a registry"); + }; + let named = findings + .into_iter() + .map(|finding| { + ( + finding.finding().evidence()[0] + .reference() + .as_str() + .to_owned(), + finding.finding().state(), + ) + }) + .collect::>(); + assert_eq!( + named.get("projects/proj_growing"), + Some(&DoctorEvidenceStateV1::Degraded) + ); + assert_eq!(named.get("profile"), Some(&DoctorEvidenceStateV1::Degraded)); + + // Distinguish failed enumeration from an authoritative empty registry, + // and from a registered root whose filesystem directory was deleted. + let empty = BTreeMap::new(); + let unregistered = + profile_footprint_from_registered_roots(&profile_root, Some(&empty), &overrides); + assert_eq!( + state_for_store(&unregistered), + ProjectRootStateV1::Unregistered + ); + let deleted_root = temp.path().join("deleted-worktree"); + let registered = BTreeMap::from([( + "proj_growing".to_owned(), + deleted_root.display().to_string(), + )]); + let missing = + profile_footprint_from_registered_roots(&profile_root, Some(®istered), &overrides); + assert_eq!(state_for_store(&missing), ProjectRootStateV1::Missing); + assert_eq!( + missing + .entries + .iter() + .find(|entry| entry.key == "projects/proj_growing") + .and_then(|entry| entry.project.as_ref()) + .and_then(|project| project.registered_root.as_ref()) + .map(String::as_str), + deleted_root.to_str() + ); + assert!(!deleted_root.exists()); + } +} + /// The generations the retention pass protects: the one the mounted scheduler /// serves, through either the sealed seat or the text projection, and every /// live native-preview candidate binding of the repository. This is the set @@ -817,35 +950,45 @@ pub async fn collect_code_generation_retention_findings( storage_family_read(vec![finding]) } -/// The journaled generation ids whose sealed artifacts are still live in the -/// project graph store: each projection's verified head, every publication +/// The journaled projection/generation identities whose sealed artifacts remain +/// live in the project graph store: each projection's verified head, every publication /// newer than its head (pending), and every generation an active replay /// depends on. A sealed directory outside this set is one the ordinary /// retirement pass reclaims; `None` when the journal cannot be read (an /// absent table on a store that never published, a lock, a corrupt row). async fn live_sealed_generations( graph: &tracedecay_runtime_core::db::Database, -) -> Option> { +) -> Option> { + let shard = serde_json::to_string(&graph.registered_binding().shard_id).ok()?; let mut rows = graph .read_connection() .query( - "SELECT replay.generation + "SELECT replay.namespace, replay.projection, replay.generation FROM graph_publication_replay_v1 AS replay LEFT JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id AND head.namespace = replay.namespace AND head.projection = replay.projection - WHERE head.replay_sequence IS NULL - OR replay.sequence >= head.replay_sequence + WHERE replay.shard_id = ?1 + AND (head.replay_sequence IS NULL + OR replay.sequence >= head.replay_sequence) UNION - SELECT generation FROM graph_publication_replay_dependencies_v1", - (), + SELECT namespace, projection, generation + FROM graph_publication_replay_dependencies_v1 + WHERE shard_id = ?1", + (shard,), ) .await .ok()?; let mut heads = BTreeSet::new(); while let Some(row) = rows.next().await.ok()? { - heads.insert(row.get::(0).ok()?); + heads.insert(( + GraphProjectionIdentity::new( + GraphNamespace::new(row.get::(0).ok()?).ok()?, + GraphProjectionId::new(row.get::(1).ok()?).ok()?, + ), + GraphGenerationId::new(row.get::(2).ok()?).ok()?, + )); } Some(heads) } @@ -856,21 +999,25 @@ async fn live_sealed_generations( /// that retirement has not yet reached. `None` when the journal cannot be /// read. async fn deferred_native_retirements(graph: &tracedecay_runtime_core::db::Database) -> Option { + let shard = serde_json::to_string(&graph.registered_binding().shard_id).ok()?; let mut rows = graph .read_connection() .query( - "SELECT (SELECT COUNT(*) FROM graph_publication_replay_tombstones_v1) + "SELECT (SELECT COUNT(*) FROM graph_publication_replay_tombstones_v1 + WHERE shard_id = ?1) + (SELECT COUNT(*) FROM graph_publication_replay_v1 AS replay JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id AND head.namespace = replay.namespace AND head.projection = replay.projection - WHERE replay.sequence < head.replay_sequence - AND replay.generation NOT IN ( - SELECT generation FROM graph_publication_replay_dependencies_v1 + WHERE replay.shard_id = ?1 + AND replay.sequence < head.replay_sequence + AND NOT EXISTS ( + SELECT 1 FROM graph_publication_replay_dependencies_v1 AS dependency + WHERE dependency.dependency_replay_sequence = replay.sequence ))", - (), + (shard,), ) .await .ok()?; diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs index b25e2bf0df4..e407ca50581 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs @@ -15,9 +15,128 @@ use tracedecay_contracts::doctor::{ ObservabilityReadV1, ObservabilityStateV1, host_integration_finding, }; use tracedecay_graph_db::GraphDbCorruptionCensusV1; +use tracedecay_runtime_core::db::{Database, DatabaseAuthority, TestDatabaseRuntimeMode}; +use tracedecay_rusqlite_runtime::repository::GRAPH_PUBLICATION_SCHEMA_V1; use super::*; +#[tokio::test] +async fn sealed_journal_liveness_and_retirement_preserve_projection_identity() { + tracedecay_global_db::register_registered_schema_installer(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("graph.db"); + let authority = + DatabaseAuthority::acquire_test(&path, "doctor sealed journal fixture").unwrap(); + let (database, _) = + Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Initialize) + .await + .unwrap(); + database + .execute_write_batch( + "install graph publication schema", + GRAPH_PUBLICATION_SCHEMA_V1, + ) + .await + .unwrap(); + let shard = serde_json::to_string(&database.registered_binding().shard_id).unwrap(); + let mut foreign = database.registered_binding().shard_id.clone(); + foreign.profile_id = tracedecay_domain::UserProfileId::new("profile.foreign").unwrap(); + let foreign = serde_json::to_string(&foreign).unwrap(); + let digest = format!("sha256:{}", "0".repeat(64)); + for (sequence, owner, namespace, projection, generation) in [ + (1_i64, &shard, "project", "code", "generation:shared"), + (2, &shard, "project", "sessions", "generation:shared"), + (3, &shard, "project", "sessions", "generation:current"), + (4, &shard, "other-project", "code", "generation:shared"), + (5, &shard, "other-project", "code", "generation:current"), + (6, &foreign, "project", "sessions", "generation:shared"), + ] { + database + .execute_write( + "publish graph journal fixture", + "INSERT INTO graph_publication_replay_v1 + (sequence, shard_id, namespace, projection, generation, idempotency_key, + input_digest, dependency_generation_closure_digest, direct_dependency_bytes, + expected_recovered_digest, canonical_replay_source_digest, canonical_replay_source) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7, 2, ?7, ?7, X'7b7d')", + (sequence, owner.as_str(), namespace, projection, generation, + format!("publication:{sequence}"), digest.as_str()), + ) + .await + .unwrap(); + } + for sequence in [1_i64, 3, 5, 6] { + database + .execute_write( + "install graph journal head", + "INSERT INTO graph_verified_heads_v1 + (shard_id, namespace, projection, replay_sequence, recovered_digest) + SELECT shard_id, namespace, projection, sequence, expected_recovered_digest + FROM graph_publication_replay_v1 WHERE sequence = ?1", + (sequence,), + ) + .await + .unwrap(); + } + database + .execute_write( + "retain exact graph dependency", + "INSERT INTO graph_publication_replay_dependencies_v1 + (owner_replay_sequence, ordinal, dependency_replay_sequence, + shard_id, namespace, projection, generation) + SELECT 3, 0, sequence, shard_id, namespace, projection, generation + FROM graph_publication_replay_v1 WHERE sequence = 1", + (), + ) + .await + .unwrap(); + let identity = |namespace: &str, projection: &str, generation: &str| { + ( + GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new(projection).unwrap(), + ), + GraphGenerationId::new(generation).unwrap(), + ) + }; + let mut expected = BTreeSet::from([ + identity("project", "code", "generation:shared"), + identity("project", "sessions", "generation:current"), + identity("other-project", "code", "generation:current"), + ]); + assert_eq!( + live_sealed_generations(&database).await, + Some(expected.clone()) + ); + assert_eq!(deferred_native_retirements(&database).await, Some(2)); + + database + .execute_write( + "retain one superseded projection as a dependency", + "INSERT INTO graph_publication_replay_dependencies_v1 + (owner_replay_sequence, ordinal, dependency_replay_sequence, + shard_id, namespace, projection, generation) + SELECT 3, 1, sequence, shard_id, namespace, projection, generation + FROM graph_publication_replay_v1 WHERE sequence = 4", + (), + ) + .await + .unwrap(); + expected.insert(identity("other-project", "code", "generation:shared")); + assert_eq!(live_sealed_generations(&database).await, Some(expected)); + assert_eq!(deferred_native_retirements(&database).await, Some(1)); + assert_eq!( + database + .query_scalar_i64( + "preserved replays", + "SELECT COUNT(*) FROM graph_publication_replay_v1" + ) + .await + .unwrap(), + 6, + ); +} + #[test] fn configuration_read_from_pin_absent_on_cold_cache() { let missing: Result = diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 64b90ebeb1e..46e055a7a75 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -96,7 +96,7 @@ impl RegisteredStore { /// Inspect a completed family with an ordinary independent read-only SQLite /// connection. Reopening the registered runtime for every observation would /// perform another full attachment lifecycle and obscure cancellation timing. - fn external_schema_objects(&self) -> Vec { + fn external_schema_objects(&self) -> Vec<(String, Option)> { let connection = rusqlite::Connection::open_with_flags( &self.path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, @@ -104,12 +104,12 @@ impl RegisteredStore { .unwrap(); let mut statement = connection .prepare( - "SELECT type || ':' || name FROM sqlite_schema + "SELECT type || ':' || name, sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", ) .unwrap(); statement - .query_map([], |row| row.get::<_, String>(0)) + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() .map(Result::unwrap) .collect() @@ -185,12 +185,14 @@ async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { #[tokio::test(flavor = "multi_thread")] async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { - let reference = RegisteredStore::create(); - let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; + let store = RegisteredStore::create(); + let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); - let installed = reference.external_schema_objects(); + let installed = store.external_schema_objects(); assert!( - installed.contains(&"table:code_projects".to_owned()), + installed + .iter() + .any(|(object, _)| object == "table:code_projects"), "a completed attach installs the registered schema: {installed:?}" ); @@ -201,12 +203,11 @@ async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { .chain((2..total_polls).step_by(stride)) .chain([total_polls + 10]) { - let store = RegisteredStore::create(); - // Cancellation cannot modify the already-published schema or - // prevent the next admission from reopening the same family. - let (owner, runtime, retirement) = store.publish().await.into_parts(); - drop((owner, runtime)); - retire_runtime(retirement).await; + // The reference admission, then each successful reopen below, leave + // this family closed with the exact baseline schema. Creating another + // family would repeat initialization without exercising another + // cancellation point. Keep real runtime retirement and reopening + // around every sampled admission. let before_admission = store.external_schema_objects(); assert_eq!(before_admission, installed); let (outcome, _) = store.attach_cancelled_at(cancel_at).await; diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 34c2c2e544b..d4a934103db 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -80,10 +80,10 @@ use crate::state::{ }; use crate::{ GraphCommit, GraphDb, GraphDbError, GraphDbLocation, GraphDbOpenOptions, GraphDurability, - GraphEntity, GraphFormatVersion, GraphGenerationManifest, GraphGenerationManifestIdentity, - GraphGenerationRowSpill, GraphGenerationRows, GraphNamespace, GraphProjectionId, - GraphProjectionIdentity, GraphRelation, GraphWatermark, GraphWriteBatch, NeverCancelled, - SourceGeneration, SpilledGraphGeneration, + GraphEntity, GraphFormatVersion, GraphGenerationId, GraphGenerationManifest, + GraphGenerationManifestIdentity, GraphGenerationRowSpill, GraphGenerationRows, GraphNamespace, + GraphProjectionId, GraphProjectionIdentity, GraphRelation, GraphWatermark, GraphWriteBatch, + NeverCancelled, SourceGeneration, SpilledGraphGeneration, }; /// A sealed generation opened without the shared staging database. @@ -531,15 +531,15 @@ pub struct SealedStoreCensusV1 { pub unrecognized_count: u64, } -/// Measure the sealed root beside `database_path` against the generation ids -/// of the store's current verified heads (as journaled: `:`). +/// Measure the sealed root beside `database_path` against the exact projection +/// and generation identities that remain live in that store's journal. /// /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the /// census stays truthful when a receipt is mid-write or corrupt. pub fn census_sealed_store( database_path: &Path, - head_generations: &std::collections::BTreeSet, + head_generations: &std::collections::BTreeSet<(GraphProjectionIdentity, GraphGenerationId)>, ) -> std::io::Result { let root = sealed_store_root(database_path); let entries = match std::fs::read_dir(&root) { @@ -575,7 +575,18 @@ pub fn census_sealed_store( census.unrecognized_count += 1; continue; }; - if head_generations.contains(&receipt.generation) { + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace), + GraphProjectionId::new(receipt.projection), + GraphGenerationId::new(receipt.generation), + ) else { + census.unrecognized_count += 1; + continue; + }; + if head_generations.contains(&( + GraphProjectionIdentity::new(namespace, projection), + generation, + )) { census.head_count += 1; census.head_bytes += bytes; } else { @@ -2885,8 +2896,9 @@ mod build_tests { use super::{ NativeCompactEncodeBoundaryV1, SEALED_STORE_DATABASE_FILE, SealedCompactRows, SealedRowSource, build_or_open_sealed_store, build_or_open_sealed_store_with_work, - build_sealed_container_with_work, push_manifest_rows, sealed_artifact_database_options, - sealed_copy_proof, sealed_generation_directory, sealed_store_root, + build_sealed_container_with_work, census_sealed_store, push_manifest_rows, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, }; use tracedecay_store::runtime::{GraphNativeWorkKindV1, GraphNativeWorkV1}; @@ -2956,6 +2968,65 @@ mod build_tests { .unwrap() } + #[test] + fn sealed_census_scopes_equal_generation_names_to_their_projection() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let mut heads = BTreeSet::new(); + let mut artifacts = Vec::new(); + let mut expected_head_bytes = 0; + let mut expected_superseded_bytes = 0; + for (namespace, projection, generation, live) in [ + ("project", "code", "generation:shared", true), + ("project", "sessions", "generation:shared", false), + ("project", "sessions", "generation:current", true), + ("other-project", "code", "generation:shared", false), + ("other-project", "code", "generation:current", true), + ] { + let mut manifest = manifest(2, 0); + manifest.projection = GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new(projection).unwrap(), + ); + manifest.generation = GraphGenerationId::new(generation).unwrap(); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + let directory = store.directory.clone(); + drop(store); + if live { + heads.insert((identity.projection, identity.generation)); + } + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + let bytes = std::fs::read(&path).unwrap(); + if live { + expected_head_bytes += bytes.len() as u64; + } else { + expected_superseded_bytes += bytes.len() as u64; + } + artifacts.push((path, bytes)); + } + } + + let census = census_sealed_store(&database_path, &heads).unwrap(); + assert_eq!(census.head_count, 3); + assert_eq!(census.superseded_count, 2); + assert_eq!(census.unrecognized_count, 0); + assert_eq!(census.head_bytes, expected_head_bytes); + assert_eq!(census.superseded_bytes, expected_superseded_bytes); + for (path, bytes) in artifacts { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + } + fn open_source(database_path: &std::path::Path) -> crate::GraphDbLeaseV1 { let owner = GraphDbOwner::open(GraphDbOpenOptions { location: GraphDbLocation::Persistent(database_path.to_path_buf()), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 3bb165ee0f6..7906c8c836f 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1354,7 +1354,7 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { std::fs::create_dir_all(&unreadable).unwrap(); std::fs::write(unreadable.join("generation.grafeo"), b"no receipt").unwrap(); - let heads = std::collections::BTreeSet::from(["census-g2".to_owned()]); + let heads = BTreeSet::from([(identity, GraphGenerationId::new("census-g2").unwrap())]); let census = tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) .unwrap(); diff --git a/crates/tracedecay-maintenance/src/retention/footprint.rs b/crates/tracedecay-maintenance/src/retention/footprint.rs index 79396636d2a..32ffbb39222 100644 --- a/crates/tracedecay-maintenance/src/retention/footprint.rs +++ b/crates/tracedecay-maintenance/src/retention/footprint.rs @@ -6,7 +6,12 @@ //! That is why the ceiling is an entry count: a byte budget small enough to be //! cheap is smaller than one real graph container. +use std::borrow::Cow; use std::collections::BTreeMap; +#[cfg(unix)] +use std::collections::BTreeSet; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -16,11 +21,13 @@ use tracedecay_contracts::doctor::{ DoctorStorageFamilyReadV1, DoctorStorageFindingKindV1, DoctorStorageFindingV1, storage_family_read, }; +use tracedecay_domain::canonical_text::sha256_hex; +use tracedecay_domain::fold_control_characters; use tracedecay_global_db::registry_maintenance::{RootLivenessV1, probe_root}; use tracedecay_runtime_core::text::format_bytes; use super::storage_report::{ - FullProfileSizeV1, GRAPH_CONTAINER_SUFFIX, ProfileTotalCoverageStateV1, StoreKindBytesV1, + FullProfileSizeV1, ProfileTotalCoverageStateV1, StoreKindBytesV1, graph_container_stem, }; /// Entries a footprint walk visits before it stops and reports a floor. @@ -231,6 +238,16 @@ pub fn walk_regular_files_with_budget( WalkInterruptionV1::DeadlineExceeded => WalkTerminationV1::DeadlineExceeded, }; } + // read_dir follows a symlink when the directory argument itself is + // a symlink. Check every pending directory, including the caller's + // root, instead of relying only on child DirEntry::file_type. + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + _ => { + visit(&directory, WalkedEntryV1::Unavailable); + continue; + } + } let Ok(entries) = std::fs::read_dir(&directory) else { visit(&directory, WalkedEntryV1::Unavailable); continue; @@ -305,6 +322,8 @@ struct EntryTally { bytes: u64, unavailable_entry_count: usize, kinds: Option, + /// An expected project directory was replaced by a regular file. + unexpected_project_file: bool, } struct FootprintKeys { @@ -342,10 +361,27 @@ pub(crate) fn census_profile_footprint_with_budget( else { return; }; + // The file itself is measurable, but it cannot be a usable + // project-store directory. Keep its bytes in the physical total, + // and report the missing store traversal as unavailable rather + // than calling the replaced project healthy. + let unexpected_project_file = matches!(walked, WalkedEntryV1::File { .. }) + && keys + .project + .as_ref() + .is_some_and(|(_, relative)| relative.as_os_str().is_empty()); + if unexpected_project_file { + unavailable_entry_count = unavailable_entry_count.saturating_add(1); + } let primary = tallies.entry(keys.primary).or_default(); tally(primary, walked); + if unexpected_project_file { + primary.unavailable_entry_count = primary.unavailable_entry_count.saturating_add(1); + primary.unexpected_project_file = true; + } if let (Some((_, store_relative)), WalkedEntryV1::File { bytes }) = (&keys.project, walked) + && !unexpected_project_file { primary .kinds @@ -369,10 +405,15 @@ pub(crate) fn census_profile_footprint_with_budget( } }); ProfileFootprintEntryV1 { - bound_bytes: bound_overrides - .get(&key) - .copied() - .unwrap_or_else(|| default_bound_bytes(family)), + bound_bytes: bound_overrides.get(&key).copied().unwrap_or_else(|| { + if tally.unexpected_project_file { + // An unrelated file must not inherit a project's 10 GiB + // allowance. An explicit owner override still wins. + default_bound_bytes(ProfileFamilyV1::Other) + } else { + default_bound_bytes(family) + } + }), key, family, bytes: tally.bytes, @@ -408,6 +449,9 @@ pub fn census_sqlite_temp_spill( ) -> Option { let mut bytes = 0u64; let mut unavailable_entry_count = 0usize; + // Multiple descriptors may refer to one unlinked SQLite allocation. + #[cfg(unix)] + let mut observed_files = BTreeSet::new(); for entry in std::fs::read_dir(fd_directory).ok()? { let fd = match entry { Ok(fd) => fd, @@ -431,7 +475,13 @@ pub fn census_sqlite_temp_spill( .is_some_and(|name| name.starts_with("etilqs_") && name.ends_with(" (deleted)")); if unlinked_temp { match std::fs::metadata(fd.path()) { - Ok(metadata) => bytes = bytes.saturating_add(metadata.len()), + Ok(metadata) => { + #[cfg(unix)] + if !observed_files.insert((metadata.dev(), metadata.ino())) { + continue; + } + bytes = bytes.saturating_add(metadata.len()); + } // A closed descriptor no longer consumes space. Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => unavailable_entry_count = unavailable_entry_count.saturating_add(1), @@ -465,6 +515,13 @@ fn is_wal(name: &str) -> bool { name.ends_with("-wal") || name.ends_with(".wal") } +// SQLite WALs are regular root-level files. Only Grafeo WALs are +// directories, so a similarly named incident tree must not inherit a WAL +// budget or hide its own independently bounded footprint. +fn is_wal_family_root(name: &str, is_file: bool) -> bool { + is_wal(name) && (is_file || name.ends_with(".grafeo.wal")) +} + /// The footprint keys one profile-relative path belongs to. A WAL directory /// is one entry keyed by the directory; its segments sum into it. fn footprint_keys(relative: &Path) -> Option { @@ -473,25 +530,31 @@ fn footprint_keys(relative: &Path) -> Option { .map(|component| component.as_os_str().to_string_lossy().into_owned()) .collect::>(); let first = components.first()?; - if first == PROJECTS_DIRECTORY && components.len() >= 3 { + // A non-traversable project root (symlink, unreadable directory, or + // unexpected file) still belongs to that project's named finding. + // Otherwise its unavailable bytes appear under a generic `projects` key + // while the affected store disappears from Doctor altogether. + if first == PROJECTS_DIRECTORY && components.len() >= 2 { let store_key = format!("{PROJECTS_DIRECTORY}/{}", components[1]); let inner = &components[2..]; + // A container's verified/spill tree owns its descendants, even when + // a scratch file is named *.wal. Only the actual container WAL + // directory (or a WAL outside a container) gets a WAL bound. let overlay = inner - .iter() - .position(|name| is_wal(name)) - .map(|end| { - ( - format!("{store_key}/{}", inner[..=end].join("/")), - ProfileFamilyV1::Wal, - ) - }) - .or_else(|| { - (inner.len() == 1 && inner[0].ends_with(GRAPH_CONTAINER_SUFFIX)).then(|| { + .split_first() + .and_then(|(first, rest)| { + graph_container_stem(first, rest.is_empty()).map(|stem| { ( - format!("{store_key}/{}", inner[0]), + format!("{store_key}/{stem}"), ProfileFamilyV1::GraphContainer, ) }) + }) + .or_else(|| { + inner + .first() + .filter(|name| is_wal_family_root(name, inner.len() == 1)) + .map(|first| (format!("{store_key}/{first}"), ProfileFamilyV1::Wal)) }); return Some(FootprintKeys { primary: (store_key, ProfileFamilyV1::ProjectStore), @@ -499,19 +562,33 @@ fn footprint_keys(relative: &Path) -> Option { project: Some((components[1].clone(), inner.iter().collect())), }); } - let primary = if let Some(end) = components.iter().position(|name| is_wal(name)) { - (components[..=end].join("/"), ProfileFamilyV1::Wal) - } else if let Some(stem) = first - .find(GRAPH_CONTAINER_SUFFIX) - .map(|at| &first[..at + GRAPH_CONTAINER_SUFFIX.len()]) - { - // A top-level container and its sidecars (`.verified`, `.spill`). + // A top-level family owns all descendants. WAL-looking nested scratch + // filenames in an otherwise unrelated tree must not escape that bound. + let primary = if let Some(stem) = graph_container_stem(first, components.len() == 1) { + // Sidecar trees take precedence over WAL-looking scratch filenames. (stem.to_owned(), ProfileFamilyV1::GraphContainer) - } else if first == "global.db" || first.starts_with("global.db-") { + } else if is_wal_family_root(first, components.len() == 1) { + (first.clone(), ProfileFamilyV1::Wal) + } else if components.len() == 1 + && matches!( + first.as_str(), + "global.db" | "global.db-shm" | "global.db-journal" + ) + { ("global.db".to_owned(), ProfileFamilyV1::ProfileRegistry) - } else if first.starts_with("user-sessions.db") { + } else if components.len() == 1 + && matches!( + first.as_str(), + "user-sessions.db" | "user-sessions.db-shm" | "user-sessions.db-journal" + ) + { ("user-sessions.db".to_owned(), ProfileFamilyV1::UserSessions) - } else if first.starts_with("user-memory.") { + } else if components.len() == 1 + && matches!( + first.as_str(), + "user-memory.db" | "user-memory.db-shm" | "user-memory.db-journal" + ) + { ("user-memory".to_owned(), ProfileFamilyV1::UserMemory) } else { let family = match first.as_str() { @@ -530,6 +607,48 @@ fn footprint_keys(relative: &Path) -> Option { }) } +/// Keep a stable evidence reference even for deeply nested filesystem names. +/// The report retains the complete key; Doctor references have a smaller bound. +fn doctor_footprint_reference_key(key: &str) -> Cow<'_, str> { + if DoctorEvidenceReferenceV1::new(key.to_owned()).is_ok() { + return Cow::Borrowed(key); + } + Cow::Owned(format!("footprint/sha256:{}", sha256_hex(key.as_bytes()))) +} + +/// Preserve the measured size and bound in Doctor even when an optional +/// registered root, kind breakdown, or long footprint identity exceeds its +/// 512-byte coverage statement. Full identities and breakdowns remain in the +/// storage report; a truncated number would make the finding unactionable. +fn bounded_entry_subject(entry: &ProfileFootprintEntryV1) -> (String, String) { + let mut evidence_key = doctor_footprint_reference_key(&entry.key).into_owned(); + let mut subject = entry_statement(entry, &evidence_key); + let fits = |subject: &str| { + let statement = footprint_statement(subject, entry.bytes, entry.bound_bytes); + // Compare the *rendered* statement. Doctor folds control characters, + // so a valid short registered path containing a tab or newline must + // not be mistaken for an overlong detail and silently omitted. + super::diagnostics::bounded_statement(&statement) + == fold_control_characters(&statement).trim() + }; + if !fits(&subject) { + let mut compact = entry.clone(); + compact.kinds = None; + if let Some(project) = compact.project.as_mut() { + // Keep the verified root state, but omit its optional long path. + project.registered_root = None; + } + subject = entry_statement(&compact, &evidence_key); + if !fits(&subject) { + // A valid Doctor reference can be longer than its statement budget. + // Retain stable evidence identity without losing the numeric bound. + evidence_key = format!("footprint/sha256:{}", sha256_hex(entry.key.as_bytes())); + subject = entry_statement(&compact, &evidence_key); + } + } + (evidence_key, subject) +} + /// One `OverBudgetStore` finding per footprint entry plus one for the profile /// total. A floor above its bound proves the overage, so an over-bound entry /// is degraded with complete coverage even when bytes went unread. @@ -537,9 +656,10 @@ fn footprint_keys(relative: &Path) -> Option { pub fn footprint_findings(footprint: &ProfileFootprintV1) -> DoctorStorageFamilyReadV1 { let mut findings = Vec::with_capacity(footprint.entries.len() + 1); for entry in &footprint.entries { + let (evidence_key, subject) = bounded_entry_subject(entry); let Some(finding) = bounded_finding( - &entry.key, - &entry_statement(entry), + evidence_key.as_str(), + &subject, entry.bytes, entry.bound_bytes, entry.unavailable_entry_count > 0 || footprint.truncated, @@ -596,7 +716,7 @@ fn registered_project_reclaim_hint(entry: &ProfileFootprintEntryV1) -> Option String { +fn entry_statement(entry: &ProfileFootprintEntryV1, evidence_key: &str) -> String { let label = match entry.family { ProfileFamilyV1::ProjectStore => "project store", ProfileFamilyV1::GraphContainer => "graph container", @@ -611,6 +731,10 @@ fn entry_statement(entry: &ProfileFootprintEntryV1) -> String { ProfileFamilyV1::SqliteTempSpill => "unlinked SQLite temp files", ProfileFamilyV1::Other => "profile entry", }; + let floor = floor_note(entry.unavailable_entry_count, false); + if evidence_key != entry.key { + return format!("{label} {evidence_key} (full path in storage report){floor}"); + } let root = entry .project @@ -630,13 +754,16 @@ fn entry_statement(entry: &ProfileFootprintEntryV1) -> String { ) }); let reclaim = registered_project_reclaim_hint(entry).unwrap_or_default(); - let floor = floor_note(entry.unavailable_entry_count, false); let render = |root: &str, reclaim: &str| { - format!("{label} {}{reclaim}{root}{breakdown}{floor}", entry.key) + format!("{label} {evidence_key}{reclaim}{root}{breakdown}{floor}") }; let fits = |subject: &str| { let statement = footprint_statement(subject, entry.bytes, entry.bound_bytes); - super::diagnostics::bounded_statement(&statement) == statement + // Compare the *rendered* statement. Doctor folds control characters, + // so a valid short registered path containing a tab or newline must + // not be mistaken for an overlong detail and silently omitted. + super::diagnostics::bounded_statement(&statement) + == fold_control_characters(&statement).trim() }; let detailed = render(&root, &reclaim); if reclaim.is_empty() || fits(&detailed) { @@ -769,12 +896,38 @@ mod tests { std::os::unix::fs::symlink(&target, fd_directory.join(fd)).expect("fd link"); } + // Duplicated file descriptors must not double-count a physical inode. + std::os::unix::fs::symlink( + temporary.path().join("etilqs_5e598e639aa4c638 (deleted)"), + fd_directory.join("7"), + ) + .expect("duplicated temp descriptor"); + let entry = census_sqlite_temp_spill(&fd_directory, &BTreeMap::new()) .expect("a readable fd directory"); assert_eq!(entry.key, SQLITE_TEMP_SPILL_KEY); assert_eq!(entry.bytes, 1_924 * MIB); assert_eq!(entry.bound_bytes, GIB); + let healthy_budget = BTreeMap::from([(SQLITE_TEMP_SPILL_KEY.to_owned(), 2 * GIB)]); + let healthy = census_sqlite_temp_spill(&fd_directory, &healthy_budget) + .expect("same unlinked SQLite allocations"); + assert_eq!(healthy.bytes, 1_924 * MIB); + assert_eq!(healthy.bound_bytes, 2 * GIB); + let healthy_footprint = ProfileFootprintV1 { + entries: vec![healthy], + total_bytes: 0, + unavailable_entry_count: 0, + truncated: false, + total_bound_bytes: PROFILE_TOTAL_BOUND_BYTES, + }; + assert_eq!( + findings(&healthy_footprint)[SQLITE_TEMP_SPILL_KEY] + .finding() + .state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "duplicate handles must not cause a false storage-pressure finding" + ); let footprint = ProfileFootprintV1 { entries: vec![entry], total_bytes: 0, @@ -1318,6 +1471,135 @@ mod tests { assert_eq!(remaining, 100 - 1 - 64 - 1 - 1); } + #[cfg(unix)] + #[test] + fn unreadable_project_root_has_its_own_partial_finding() { + let temporary = tempfile::tempdir().expect("footprint fixture"); + let profile = temporary.path().join("profile"); + let foreign = temporary.path().join("foreign"); + sized(&foreign.join("private"), 2 * GIB); + sized(&profile.join("projects/proj_live/tracedecay.db"), 11); + std::os::unix::fs::symlink(&foreign, profile.join("projects/proj_blocked")) + .expect("substituted project directory"); + + let registered_root = temporary.path().join("registered-checkout"); + std::fs::create_dir(®istered_root).expect("registered root"); + let registered = BTreeMap::from([( + "proj_blocked".to_owned(), + registered_root.display().to_string(), + )]); + let overrides = BTreeMap::from([ + ("projects/proj_blocked".to_owned(), 1), + ("projects/proj_live".to_owned(), 11), + (PROFILE_TOTAL_KEY.to_owned(), 11), + ]); + + let footprint = census_profile_footprint(&profile, ®istered, &overrides); + assert_eq!( + footprint.total_bytes, 11, + "foreign bytes must not be counted" + ); + assert_eq!(footprint.unavailable_entry_count, 1); + assert_eq!(footprint.entries.len(), 2, "no generic projects entry"); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Partial + ); + let blocked = entry(&footprint, "projects/proj_blocked"); + assert_eq!(blocked.family, ProfileFamilyV1::ProjectStore); + assert_eq!(blocked.bytes, 0); + assert_eq!(blocked.unavailable_entry_count, 1); + assert_eq!(blocked.bound_bytes, 1); + assert_eq!( + blocked.project.as_ref().expect("registered project").state, + ProjectRootStateV1::Present + ); + assert_eq!(entry(&footprint, "projects/proj_live").bytes, 11); + + let observed = findings(&footprint); + assert_eq!(observed.len(), 3); + assert_eq!( + observed["projects/proj_blocked"].finding().state(), + DoctorEvidenceStateV1::Partial, + "an untraversable project store cannot be reported healthy or omitted" + ); + assert_eq!( + observed["projects/proj_live"].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage + ); + assert_eq!( + observed[PROFILE_TOTAL_KEY].finding().state(), + DoctorEvidenceStateV1::Partial + ); + } + + #[test] + fn regular_file_replacing_project_directory_is_not_healthy_or_project_budgeted() { + let temporary = tempfile::tempdir().expect("footprint fixture"); + let profile = temporary.path().join("profile"); + sized(&profile.join("projects/proj_replaced"), 11); + sized(&profile.join("projects/proj_healthy/tracedecay.db"), 7); + let registered_root = temporary.path().join("registered-checkout"); + std::fs::create_dir(®istered_root).expect("registered root"); + let registered = BTreeMap::from([( + "proj_replaced".to_owned(), + registered_root.display().to_string(), + )]); + + let footprint = census_profile_footprint(&profile, ®istered, &BTreeMap::new()); + assert_eq!(footprint.total_bytes, 18, "count all regular-file bytes"); + assert_eq!(footprint.unavailable_entry_count, 1); + assert_eq!(footprint.entries.len(), 2, "name the replaced store"); + let replaced = entry(&footprint, "projects/proj_replaced"); + assert_eq!(replaced.family, ProfileFamilyV1::ProjectStore); + assert_eq!(replaced.bytes, 11, "do not hide the replacement file"); + assert_eq!(replaced.unavailable_entry_count, 1); + assert_eq!( + replaced.bound_bytes, + default_bound_bytes(ProfileFamilyV1::Other), + "a replacement file must not inherit the 10 GiB project allowance" + ); + assert!(replaced.kinds.is_none(), "not a valid project-store file"); + assert_eq!( + replaced.project.as_ref().expect("registration").state, + ProjectRootStateV1::Present + ); + let healthy = entry(&footprint, "projects/proj_healthy"); + assert_eq!(healthy.bytes, 7); + assert_eq!(healthy.unavailable_entry_count, 0); + assert_eq!( + healthy.bound_bytes, + default_bound_bytes(ProfileFamilyV1::ProjectStore) + ); + + let observed = findings(&footprint); + assert_eq!(observed.len(), 3); + assert_eq!( + observed["projects/proj_replaced"].finding().state(), + DoctorEvidenceStateV1::Partial + ); + assert_eq!( + observed["projects/proj_healthy"].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage + ); + assert_eq!( + observed[PROFILE_TOTAL_KEY].finding().state(), + DoctorEvidenceStateV1::Partial + ); + + // Exercise the same overage path without allocating a 1 GiB fixture + // on platforms that eagerly provision extended regular files. + let overrides = BTreeMap::from([("projects/proj_replaced".to_owned(), 10)]); + let over = census_profile_footprint(&profile, ®istered, &overrides); + assert_eq!(over.total_bytes, 18); + assert_eq!(entry(&over, "projects/proj_replaced").bound_bytes, 10); + assert_eq!( + findings(&over)["projects/proj_replaced"].finding().state(), + DoctorEvidenceStateV1::Degraded, + "a large replacement file must trigger its tighter named bound" + ); + } + #[test] fn one_entry_budget_is_shared_across_walk_roots() { let temporary = tempfile::tempdir().expect("walk fixture"); @@ -1365,6 +1647,592 @@ mod tests { assert_eq!(remaining, 9); } + #[test] + fn hashed_wal_reference_keeps_overages_and_partial_coverage() { + // A real WAL overlay is anchored at a store root; nested payloads do + // not create WAL families. Construct the oversized diagnostic key + // directly to exercise the bounded Doctor evidence contract. + let key = format!("projects/proj_deep/{}.wal", "x".repeat(1025)); + let mut footprint = ProfileFootprintV1 { + entries: vec![ + ProfileFootprintEntryV1 { + key: "projects/proj_deep".to_owned(), + family: ProfileFamilyV1::ProjectStore, + bytes: 2 * GIB, + unavailable_entry_count: 1, + bound_bytes: 1, + project: None, + kinds: None, + }, + ProfileFootprintEntryV1 { + key: key.clone(), + family: ProfileFamilyV1::Wal, + bytes: 2 * GIB, + unavailable_entry_count: 1, + bound_bytes: default_bound_bytes(ProfileFamilyV1::Wal), + project: None, + kinds: None, + }, + ], + total_bytes: 2 * GIB, + unavailable_entry_count: 1, + truncated: false, + total_bound_bytes: 1, + }; + assert!(DoctorEvidenceReferenceV1::new(key.clone()).is_err()); + + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + let reference = format!("footprint/sha256:{}", sha256_hex(key.as_bytes())); + let wal_finding = &observed[&reference]; + assert_eq!( + wal_finding.finding().state(), + DoctorEvidenceStateV1::Degraded + ); + let statement = wal_finding.finding().coverage().statement(); + assert!( + statement.contains("full path in storage report"), + "{statement}" + ); + assert!(statement.contains(&format_bytes(2 * GIB)), "{statement}"); + assert!( + statement.contains(&format_bytes(default_bound_bytes(ProfileFamilyV1::Wal))), + "{statement}" + ); + assert!(statement.contains("1 unreadable entries"), "{statement}"); + for reference in ["projects/proj_deep", PROFILE_TOTAL_KEY] { + assert_eq!( + observed[reference].finding().state(), + DoctorEvidenceStateV1::Degraded, + "an oversized identity must not suppress other overages" + ); + } + + footprint.entries[1].bound_bytes = 3 * GIB; + let observed = findings(&footprint); + let finding = observed[&reference].finding(); + assert_eq!(finding.state(), DoctorEvidenceStateV1::Partial); + let statement = finding.coverage().statement(); + assert!(statement.contains("WAL "), "{statement}"); + assert!(statement.contains("1 unreadable entries"), "{statement}"); + } + + #[test] + fn top_level_container_bound_excludes_unrelated_grafeo_suffixes() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (name, bytes) in [ + ("user-sessions.grafeo", 5), + ("user-sessions.grafeo.verified", 7), + ("user-sessions.grafeo.spill/scratch/inflight", 11), + ("user-sessions.grafeo.notes", 30), + ("other.grafeo.metrics", 40), + ("scratch.grafeo/unrelated", 13), + ] { + sized(&profile.join(name), bytes); + } + let container_key = "user-sessions.grafeo"; + let overrides = BTreeMap::from([(container_key.to_owned(), 24)]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + + assert_eq!(footprint.total_bytes, 106); + assert_eq!(entry(&footprint, container_key).bytes, 23); + assert_eq!( + entry(&footprint, container_key).family, + ProfileFamilyV1::GraphContainer + ); + for (name, bytes) in [ + ("user-sessions.grafeo.notes", 30), + ("other.grafeo.metrics", 40), + ("scratch.grafeo", 13), + ] { + let unrelated = entry(&footprint, name); + assert_eq!(unrelated.family, ProfileFamilyV1::Other); + assert_eq!(unrelated.bytes, bytes); + } + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + let observed = findings(&footprint); + assert_eq!( + observed[container_key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "unowned files must not falsely trip the graph-container bound" + ); + + let over = census_profile_footprint( + &profile, + &BTreeMap::new(), + &BTreeMap::from([(container_key.to_owned(), 22)]), + ); + assert_eq!( + findings(&over)[container_key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "actual graph sidecar bytes must still trip their bound" + ); + } + + #[test] + fn project_container_bound_includes_verified_and_spill_sidecars() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + let project = "projects/proj_sidecars"; + for (name, bytes) in [ + ("graph.grafeo", 5), + ("graph.grafeo.verified", 7), + ("graph.grafeo.spill", 11), + ("graph.grafeo.wal/segment", 13), + ("other.grafeo.spill/scratch/inflight", 17), + ("not-a-graph.grafeo.tmp", 19), + ] { + sized(&profile.join(project).join(name), bytes); + } + let container_key = format!("{project}/graph.grafeo"); + let sibling_key = format!("{project}/other.grafeo"); + let wal_key = format!("{project}/graph.grafeo.wal"); + let overrides = BTreeMap::from([(container_key.clone(), 10), (sibling_key.clone(), 16)]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 72); + assert_eq!(entry(&footprint, project).bytes, 72); + assert_eq!( + entry(&footprint, project) + .kinds + .expect("physical project-kind census") + .graph_container, + 53 + ); + assert_eq!( + entry(&footprint, &container_key).family, + ProfileFamilyV1::GraphContainer + ); + assert_eq!(entry(&footprint, &container_key).bytes, 23); + assert_eq!(entry(&footprint, &sibling_key).bytes, 17); + assert_eq!(entry(&footprint, &wal_key).family, ProfileFamilyV1::Wal); + assert_eq!(entry(&footprint, &wal_key).bytes, 13); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + + let observed = findings(&footprint); + assert_eq!( + observed[&container_key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "the former main-file-only overlay would falsely report healthy" + ); + assert_eq!( + observed[&sibling_key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "an orphaned spill still consumes a named container bound" + ); + assert_eq!( + observed[&wal_key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "WAL pressure remains its own bound, not double-charged to containers" + ); + } + + #[test] + fn nested_wal_named_graph_spill_files_remain_under_container_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("user-sessions.grafeo", 5), + ("user-sessions.grafeo.spill/scratch/chunk.wal", 17), + ("user-sessions.grafeo.spill/scratch/segment-wal/part", 19), + ("user-sessions.grafeo.wal/segment", 7), + ("projects/proj_spill/graph.grafeo", 3), + ( + "projects/proj_spill/graph.grafeo.spill/scratch/chunk.wal", + 23, + ), + ( + "projects/proj_spill/graph.grafeo.spill/segment-wal/part", + 29, + ), + ("projects/proj_spill/graph.grafeo.wal/segment", 11), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("user-sessions.grafeo".to_owned(), 40), + ("projects/proj_spill/graph.grafeo".to_owned(), 54), + ("user-sessions.grafeo.wal".to_owned(), 7), + ("projects/proj_spill/graph.grafeo.wal".to_owned(), 11), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 114); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 5); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + + for (key, family, bytes) in [ + ("user-sessions.grafeo", ProfileFamilyV1::GraphContainer, 41), + ( + "projects/proj_spill/graph.grafeo", + ProfileFamilyV1::GraphContainer, + 55, + ), + ("user-sessions.grafeo.wal", ProfileFamilyV1::Wal, 7), + ( + "projects/proj_spill/graph.grafeo.wal", + ProfileFamilyV1::Wal, + 11, + ), + ("projects/proj_spill", ProfileFamilyV1::ProjectStore, 66), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + + let observed = findings(&footprint); + assert_eq!(observed.len(), 6); + for key in ["user-sessions.grafeo", "projects/proj_spill/graph.grafeo"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must detect spill bytes despite WAL-looking filenames" + ); + } + for key in [ + "user-sessions.grafeo.wal", + "projects/proj_spill/graph.grafeo.wal", + "projects/proj_spill", + PROFILE_TOTAL_KEY, + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must retain its actual bound" + ); + } + } + + #[test] + fn database_family_bounds_do_not_swallow_unrelated_suffix_files() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (name, bytes) in [ + ("global.db", 5), + ("global.db-shm", 7), + ("global.db-journal", 11), + ("global.db-wal", 13), + ("global.db-mystery", 17), + ("user-sessions.db", 19), + ("user-sessions.db-shm", 23), + ("user-sessions.db-journal", 29), + ("user-sessions.db-wal", 31), + ("user-sessions.db-backup", 37), + ("user-memory.db", 41), + ("user-memory.db-shm", 43), + ("user-memory.db-journal", 47), + ("user-memory.db-wal", 53), + ("user-memory.db-backup", 59), + ("user-memory.grafeo.notes", 61), + ] { + sized(&profile.join(name), bytes); + } + let overrides = BTreeMap::from([ + ("global.db".to_owned(), 23), + ("user-sessions.db".to_owned(), 71), + ("global.db-mystery".to_owned(), 16), + ("user-sessions.db-backup".to_owned(), 36), + ("user-memory".to_owned(), 132), + ("user-memory.db-wal".to_owned(), 54), + ("user-memory.db-backup".to_owned(), 58), + ("user-memory.grafeo.notes".to_owned(), 60), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 496); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + + for (key, family, bytes) in [ + ("global.db", ProfileFamilyV1::ProfileRegistry, 23), + ("user-sessions.db", ProfileFamilyV1::UserSessions, 71), + ("global.db-wal", ProfileFamilyV1::Wal, 13), + ("user-sessions.db-wal", ProfileFamilyV1::Wal, 31), + ("global.db-mystery", ProfileFamilyV1::Other, 17), + ("user-sessions.db-backup", ProfileFamilyV1::Other, 37), + ("user-memory", ProfileFamilyV1::UserMemory, 131), + ("user-memory.db-wal", ProfileFamilyV1::Wal, 53), + ("user-memory.db-backup", ProfileFamilyV1::Other, 59), + ("user-memory.grafeo.notes", ProfileFamilyV1::Other, 61), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + for key in [ + "global.db", + "user-sessions.db", + "global.db-wal", + "user-sessions.db-wal", + "user-memory", + "user-memory.db-wal", + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must not inherit unrelated suffix bytes" + ); + } + for key in [ + "global.db-mystery", + "user-sessions.db-backup", + "user-memory.db-backup", + "user-memory.grafeo.notes", + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must retain its own named bound" + ); + } + } + + #[test] + fn managed_profile_families_bound_wal_named_payloads() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (name, bytes) in [ + ("lcm-payloads/payload-wal/part", 25), + ("lcm-payloads/archive.wal", 34), + ("lcm-payloads/plain", 3), + ("response-handles/handle-wal/part", 11), + ("maintenance/cursor.wal", 13), + ("global.db-wal", 7), + ] { + sized(&profile.join(name), bytes); + } + let overrides = BTreeMap::from([ + ("lcm-payloads".to_owned(), 61), + ("response-handles".to_owned(), 10), + ("maintenance".to_owned(), 12), + ("global.db-wal".to_owned(), 7), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 93); + assert_eq!( + footprint.entries.len(), + 4, + "owner aggregates must not be split" + ); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + for (key, family, bytes) in [ + ("lcm-payloads", ProfileFamilyV1::LcmPayloads, 62), + ("response-handles", ProfileFamilyV1::ResponseHandles, 11), + ("maintenance", ProfileFamilyV1::Maintenance, 13), + ("global.db-wal", ProfileFamilyV1::Wal, 7), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + for key in ["lcm-payloads", "response-handles", "maintenance"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must retain a named over-budget finding" + ); + } + for key in ["global.db-wal", "profile"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key}" + ); + } + } + + #[test] + fn project_payloads_with_wal_names_retain_only_real_root_wal_overlays() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ( + "projects/proj_payload/code-index-v1/objects/archive.wal", + 40, + ), + ("projects/proj_payload/lcm-payloads/payload-wal/chunk", 30), + ("projects/proj_payload/response-handles/result.wal", 17), + ("projects/proj_payload/plain", 3), + ("projects/proj_payload/sessions.db-wal", 7), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("projects/proj_payload".to_owned(), 96), + ("projects/proj_payload/sessions.db-wal".to_owned(), 7), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 97); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!( + footprint.entries.len(), + 2, + "nested payloads cannot be false WALs" + ); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + let project = entry(&footprint, "projects/proj_payload"); + assert_eq!(project.family, ProfileFamilyV1::ProjectStore); + assert_eq!(project.bytes, 97); + let kinds = project.kinds.expect("project breakdown"); + assert_eq!(kinds.generation_artifacts, 40); + assert_eq!(kinds.sessions, 7); + assert_eq!(kinds.other, 50); + assert_eq!(kinds.total(), 97); + let wal = entry(&footprint, "projects/proj_payload/sessions.db-wal"); + assert_eq!(wal.family, ProfileFamilyV1::Wal); + assert_eq!(wal.bytes, 7); + let observed = findings(&footprint); + assert_eq!(observed.len(), 3); + assert_eq!( + observed["projects/proj_payload"].finding().state(), + DoctorEvidenceStateV1::Degraded + ); + for key in ["projects/proj_payload/sessions.db-wal", PROFILE_TOTAL_KEY] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key}" + ); + } + } + + #[test] + fn unrelated_profile_trees_keep_wal_named_descendants_under_their_own_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("host-bundle-stage/artifact-wal/chunk", 40), + ("host-bundle-stage/cache/trace.wal", 30), + ("host-components/bundle.wal", 17), + ("host-components/plain", 3), + ("global.db-wal", 7), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("host-bundle-stage".to_owned(), 69), + ("host-components".to_owned(), 19), + ("global.db-wal".to_owned(), 7), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 97); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 3); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + for (key, family, bytes) in [ + ("host-bundle-stage", ProfileFamilyV1::Other, 70), + ("host-components", ProfileFamilyV1::Other, 20), + ("global.db-wal", ProfileFamilyV1::Wal, 7), + ] { + let measured = entry(&footprint, key); + assert_eq!(measured.family, family, "{key}"); + assert_eq!(measured.bytes, bytes, "{key}"); + } + let observed = findings(&footprint); + assert_eq!(observed.len(), 4); + for key in ["host-bundle-stage", "host-components"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must retain its named over-budget finding" + ); + } + for key in ["global.db-wal", PROFILE_TOTAL_KEY] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must keep its actual bound" + ); + } + } + + #[test] + fn sqlite_sidecar_named_directories_have_independent_doctor_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + // A directory bearing a SQLite sidecar name is not a SQLite file. + for (relative, bytes) in [ + ("global.db", 5), + ("user-sessions.db", 7), + ("user-memory.db", 9), + ("global.db-journal/incident.bin", 11), + ("user-memory.db-shm/incident.bin", 13), + ("user-sessions.db-shm/incident.bin", 17), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("global.db".to_owned(), 5), + ("user-sessions.db".to_owned(), 7), + ("user-memory".to_owned(), 9), + ("global.db-journal".to_owned(), 10), + ("user-memory.db-shm".to_owned(), 12), + ("user-sessions.db-shm".to_owned(), 16), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 62); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + let observed = findings(&footprint); + assert_eq!(observed.len(), footprint.entries.len() + 1); + for (key, family, bytes) in [ + ("global.db", ProfileFamilyV1::ProfileRegistry, 5), + ("user-sessions.db", ProfileFamilyV1::UserSessions, 7), + ("user-memory", ProfileFamilyV1::UserMemory, 9), + ("global.db-journal", ProfileFamilyV1::Other, 11), + ("user-memory.db-shm", ProfileFamilyV1::Other, 13), + ("user-sessions.db-shm", ProfileFamilyV1::Other, 17), + ] { + assert_eq!(entry(&footprint, key).family, family, "{key}"); + assert_eq!(entry(&footprint, key).bytes, bytes, "{key}"); + } + for key in ["global.db", "user-sessions.db", "user-memory", "profile"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key}" + ); + } + for key in [ + "global.db-journal", + "user-memory.db-shm", + "user-sessions.db-shm", + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must have an independent named over-budget finding" + ); + } + } + #[test] fn an_unreadable_walk_root_is_accounted_as_unavailable() { let temporary = tempfile::tempdir().expect("walk fixture"); @@ -1388,4 +2256,215 @@ mod tests { assert_eq!(unavailable, 1); assert_eq!(remaining, 10); } + + #[cfg(unix)] + #[test] + fn linked_profile_root_cannot_measure_foreign_storage_as_owned() { + let temporary = tempfile::tempdir().expect("footprint fixture"); + let outside = temporary.path().join("outside"); + let profile = temporary.path().join("profile"); + sized(&outside.join("lcm-payloads/foreign"), 2048); + std::os::unix::fs::symlink(&outside, &profile).expect("linked profile root"); + let overrides = BTreeMap::from([("lcm-payloads".to_owned(), 1)]); + + let linked = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(linked.total_bytes, 0, "foreign bytes must not be owned"); + assert_eq!(linked.unavailable_entry_count, 1); + assert_eq!(linked.entries, Vec::::new()); + assert!(!linked.truncated); + assert_eq!( + linked.full_profile_size().state, + ProfileTotalCoverageStateV1::Partial + ); + let linked_findings = findings(&linked); + assert_eq!(linked_findings.len(), 1); + assert_eq!( + linked_findings["profile"].finding().state(), + DoctorEvidenceStateV1::Partial, + "unverifiable root must not be reported healthy" + ); + + // The same bytes must be measured and over-budget when the actual + // directory is explicitly selected instead of its symbolic link. + let owned = census_profile_footprint(&outside, &BTreeMap::new(), &overrides); + assert_eq!(owned.total_bytes, 2048); + assert_eq!(owned.unavailable_entry_count, 0); + assert_eq!(entry(&owned, "lcm-payloads").bytes, 2048); + assert_eq!( + findings(&owned)["lcm-payloads"].finding().state(), + DoctorEvidenceStateV1::Degraded + ); + } + + #[test] + fn sqlite_wal_lookalike_directories_keep_independent_bounds() { + let temp = tempfile::tempdir().expect("footprint fixture"); + let profile = temp.path().join("profile"); + for (relative, bytes) in [ + ("global.db", 5), + ("global.db-wal/incident.bin", 11), + ("scratch.wal/chunk", 13), + ("user-sessions.db-wal", 7), + ("projects/proj_wal/sessions.db-wal", 17), + ("projects/proj_wal/tracedecay.db-wal/incident.bin", 19), + ("projects/proj_wal/tracedecay.grafeo.wal/segment", 23), + ] { + sized(&profile.join(relative), bytes); + } + let overrides = BTreeMap::from([ + ("global.db-wal".to_owned(), 10), + ("scratch.wal".to_owned(), 12), + ("projects/proj_wal".to_owned(), 58), + ("user-sessions.db-wal".to_owned(), 7), + ("projects/proj_wal/sessions.db-wal".to_owned(), 17), + ("projects/proj_wal/tracedecay.grafeo.wal".to_owned(), 23), + (PROFILE_TOTAL_KEY.to_owned(), 95), + ]); + let footprint = census_profile_footprint(&profile, &BTreeMap::new(), &overrides); + assert_eq!(footprint.total_bytes, 95); + assert_eq!(footprint.unavailable_entry_count, 0); + assert_eq!(footprint.entries.len(), 7); + assert_eq!( + footprint.full_profile_size().state, + ProfileTotalCoverageStateV1::Complete + ); + for (key, family, bytes) in [ + ("global.db", ProfileFamilyV1::ProfileRegistry, 5), + ("global.db-wal", ProfileFamilyV1::Other, 11), + ("scratch.wal", ProfileFamilyV1::Other, 13), + ("user-sessions.db-wal", ProfileFamilyV1::Wal, 7), + ("projects/proj_wal", ProfileFamilyV1::ProjectStore, 59), + ( + "projects/proj_wal/sessions.db-wal", + ProfileFamilyV1::Wal, + 17, + ), + ( + "projects/proj_wal/tracedecay.grafeo.wal", + ProfileFamilyV1::Wal, + 23, + ), + ] { + assert_eq!(entry(&footprint, key).family, family, "{key}"); + assert_eq!(entry(&footprint, key).bytes, bytes, "{key}"); + } + let kinds = entry(&footprint, "projects/proj_wal") + .kinds + .expect("production project-kind breakdown"); + assert_eq!(kinds.sessions, 17); + assert_eq!(kinds.graph_container, 23); + assert_eq!(kinds.other, 19); + assert_eq!(kinds.total(), 59); + + let observed = findings(&footprint); + assert_eq!(observed.len(), 8); + for key in ["global.db-wal", "scratch.wal", "projects/proj_wal"] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::Degraded, + "{key} must not evade its physical footprint bound" + ); + } + for key in [ + "global.db", + "user-sessions.db-wal", + "projects/proj_wal/sessions.db-wal", + "projects/proj_wal/tracedecay.grafeo.wal", + PROFILE_TOTAL_KEY, + ] { + assert_eq!( + observed[key].finding().state(), + DoctorEvidenceStateV1::HealthyCompleteCoverage, + "{key} must retain its real, independent bound" + ); + } + } + + #[test] + fn long_footprint_details_keep_doctor_size_and_bound_visible() { + let temporary = tempfile::tempdir().unwrap(); + let profile = temporary.path().join("profile"); + let key = "projects/proj_long"; + let bytes = 3 * 1024; + sized(&profile.join(key).join("tracedecay.db"), bytes); + let registered = BTreeMap::from([( + "proj_long".to_owned(), + temporary.path().display().to_string(), + )]); + let overrides = BTreeMap::from([(key.to_owned(), 1)]); + let mut footprint = census_profile_footprint(&profile, ®istered, &overrides); + let project = footprint + .entries + .iter_mut() + .find(|entry| entry.key == key) + .expect("measured project"); + assert_eq!( + project.project.as_ref().unwrap().state, + ProjectRootStateV1::Present + ); + // A short registered checkout path with control characters fits once + // folded by Doctor. Preserve its useful identity rather than dropping + // it just because the sanitized representation differs byte-for-byte. + project.project.as_mut().unwrap().registered_root = + Some("checkout\tbranch\nalpha".to_owned()); + let sanitized = findings(&footprint); + let sanitized_statement = sanitized[key].finding().coverage().statement(); + assert!( + sanitized_statement.contains("for checkout branch alpha (root present)"), + "sanitized registered checkout was omitted: {sanitized_statement}" + ); + assert!(!sanitized_statement.contains('\t'), "{sanitized_statement}"); + assert!(!sanitized_statement.contains('\n'), "{sanitized_statement}"); + assert!( + sanitized_statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), + "a sanitized identity must keep the exact measured overage: {sanitized_statement}" + ); + + // The complete state remains available in the storage report; only + // the Doctor coverage text is bounded. + let project = footprint + .entries + .iter_mut() + .find(|entry| entry.key == key) + .expect("measured project"); + project.project.as_mut().unwrap().registered_root = Some("long-root/".repeat(120)); + let observed = findings(&footprint); + let finding = &observed[key]; + let statement = finding.finding().coverage().statement(); + assert_eq!(finding.finding().state(), DoctorEvidenceStateV1::Degraded); + assert!(statement.contains("root present"), "{statement}"); + assert!( + statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), + "the measured overage must not be clipped after a long root: {statement}" + ); + assert!(!statement.contains("long-root/long-root"), "{statement}"); + + // A valid long evidence key can also fill the statement budget before + // the numeric overage. Hash the reference, never the reported bytes. + let project = footprint + .entries + .iter_mut() + .find(|entry| entry.key == key) + .expect("measured project"); + project.key = format!("projects/proj_{}/graph.grafeo", "x".repeat(520)); + let hash_key = format!("footprint/sha256:{}", sha256_hex(project.key.as_bytes())); + let observed = findings(&footprint); + let finding = &observed[&hash_key]; + let statement = finding.finding().coverage().statement(); + assert_eq!(finding.finding().state(), DoctorEvidenceStateV1::Degraded); + assert!( + statement.contains("full path in storage report"), + "{statement}" + ); + assert!( + statement.ends_with(&format!("{} of 1 B bound", format_bytes(bytes))), + "the complete numeric bound must survive a long evidence key: {statement}" + ); + assert_eq!( + std::fs::metadata(profile.join(key).join("tracedecay.db")) + .unwrap() + .len(), + bytes + ); + } } diff --git a/crates/tracedecay-maintenance/src/retention/storage_report.rs b/crates/tracedecay-maintenance/src/retention/storage_report.rs index cea9e019211..fdb5d7f5a61 100644 --- a/crates/tracedecay-maintenance/src/retention/storage_report.rs +++ b/crates/tracedecay-maintenance/src/retention/storage_report.rs @@ -27,8 +27,12 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::io::Read; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; + use tracedecay_domain::errors::TraceDecayError; use tracedecay_domain::{ CodeGenerationId, CursorBindingMismatchV1, CursorBindingV1, decode_bound_cursor, @@ -61,7 +65,7 @@ const CODE_GENERATIONS_DIRECTORY: &str = "code-generations-v1"; const CODE_INDEX_DIRECTORY: &str = "code-index-v1"; const SEALED_GRAPH_DIRECTORY: &str = "tracedecay.sealed"; const GRAPH_REPLAY_POOL_DIRECTORY: &str = "tracedecay.graph-replay"; -pub(crate) const GRAPH_CONTAINER_SUFFIX: &str = ".grafeo"; +const GRAPH_CONTAINER_SUFFIX: &str = ".grafeo"; pub(crate) const GRAPH_CONTAINER_WAL_SUFFIX: &str = ".grafeo.wal"; /// Why a report without the daemon's scheduler authority cannot plan a /// retention dry run. @@ -188,25 +192,42 @@ pub struct StoreKindBytesV1 { pub other: u64, } +/// A container is a regular file at the store root. Its recognized sidecars +/// can also be directories, whose descendants belong to the same container. +pub(super) fn graph_container_stem(name: &str, is_leaf: bool) -> Option<&str> { + let stem = name + .strip_suffix(".verified") + .or_else(|| name.strip_suffix(".spill")) + .or_else(|| is_leaf.then_some(name))?; + stem.ends_with(GRAPH_CONTAINER_SUFFIX).then_some(stem) +} + +// Only SQLite-owned sidecars belong to the database family. Incident debris, +// backups and unrelated adjacent files must remain visible as "other" bytes. +fn is_sqlite_family_member(name: &str, main: &str) -> bool { + name == main + || name + .strip_prefix(main) + .is_some_and(|suffix| matches!(suffix, "-wal" | "-shm" | "-journal")) +} + impl StoreKindBytesV1 { pub(crate) fn add(&mut self, relative: &Path, bytes: u64) { let mut components = relative .components() - .map(|component| component.as_os_str().to_string_lossy()); + .map(|component| component.as_os_str().to_string_lossy()) + .peekable(); let first = components.next().unwrap_or_default(); + let root_file = components.peek().is_none(); let db_filename = tracedecay_runtime_core::config::DB_FILENAME; - let family = if first == db_filename - || first - .strip_prefix(db_filename) - .is_some_and(|suffix| suffix.starts_with('-')) - { + let family = if root_file && is_sqlite_family_member(&first, db_filename) { &mut self.graph_database } else if first == SEALED_GRAPH_DIRECTORY { &mut self.sealed_graph } else if first == GRAPH_REPLAY_POOL_DIRECTORY { &mut self.graph_replay_pool } else if first.ends_with(GRAPH_CONTAINER_WAL_SUFFIX) - || (first.ends_with(GRAPH_CONTAINER_SUFFIX) && components.next().is_none()) + || graph_container_stem(&first, components.peek().is_none()).is_some() { &mut self.graph_container } else if first == CODE_INDEX_DIRECTORY { @@ -219,10 +240,12 @@ impl StoreKindBytesV1 { } else { &mut self.generation_artifacts } - } else if first.starts_with(SESSIONS_DB_FILENAME) + } else if (root_file && is_sqlite_family_member(&first, SESSIONS_DB_FILENAME)) + || first.strip_prefix(SESSIONS_DB_FILENAME) == Some(".delivery-settlement-spool-v1") || first .strip_prefix('.') - .is_some_and(|name| name.starts_with(SESSIONS_DB_FILENAME)) + .and_then(|name| name.strip_prefix(SESSIONS_DB_FILENAME)) + == Some(".host-admission") { &mut self.sessions } else { @@ -603,13 +626,28 @@ async fn build_storage_report_with_walk_budget( profile_root: &Path, walk_entry_budget: usize, ) -> tracedecay_domain::errors::Result { + reject_linked_profile_root(profile_root)?; let global_db_path = profile_root.join(GLOBAL_DB_FILENAME); // The registry rows come from an off-profile copy, so reading them first // leaves the profile untouched. - let registered_projects = if global_db_path.exists() { - load_registered_project_rows(profile_root, &global_db_path).await? - } else { - Vec::new() + // `Path::exists` hides permission errors and dangling symlinks as + // absence. A valid symlink is also not a profile-owned registry: following + // it would read a foreign database the footprint walker does not charge. + let registered_projects = match std::fs::symlink_metadata(&global_db_path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(report_error( + "inspect profile registry for storage report", + "registry path is a symbolic link; its target is outside the measured registry authority", + )); + } + Ok(_) => load_registered_project_rows(profile_root, &global_db_path).await?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => { + return Err(report_error( + "inspect profile registry for storage report", + error, + )); + } }; // Capture the profile's physical footprint before opening any SQLite // readers. A report must describe the profile supplied by its caller, not @@ -877,6 +915,7 @@ async fn build_storage_report_page_with_walk_budget( cancellation: Option, deadline: Option, ) -> tracedecay_domain::errors::Result { + reject_linked_profile_root(profile_root)?; let limit = limit.clamp(1, MAX_STORAGE_REPORT_PAGE_LIMIT); let walk_control = StorageReportWalkControl { cancellation, @@ -1212,6 +1251,7 @@ fn build_project_storage_report_with_control( sample_free_page_header: bool, walk_control: &StorageReportWalkControl, ) -> tracedecay_domain::errors::Result { + reject_linked_profile_root(profile_root)?; tracedecay_runtime_core::storage::validate_project_id(project_id).map_err(|message| { tracedecay_domain::errors::TraceDecayError::Config { message: message.to_owned(), @@ -1249,6 +1289,21 @@ fn build_project_storage_report_with_control( }) } +/// A caller-supplied profile symlink is not the authority that owns its target. +/// The physical footprint walker refuses that link; report readers must also +/// refuse it before inspecting registry rows or sampling individual stores. +fn reject_linked_profile_root(profile_root: &Path) -> tracedecay_domain::errors::Result<()> { + match std::fs::symlink_metadata(profile_root) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(report_error( + "inspect storage report profile root", + "profile root is a symbolic link; refusing to read foreign storage", + )), + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(report_error("inspect storage report profile root", error)), + } +} + fn directory_chain_is_nofollow(root: &Path, directory: &Path) -> bool { if !std::fs::symlink_metadata(root).is_ok_and(|metadata| metadata.is_dir()) { return false; @@ -1278,6 +1333,14 @@ fn append_project_report( remaining_entries: &mut usize, walk_control: &StorageReportWalkControl, ) -> tracedecay_domain::errors::Result<()> { + // Offline registry snapshots and live registry rows must obey the same + // path-segment authority as explicit project-scoped reports. Otherwise a + // corrupt project ID can escape projects/ and measure an unrelated tree. + tracedecay_runtime_core::storage::validate_project_id(project_id).map_err(|message| { + TraceDecayError::Config { + message: format!("invalid registered project id {project_id:?}: {message}"), + } + })?; let data_root = profile_root.join("projects").join(project_id); let code_index_root = data_root.join("code-index-v1"); let code_index_store_root = @@ -1493,8 +1556,14 @@ fn user_sessions_family_bytes(profile_root: &Path) -> u64 { fn database_family_bytes(database_path: &Path) -> u64 { let mut total_bytes = 0u64; - for suffix in ["", "-wal", "-shm"] { - if let Ok(metadata) = std::fs::metadata(sqlite_family_member(database_path, suffix)) { + // PERSIST rollback journals remain allocated after a committed transaction; + // TRUNCATE journals are zero-length. Both are physical family bytes. + for suffix in ["", "-wal", "-shm", "-journal"] { + // Physical profile bytes must not include a symlink's foreign target. + // The full-profile walk separately marks a linked member unavailable. + if let Ok(metadata) = std::fs::symlink_metadata(sqlite_family_member(database_path, suffix)) + && metadata.file_type().is_file() + { total_bytes = total_bytes.saturating_add(metadata.len()); } } @@ -1507,16 +1576,208 @@ fn database_family_bytes(database_path: &Path) -> u64 { fn copy_sqlite_family(source: &Path, destination: &Path) -> std::io::Result<()> { let generation = tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(source)?; + copy_sqlite_family_at_generation(source, destination, &generation) +} + +fn copy_sqlite_family_at_generation( + source: &Path, + destination: &Path, + generation: &tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration, +) -> std::io::Result<()> { + // Rollback journals can outlive commits in TRUNCATE and PERSIST mode. + // Only an absent, truncated, or explicitly invalidated journal is safe + // to omit from a snapshot. Recheck its witness after copying: a writer + // must not turn a stable registry into an uncommitted main-file image. + let journal_before = inspect_inert_rollback_journal(source)?; + // Refuse nonregular and linked source members before copying any bytes. + // Path::is_file() follows symlinks, potentially copying an arbitrarily + // large foreign file into the report's temporary SQLite snapshot. + let mut members = Vec::with_capacity(3); for suffix in ["", "-wal", "-shm"] { - let source_member = sqlite_family_member(source, suffix); - if !source_member.is_file() { - continue; + let member = sqlite_family_member(source, suffix); + match std::fs::symlink_metadata(&member) { + Ok(metadata) if metadata.file_type().is_file() => { + members.push((member, suffix, metadata)); + } + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "SQLite report source member '{}' is not a regular file; symbolic links are not allowed", + member.display() + ), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), } - std::fs::copy(&source_member, sqlite_family_member(destination, suffix))?; + } + copy_sqlite_members_nofollow(source, destination, &members)?; + let journal_after = inspect_inert_rollback_journal(source)?; + if !rollback_journal_unchanged(journal_before.as_ref(), journal_after.as_ref()) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "SQLite report rollback journal changed while copying registry", + )); } generation.validate() } +fn rollback_journal_uncommitted(path: &Path) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "SQLite report source has active or uncertain rollback journal '{}'; main/WAL/SHM snapshot is unsafe", + path.display() + ), + ) +} + +/// A committed TRUNCATE journal is empty; PERSIST clears the first 28 header +/// bytes instead of deleting the file. A zero magic prefix alone is NOT enough: +/// SQLite may leave those first eight bytes zero during an active transaction, +/// while the nonce and page/sector fields are already populated. +fn inspect_inert_rollback_journal(source: &Path) -> std::io::Result> { + let path = sqlite_family_member(source, "-journal"); + let checked = match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_file() => metadata, + Ok(_) => return Err(rollback_journal_uncommitted(&path)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if checked.len() == 0 { + return Ok(Some(checked)); + } + if checked.len() < 28 { + return Err(rollback_journal_uncommitted(&path)); + } + + let parent = source + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let directory = cap_std::fs::Dir::open_ambient_dir(parent, cap_std::ambient_authority())?; + let mut options = cap_std::fs::OpenOptions::new(); + options.read(true); + options.follow(FollowSymlinks::No); + let name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "SQLite rollback journal has no filename", + ) + })?; + let mut input = directory.open_with(name, &options)?.into_std(); + let opened = input.metadata()?; + if !matches_checked_sqlite_member(&checked, &opened) + || checked.modified()? != opened.modified()? + { + return Err(rollback_journal_uncommitted(&path)); + } + let mut header = [0u8; 28]; + input.read_exact(&mut header)?; + if header != [0u8; 28] { + return Err(rollback_journal_uncommitted(&path)); + } + let after_read = input.metadata()?; + if !matches_checked_sqlite_member(&checked, &after_read) + || checked.modified()? != after_read.modified()? + { + return Err(rollback_journal_uncommitted(&path)); + } + Ok(Some(checked)) +} + +fn rollback_journal_unchanged( + before: Option<&std::fs::Metadata>, + after: Option<&std::fs::Metadata>, +) -> bool { + match (before, after) { + (None, None) => true, + (Some(before), Some(after)) => { + matches_checked_sqlite_member(before, after) + && before + .modified() + .ok() + .zip(after.modified().ok()) + .is_some_and(|(left, right)| left == right) + } + _ => false, + } +} + +fn matches_checked_sqlite_member(checked: &std::fs::Metadata, opened: &std::fs::Metadata) -> bool { + if !opened.is_file() || checked.len() != opened.len() { + return false; + } + // A no-follow open blocks symlink swaps but not regular-file replacement. + // Revalidate the held file's identity before creating any scratch output. + #[cfg(unix)] + { + checked.dev() == opened.dev() && checked.ino() == opened.ino() + } + #[cfg(not(unix))] + { + // A size-changing swap is rejected above. On platforms without the + // Unix inode API, compare timestamps and let the generation witness + // independently reject a changed source after copying. + checked + .created() + .ok() + .zip(opened.created().ok()) + .zip(checked.modified().ok().zip(opened.modified().ok())) + .is_some_and(|((created, opened_created), (modified, opened_modified))| { + created == opened_created && modified == opened_modified + }) + } +} + +/// Open the checked member through a held directory, refusing symlink or +/// regular-file replacement after preflight. A pathname copy would import +/// unrelated bytes into report scratch even if later validation rejected it. +fn copy_sqlite_members_nofollow( + source: &Path, + destination: &Path, + members: &[(PathBuf, &str, std::fs::Metadata)], +) -> std::io::Result<()> { + let parent = source + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let directory = cap_std::fs::Dir::open_ambient_dir(parent, cap_std::ambient_authority())?; + let mut options = cap_std::fs::OpenOptions::new(); + options.read(true); + options.follow(FollowSymlinks::No); + for (member, suffix, checked) in members { + let name = member.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "SQLite member has no filename", + ) + })?; + let input = directory.open_with(name, &options)?.into_std(); + if !matches_checked_sqlite_member(checked, &input.metadata()?) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "SQLite report source member '{}' changed after preflight", + member.display() + ), + )); + } + let mut output = std::fs::File::create(sqlite_family_member(destination, suffix))?; + // A live WAL may grow after inspection; do not copy an unbounded tail. + let cap = checked.len().saturating_add(1); + let copied = std::io::copy(&mut input.take(cap), &mut output)?; + if copied != checked.len() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "SQLite report source member changed size during copy", + )); + } + } + Ok(()) +} + fn validate_external_scratch(profile_root: &Path, scratch_root: &Path) -> std::io::Result<()> { let profile = profile_root.canonicalize()?; let scratch = scratch_root.canonicalize()?; @@ -1759,6 +2020,66 @@ mod tests { use crate::retention::footprint::census_profile_footprint; use tracedecay_runtime_core::db::engine::TestConnection; + #[test] + fn registered_report_refuses_project_ids_that_escape_the_profile() { + let temp = tempfile::tempdir().unwrap(); + let profile = temp.path().join("profile"); + let outside = temp.path().join("outside"); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::create_dir_all(profile.join("projects/proj_valid")).unwrap(); + std::fs::write(outside.join("foreign"), [7u8; 16]).unwrap(); + std::fs::write(profile.join("projects/proj_valid/local"), [3u8; 9]).unwrap(); + + let mut stores = Vec::new(); + let mut retention = Vec::new(); + let mut availability = Vec::new(); + let mut budget = MAX_STORAGE_REPORT_WALK_ENTRIES; + for project_id in [ + "../../outside", + "../outside", + "nested/store", + "..", + ".", + "", + "bad\\name", + ] { + let error = append_project_report( + &profile, + project_id, + "/work/valid", + Err(RETENTION_PROTECTION_UNRESOLVED), + &mut stores, + &mut retention, + &mut availability, + false, + &mut budget, + &StorageReportWalkControl::default(), + ) + .unwrap_err(); + assert!( + error.to_string().contains("invalid registered project id"), + "{project_id:?}: {error}" + ); + assert!(stores.is_empty(), "foreign storage was adopted"); + assert_eq!(budget, MAX_STORAGE_REPORT_WALK_ENTRIES); + } + assert_eq!(std::fs::read(outside.join("foreign")).unwrap(), [7u8; 16]); + + let valid = build_project_storage_report( + &profile, + "proj_valid", + Path::new("/work/valid"), + Err(RETENTION_PROTECTION_UNRESOLVED), + ) + .unwrap(); + assert_eq!(valid.stores.len(), 1); + assert_eq!(valid.stores[0].total_bytes, 9); + assert_eq!( + valid.stores[0].size_coverage.state, + StorageReportSizeStateV1::Complete + ); + } + fn store(project_id: &str, total_bytes: u64) -> StoreSizeReportEntry { StoreSizeReportEntry { project_id: project_id.to_owned(), @@ -2446,6 +2767,574 @@ mod tests { ); } + #[tokio::test] + async fn committed_persist_journal_is_charged_to_user_sessions_family() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let database = profile_root.join(USER_SESSIONS_DB_FILENAME); + let writer = rusqlite::Connection::open(&database).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = PERSIST; + CREATE TABLE receipts(payload BLOB NOT NULL); + INSERT INTO receipts VALUES (zeroblob(262144));", + ) + .unwrap(); + + let journal = sqlite_family_member(&database, "-journal"); + let main_bytes = std::fs::metadata(&database).unwrap().len(); + let journal_bytes = std::fs::metadata(&journal).unwrap().len(); + assert!(journal_bytes > 0, "PERSIST must retain a committed journal"); + let retained_header = std::fs::read(&journal).unwrap(); + assert_eq!( + &retained_header[..28], + &[0u8; 28], + "the retained journal must be committed, not active" + ); + + let report = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(report.user_sessions_db_bytes, main_bytes + journal_bytes); + let footprint = report.full_profile_size.unwrap(); + assert_eq!(footprint.total_bytes, main_bytes + journal_bytes); + assert_eq!(footprint.state, ProfileTotalCoverageStateV1::Complete); + } + + #[cfg(unix)] + #[test] + fn linked_rollback_journal_does_not_charge_foreign_storage() { + let temporary = tempfile::TempDir::new().unwrap(); + let database = temporary.path().join("owned.db"); + std::fs::write(&database, b"owned").unwrap(); + let foreign = temporary.path().join("foreign.bin"); + std::fs::write(&foreign, vec![0u8; 1024 * 1024]).unwrap(); + let journal = sqlite_family_member(&database, "-journal"); + std::os::unix::fs::symlink(&foreign, &journal).unwrap(); + + assert_eq!(database_family_bytes(&database), 5); + } + + #[cfg(unix)] + #[tokio::test] + async fn dangling_profile_registry_cannot_make_registered_stores_look_unregistered() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + let project = profile_root.join("projects/proj_retained"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("sessions.db"), b"retained sessions").unwrap(); + + let registry = profile_root.join(GLOBAL_DB_FILENAME); + std::os::unix::fs::symlink(temporary.path().join("missing-global.db"), ®istry).unwrap(); + assert!(std::fs::symlink_metadata(®istry).is_ok()); + assert!( + !registry.exists(), + "the old existence test silently lost this registry" + ); + + let error = build_storage_report(&profile_root) + .await + .expect_err("a broken registry link must not imply zero registered projects"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "the unreadable registry must have a typed storage failure: {error}" + ); + + std::fs::remove_file(®istry).unwrap(); + let absent = build_storage_report(&profile_root) + .await + .expect("a genuinely absent registry is still reportable"); + assert_eq!(absent.stores, Vec::::new()); + assert_eq!(absent.unregistered_dir_count, 1); + assert!(absent.unregistered_bytes >= 17); + } + + #[cfg(unix)] + #[tokio::test] + async fn linked_foreign_registry_is_not_misattributed_to_the_profile() { + let temporary = tempfile::TempDir::new().unwrap(); + let foreign_root = temporary.path().join("foreign"); + std::fs::create_dir_all(&foreign_root).unwrap(); + seed_global_db(&foreign_root, &[("proj_foreign", "/repos/foreign")]).await; + + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let registry = profile_root.join(GLOBAL_DB_FILENAME); + std::os::unix::fs::symlink(foreign_root.join(GLOBAL_DB_FILENAME), ®istry).unwrap(); + assert!( + registry.is_file(), + "the old path probe follows the foreign registry" + ); + + let error = build_storage_report(&profile_root) + .await + .expect_err("a foreign registry cannot supply project identity for this profile"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "the symlink must be refused as a typed storage error: {error}" + ); + assert!( + error.to_string().contains("symbolic link"), + "the error must name the profile registry's disallowed link: {error}" + ); + } + + #[test] + fn registry_snapshot_refuses_rollback_journal_and_recovers_after_rollback() { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, canonical_root TEXT NOT NULL + ); + INSERT INTO code_projects VALUES ('proj_committed', '/repos/committed'); + BEGIN IMMEDIATE; + UPDATE code_projects SET canonical_root = '/repos/uncommitted' + WHERE project_id = 'proj_committed';", + ) + .unwrap(); + let journal = sqlite_family_member(®istry, "-journal"); + assert!( + journal.is_file(), + "the uncommitted update needs an undo journal" + ); + + let destination = temporary.path().join("snapshot.db"); + let error = copy_sqlite_family(®istry, &destination) + .expect_err("a main-only snapshot must not read uncommitted rollback pages"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains("rollback journal"), "{error}"); + assert!(!destination.exists(), "refusal must happen before any copy"); + + writer.execute_batch("ROLLBACK").unwrap(); + drop(writer); + assert!( + !journal.exists(), + "DELETE-mode rollback removes the journal" + ); + copy_sqlite_family(®istry, &destination).unwrap(); + let snapshot = rusqlite::Connection::open(&destination).unwrap(); + let root: String = snapshot + .query_row( + "SELECT canonical_root FROM code_projects WHERE project_id = 'proj_committed'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(root, "/repos/committed"); + } + + #[tokio::test] + async fn offline_report_refuses_rollback_journal_instead_of_misclassifying_stores() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + let project = profile_root.join("projects/proj_committed"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("sessions.db"), b"retained session").unwrap(); + let registry = profile_root.join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch( + "PRAGMA journal_mode = DELETE; + CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, canonical_root TEXT NOT NULL + ); + INSERT INTO code_projects VALUES ('proj_committed', '/repos/committed'); + BEGIN IMMEDIATE; + UPDATE code_projects SET canonical_root = '/repos/uncommitted' + WHERE project_id = 'proj_committed';", + ) + .unwrap(); + assert!(sqlite_family_member(®istry, "-journal").is_file()); + + let error = build_storage_report(&profile_root) + .await + .expect_err("an active rollback journal must not become registration truth"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "{error}" + ); + assert!(error.to_string().contains("rollback journal"), "{error}"); + + writer.execute_batch("ROLLBACK").unwrap(); + drop(writer); + let report = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(report.stores.len(), 1); + assert_eq!(report.stores[0].project_id, "proj_committed"); + assert_eq!(report.stores[0].canonical_root, "/repos/committed"); + assert_eq!(report.unregistered_dir_count, 0); + } + + #[tokio::test] + async fn committed_persist_and_truncate_journals_remain_reportable_but_active_writes_refuse() { + for mode in ["PERSIST", "TRUNCATE"] { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + let project = profile_root.join("projects/proj_committed"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("sessions.db"), b"retained session").unwrap(); + + let registry = profile_root.join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch(&format!( + "PRAGMA journal_mode = {mode}; + CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, canonical_root TEXT NOT NULL + ); + INSERT INTO code_projects VALUES ('proj_committed', '/repos/committed');" + )) + .unwrap(); + let journal = sqlite_family_member(®istry, "-journal"); + let journal_bytes = std::fs::metadata(&journal).unwrap().len(); + if mode == "TRUNCATE" { + assert_eq!(journal_bytes, 0, "TRUNCATE retains an empty journal"); + } else { + assert!( + journal_bytes >= 28, + "PERSIST retains an invalidated journal" + ); + let mut header = [0u8; 28]; + std::fs::File::open(&journal) + .unwrap() + .read_exact(&mut header) + .unwrap(); + assert_eq!(header, [0u8; 28]); + } + + let committed = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(committed.stores.len(), 1); + assert_eq!(committed.stores[0].project_id, "proj_committed"); + assert_eq!(committed.stores[0].canonical_root, "/repos/committed"); + assert_eq!(committed.unregistered_dir_count, 0); + + writer + .execute_batch( + "BEGIN IMMEDIATE; + UPDATE code_projects SET canonical_root = '/repos/uncommitted' + WHERE project_id = 'proj_committed';", + ) + .unwrap(); + assert!( + std::fs::metadata(&journal).unwrap().len() >= 28, + "the active rollback transaction must have a journal header" + ); + let error = build_storage_report(&profile_root) + .await + .expect_err("active rollback journal must not leak uncommitted registrations"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "report refusal must be a typed storage error: {error}" + ); + assert!(error.to_string().contains("rollback journal"), "{error}"); + + writer.execute_batch("ROLLBACK").unwrap(); + let recovered = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(recovered.stores.len(), 1); + assert_eq!(recovered.stores[0].canonical_root, "/repos/committed"); + assert_eq!(recovered.unregistered_dir_count, 0); + drop(writer); + } + } + + #[cfg(unix)] + #[test] + fn registry_snapshot_refuses_foreign_sidecar_links_before_copying() { + for suffix in ["-wal", "-shm"] { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let connection = rusqlite::Connection::open(®istry).unwrap(); + connection + .execute_batch( + "CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, + canonical_root TEXT NOT NULL + );", + ) + .unwrap(); + drop(connection); + + let foreign = temporary.path().join("foreign.bin"); + let foreign_bytes = 512 * 1024 * 1024; + std::fs::File::create(&foreign) + .unwrap() + .set_len(foreign_bytes) + .unwrap(); + std::os::unix::fs::symlink(&foreign, sqlite_family_member(®istry, suffix)).unwrap(); + + let destination = temporary.path().join("snapshot.db"); + let error = copy_sqlite_family(®istry, &destination) + .expect_err("a linked SQLite family member cannot supply snapshot bytes"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains("symbolic links"), "{error}"); + assert!( + !destination.exists(), + "the invalid family must be refused before making a partial copy" + ); + assert_eq!(foreign.metadata().unwrap().len(), foreign_bytes); + } + } + + #[cfg(unix)] + #[test] + fn registry_snapshot_member_cannot_follow_a_post_preflight_link_swap() { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let wal = sqlite_family_member(®istry, "-wal"); + std::fs::write(&wal, b"original WAL member").unwrap(); + assert!( + std::fs::symlink_metadata(&wal) + .unwrap() + .file_type() + .is_file() + ); + // The production preflight recorded this pathname as a regular file. + let checked = vec![( + wal.clone(), + "-wal", + std::fs::symlink_metadata(&wal).unwrap(), + )]; + std::fs::remove_file(&wal).unwrap(); + let foreign = temporary.path().join("foreign-data.bin"); + std::fs::write(&foreign, vec![73u8; 4096]).unwrap(); + std::os::unix::fs::symlink(&foreign, &wal).unwrap(); + + let destination = temporary.path().join("report-snapshot.db"); + copy_sqlite_members_nofollow(®istry, &destination, &checked) + .expect_err("a swapped link must never supply snapshot bytes"); + assert!( + !sqlite_family_member(&destination, "-wal").exists(), + "a link swapped after preflight cannot be copied into report scratch" + ); + assert_eq!(std::fs::metadata(&foreign).unwrap().len(), 4096); + } + + #[cfg(unix)] + #[test] + fn registry_snapshot_refuses_regular_member_replacement_before_copying() { + for replacement_bytes in [4096_u64, 512 * 1024 * 1024] { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let wal = sqlite_family_member(®istry, "-wal"); + std::fs::write(&wal, vec![17u8; 4096]).unwrap(); + let checked = vec![( + wal.clone(), + "-wal", + std::fs::symlink_metadata(&wal).unwrap(), + )]; + + // Both files are regular, so a no-follow open still succeeds. + // Include a same-sized swap: checking size alone is insufficient. + let incoming = temporary.path().join("foreign-regular.bin"); + std::fs::File::create(&incoming) + .unwrap() + .set_len(replacement_bytes) + .unwrap(); + std::fs::rename(&incoming, &wal).unwrap(); + assert!( + std::fs::symlink_metadata(&wal) + .unwrap() + .file_type() + .is_file() + ); + + let destination = temporary.path().join("report-snapshot.db"); + let error = copy_sqlite_members_nofollow(®istry, &destination, &checked) + .expect_err("a regular member swapped after preflight must be refused"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!( + error.to_string().contains("changed after preflight"), + "{error}" + ); + assert!( + !sqlite_family_member(&destination, "-wal").exists(), + "the replacement must be refused before any bytes enter scratch" + ); + assert_eq!(std::fs::metadata(&wal).unwrap().len(), replacement_bytes); + } + } + + #[cfg(unix)] + #[test] + fn registry_snapshot_rejects_wal_committed_after_generation_capture() { + let temporary = tempfile::TempDir::new().unwrap(); + let registry = temporary.path().join(GLOBAL_DB_FILENAME); + let writer = rusqlite::Connection::open(®istry).unwrap(); + writer + .execute_batch( + "CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, + canonical_root TEXT NOT NULL + );", + ) + .unwrap(); + let generation = + tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(®istry) + .unwrap(); + // A new committed WAL must invalidate the preexisting generation. + // The old order captured membership before the generation and then + // silently copied only the main database, losing this project. + writer + .execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + INSERT INTO code_projects VALUES ('proj_new', '/repos/new');", + ) + .unwrap(); + let wal = sqlite_family_member(®istry, "-wal"); + assert!(wal.metadata().unwrap().len() > 0); + + let destination = temporary.path().join("report-snapshot.db"); + let error = copy_sqlite_family_at_generation(®istry, &destination, &generation) + .expect_err("a post-capture WAL commit must invalidate the snapshot"); + assert!( + error.to_string().contains("changed after inspection"), + "the snapshot must fail closed instead of misclassifying new projects: {error}" + ); + let canonical_root: String = writer + .query_row( + "SELECT canonical_root FROM code_projects WHERE project_id = 'proj_new'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(canonical_root, "/repos/new"); + + // A new complete generation must still be usable on the same live + // registry; the rejection is for racing changes, not WAL mode itself. + let stable = + tracedecay_runtime_core::sqlite_read_snapshot::SourceGeneration::capture(®istry) + .unwrap(); + let stable_destination = temporary.path().join("stable-snapshot.db"); + copy_sqlite_family_at_generation(®istry, &stable_destination, &stable).unwrap(); + let snapshot = rusqlite::Connection::open(&stable_destination).unwrap(); + let snapshot_root: String = snapshot + .query_row( + "SELECT canonical_root FROM code_projects WHERE project_id = 'proj_new'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(snapshot_root, "/repos/new"); + } + + #[cfg(unix)] + #[tokio::test] + async fn offline_report_refuses_a_foreign_registry_wal_before_sampling() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + let registry = profile_root.join(GLOBAL_DB_FILENAME); + let connection = rusqlite::Connection::open(®istry).unwrap(); + connection + .execute_batch( + "CREATE TABLE code_projects ( + project_id TEXT PRIMARY KEY, + canonical_root TEXT NOT NULL + );", + ) + .unwrap(); + drop(connection); + + let foreign = temporary.path().join("foreign-wal.bin"); + std::fs::File::create(&foreign) + .unwrap() + .set_len(512 * 1024 * 1024) + .unwrap(); + std::os::unix::fs::symlink(&foreign, sqlite_family_member(®istry, "-wal")).unwrap(); + + let error = build_storage_report(&profile_root) + .await + .expect_err("a linked registry WAL must fail instead of copying a foreign file"); + assert!( + matches!(&error, TraceDecayError::Database { .. }), + "expected a typed storage report failure: {error}" + ); + assert!(error.to_string().contains("symbolic links"), "{error}"); + assert_eq!(foreign.metadata().unwrap().len(), 512 * 1024 * 1024); + } + + #[cfg(unix)] + #[tokio::test] + async fn user_sessions_family_excludes_foreign_wal_bytes_and_downgrades_coverage() { + let temporary = tempfile::TempDir::new().unwrap(); + let profile_root = temporary.path().join("profile"); + std::fs::create_dir_all(&profile_root).unwrap(); + seed_global_db(&profile_root, &[]).await; + std::fs::write( + profile_root.join(USER_SESSIONS_DB_FILENAME), + vec![1u8; 4_096], + ) + .unwrap(); + let foreign = temporary.path().join("foreign-sessions-wal.bin"); + std::fs::File::create(&foreign) + .unwrap() + .set_len(512 * 1024 * 1024) + .unwrap(); + std::os::unix::fs::symlink( + &foreign, + profile_root.join(format!("{USER_SESSIONS_DB_FILENAME}-wal")), + ) + .unwrap(); + + let report = build_storage_report(&profile_root).await.unwrap(); + assert_eq!(report.user_sessions_db_bytes, 4_096); + let profile_size = report.full_profile_size.expect("measured profile"); + assert_eq!(profile_size.state, ProfileTotalCoverageStateV1::Partial); + assert!( + profile_size.unavailable_entry_count > 0, + "the linked WAL must remain explicitly unavailable, not be counted as zero" + ); + assert_eq!(foreign.metadata().unwrap().len(), 512 * 1024 * 1024); + } + + #[cfg(unix)] + #[tokio::test] + async fn a_linked_profile_cannot_read_foreign_registry_or_store_bytes() { + let temporary = tempfile::tempdir().unwrap(); + let foreign = temporary.path().join("foreign"); + let linked = temporary.path().join("profile-link"); + std::fs::create_dir_all(foreign.join("projects/proj_foreign")).unwrap(); + std::fs::write(foreign.join("projects/proj_foreign/payload"), [1u8; 16]).unwrap(); + std::fs::write(foreign.join(USER_SESSIONS_DB_FILENAME), [2u8; 4_096]).unwrap(); + seed_global_db(&foreign, &[("proj_foreign", "/repos/foreign")]).await; + std::os::unix::fs::symlink(&foreign, &linked).unwrap(); + + // Before the guard, the footprint was partial, but offline sampling + // still followed this profile link and attributed foreign store bytes. + let offline = build_storage_report(&linked) + .await + .expect_err("offline report must not read linked profile storage"); + assert!(offline.to_string().contains("symbolic link"), "{offline}"); + + // The explicit project route also reports shared profile DB sizes, + // so it must not silently measure a linked profile either. + let project = build_project_storage_report( + &linked, + "proj_foreign", + Path::new("/repos/foreign"), + Err(RETENTION_PROTECTION_UNRESOLVED), + ) + .expect_err("project-scoped report must refuse linked profile storage"); + assert!(project.to_string().contains("symbolic link"), "{project}"); + + // Explicitly selecting the real directory remains measurable. The + // refusal is for a linked authority, not for the fixture's contents. + let real = build_storage_report(&foreign).await.unwrap(); + assert_eq!(real.user_sessions_db_bytes, 4_096); + assert_eq!(real.stores.len(), 1); + assert_eq!(real.stores[0].project_id, "proj_foreign"); + assert_eq!(real.stores[0].total_bytes, 16); + assert_eq!(real.unregistered_dir_count, 0); + assert_eq!(real.unregistered_bytes, 0); + let registry_bytes = std::fs::metadata(foreign.join(GLOBAL_DB_FILENAME)) + .unwrap() + .len(); + assert!(real.full_profile_size.unwrap().total_bytes >= registry_bytes + 4_112); + } + #[tokio::test] async fn storage_report_names_user_sessions_db_family() { let tmp = tempfile::TempDir::new().unwrap(); @@ -2757,9 +3646,18 @@ mod tests { ("tracedecay.graph-replay/generation.json", 40), ("tracedecay.grafeo", 500), ("tracedecay.grafeo.wal/wal_00000001.log", 60), + ("tracedecay.grafeo.verified", 80), + ("tracedecay.grafeo.spill/inflight", 120), ("sessions.db", 900), ("sessions.db-wal", 10), (".sessions.db.host-admission/claim", 5), + ("sessions.db.delivery-settlement-spool-v1/claim", 11), + ("tracedecay.db-archive", 17), + ("tracedecay.db-journal/incident.bin", 37), + ("sessions.db-shm/incident.bin", 41), + ("sessions.db-backup", 19), + ("sessions.db.corrupt-incident", 23), + (".sessions.db.host-admission-backup/claim", 13), ("hook-delivery-spool/claude/record", 7), ] { let path = data_root.join(relative); @@ -2795,13 +3693,13 @@ mod tests { sealed_graph: 3_000, text_artifacts: 2_050, generation_artifacts: 1_000, - graph_container: 560, + graph_container: 760, graph_replay_pool: 40, - sessions: 915, - other: 7, + sessions: 926, + other: 157, } ); - assert_eq!(store.total_bytes, 11_768); + assert_eq!(store.total_bytes, 12_129); assert_eq!(store.total_bytes, walked_bytes); assert_eq!(store.unavailable_entry_count, 0); } diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs index ecb10583f2b..49dbd66d7c3 100644 --- a/crates/tracedecay-maintenance/src/telemetry.rs +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -26,6 +26,8 @@ use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; use crate::tick::MaintenanceTickOutcome; +#[cfg(test)] +mod reclaim_tests; mod text_artifact_scans; use text_artifact_scans::TextArtifactScansV1; @@ -35,6 +37,9 @@ const STORAGE_TELEMETRY_USE_CASE: &str = "use-case.application.storage.telemetry const TABLE_GROWTH_STATE_DIRECTORY: &str = "maintenance"; const TABLE_GROWTH_STATE_FILE: &str = "table-growth-v1.json"; +/// The durable baseline is metadata, never an unbounded profile payload. +/// Leave room below the maintenance-family soft bound for other cursors. +const MAX_TABLE_GROWTH_STATE_BYTES: usize = 16 * 1024 * 1024; /// How long a walk that ran out of its deadline suppresses the next one. const TABLE_SIZE_WALK_REPROBE_MICROS: i64 = 24 * 60 * 60 * 1_000_000; @@ -44,22 +49,24 @@ struct TableWatermark { observed_at: UtcMicros, } -/// The store size at which a per-table `dbstat` walk last ran out of its -/// deadline. +/// The live-page count at which a per-table `dbstat` walk timed out. /// -/// The walk reads every page of every table, so its cost tracks the store -/// size: a multi-gigabyte session store needs minutes. Once a walk has run -/// out of time, repeating it on a store at least as large only spends the -/// deadline again. A smaller store, or a day-old mark, is walked again. -#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +/// Deleting data can reclaim table pages to SQLite's freelist without shrinking +/// the database file. `dbstat` no longer walks those pages, so the retry +/// decision uses allocated pages minus freelist pages instead of file size. +/// A store with fewer live pages, or a day-old mark, gets another measurement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] struct OverBudgetTableSizeWalk { + /// Allocated pages minus freelist pages when the walk timed out. page_count: u64, observed_at: UtcMicros, } impl OverBudgetTableSizeWalk { fn still_applies(self, page_count: u64, now: UtcMicros) -> bool { + // A regressed wall clock cannot extend a failed-walk hold indefinitely. page_count >= self.page_count + && now.0 >= self.observed_at.0 && now.0.saturating_sub(self.observed_at.0) < TABLE_SIZE_WALK_REPROBE_MICROS } } @@ -99,6 +106,10 @@ fn settle_table_size_walk( struct TableGrowthStateV1 { watermarks: Option>, latest: Option, + // A timed-out census describes the work cost of this executable, not a + // durable store limit. Restart must re-probe after reader improvements. + // Persist only actual samples and their watermarks. + #[serde(skip)] over_budget: Option, } @@ -205,10 +216,13 @@ impl GuardedStoreTelemetryPort { let over_budget = self.table_growth_state().over_budget; let walk = native_runtime::spawn_blocking(move || { let interrupted = || telemetry_interruption(&context); - let page_count = handle + let sample = handle .store_size_telemetry(reader_wait, interrupted) - .map_err(|error| error.table_growth_unavailable_reason())? - .page_count; + .map_err(|error| error.table_growth_unavailable_reason())?; + let page_count = sample + .page_count + .checked_sub(sample.freelist_pages) + .ok_or(TableGrowthUnavailableReasonV1::StorageOther)?; if over_budget.is_some_and(|walk| walk.still_applies(page_count, now_micros())) { return Ok(TableSizeWalk::Skipped); } @@ -564,7 +578,17 @@ impl StoreTelemetrySamplingRegistry { .table_growth_state_path .as_deref() .zip(path.to_str()) - .and_then(|(state_path, key)| load_table_growth_states(state_path).remove(key)) + .and_then(|(state_path, key)| match load_table_growth_states(state_path) { + Ok(mut states) => states.remove(key), + Err(error) => { + tracing::warn!( + path = %state_path.display(), + %error, + "table growth checkpoint is unavailable; preserving it and re-baselining this store" + ); + None + } + }) .unwrap_or_default(); let port = GuardedStoreTelemetryPort::new( handle, @@ -881,11 +905,31 @@ impl StoreTelemetrySamplingRegistry { /// Every persisted store state, keyed by database path. A missing or /// unreadable file is an empty state: the next pass re-baselines. -fn load_table_growth_states(state_path: &Path) -> BTreeMap { - std::fs::read(state_path) - .ok() - .and_then(|bytes| serde_json::from_slice(&bytes).ok()) - .unwrap_or_default() +fn load_table_growth_states( + state_path: &Path, +) -> std::io::Result> { + // A stale or substituted maintenance file need not respect the size of + // data this daemon normally writes. Do not read a sparse multi-GB file, + // follow a foreign symlink, or wait forever on a FIFO during startup. + // Refuse unsafe reads rather than replacing their sole existing bytes. + let bytes = match tracedecay_private_fs::framed_log::read_bounded( + state_path, + MAX_TABLE_GROWTH_STATE_BYTES, + )? { + Some(bytes) => bytes, + None => return Ok(BTreeMap::new()), + }; + match serde_json::from_slice(&bytes) { + Ok(states) => Ok(states), + Err(error) => { + tracing::warn!( + path = %state_path.display(), + %error, + "table growth checkpoint is invalid; measurements will re-baseline" + ); + Ok(BTreeMap::new()) + } + } } /// Merge `updates` into the persisted state and drop stores whose database no @@ -899,10 +943,16 @@ fn persist_table_growth_states( state_path: &Path, updates: Vec<(String, TableGrowthStateV1)>, ) -> std::io::Result<()> { - let mut states = load_table_growth_states(state_path); + let mut states = load_table_growth_states(state_path)?; states.extend(updates); states.retain(|database_path, _| Path::new(database_path).exists()); let bytes = serde_json::to_vec(&states).map_err(std::io::Error::other)?; + if bytes.len() > MAX_TABLE_GROWTH_STATE_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "table growth checkpoint exceeds the bounded maintenance-state size", + )); + } if let Some(parent) = state_path.parent() { std::fs::create_dir_all(parent)?; } @@ -962,6 +1012,98 @@ fn storage_telemetry_request_context( ) } +#[cfg(test)] +#[allow(clippy::expect_used, clippy::panic)] +mod table_growth_checkpoint_tests { + use super::{ + MAX_TABLE_GROWTH_STATE_BYTES, TableGrowthStateV1, load_table_growth_states, + persist_table_growth_states, + }; + + #[test] + fn checkpoint_reads_refuse_oversized_and_foreign_files_without_losing_valid_state() { + let temp = tempfile::tempdir().expect("isolated checkpoint fixture"); + let database = temp.path().join("project.db"); + std::fs::write(&database, b"owned").expect("existing database identity"); + let checkpoint = temp.path().join("maintenance/table-growth-v1.json"); + let key = database.to_str().expect("UTF-8 fixture path").to_owned(); + persist_table_growth_states( + &checkpoint, + vec![(key.clone(), TableGrowthStateV1::default())], + ) + .expect("persist genuine bounded state"); + let loaded = load_table_growth_states(&checkpoint).expect("valid checkpoint readback"); + assert_eq!(loaded.len(), 1); + assert!(loaded.contains_key(&key)); + + // A sparse file exercises the held-file length refusal, without + // allocating or reading an oversized JSON payload in the test. + std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(&checkpoint) + .expect("replace disposable checkpoint") + .set_len(MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1) + .expect("sparse oversized checkpoint"); + assert!(matches!( + load_table_growth_states(&checkpoint), + Err(error) if error.kind() == std::io::ErrorKind::InvalidData + )); + assert!( + persist_table_growth_states( + &checkpoint, + vec![(key.clone(), TableGrowthStateV1::default())], + ) + .is_err(), + "an oversized existing checkpoint must not be replaced" + ); + assert_eq!( + std::fs::metadata(&checkpoint).unwrap().len(), + MAX_TABLE_GROWTH_STATE_BYTES as u64 + 1 + ); + + #[cfg(unix)] + { + let foreign = temp.path().join("foreign-state.json"); + std::fs::write( + &foreign, + serde_json::to_vec(&std::collections::BTreeMap::from([( + key.clone(), + TableGrowthStateV1::default(), + )])) + .expect("valid foreign state"), + ) + .expect("foreign contents"); + std::fs::remove_file(&checkpoint).expect("remove disposable oversized file"); + std::os::unix::fs::symlink(&foreign, &checkpoint) + .expect("substitute foreign checkpoint symlink"); + assert!( + matches!( + load_table_growth_states(&checkpoint), + Err(error) if error.kind() == std::io::ErrorKind::InvalidInput + ), + "the previous std::fs::read would import this valid foreign state" + ); + assert!( + persist_table_growth_states( + &checkpoint, + vec![(key, TableGrowthStateV1::default())], + ) + .is_err(), + "a foreign symlink must never be replaced by persistence" + ); + assert!( + checkpoint + .symlink_metadata() + .unwrap() + .file_type() + .is_symlink() + ); + assert!(std::fs::read(&foreign).unwrap().len() > 2); + } + } +} + #[cfg(test)] mod retention_tick_log_tests { use super::StoreTelemetrySamplingRegistry; @@ -1323,6 +1465,98 @@ mod table_growth_tests { "a day-old over-budget mark is probed again" ); assert_eq!(port.table_growth_state().over_budget, None); + + // A prior timed-out walk can have a future wall-clock timestamp after + // time synchronization moves backward. It must not hide table growth + // for the entire clock skew plus the normal retry interval. + mark( + 1, + UtcMicros( + now_micros() + .0 + .saturating_add(super::TABLE_SIZE_WALK_REPROBE_MICROS), + ), + ); + assert!( + matches!( + port.table_growth(&context, &store).await, + TableGrowthTelemetryReadV1::Observed { .. } + ), + "a future timeout marker must not suppress a live measurement" + ); + assert_eq!(port.table_growth_state().over_budget, None); + } + + #[tokio::test] + async fn freed_pages_retry_a_timed_out_census_without_file_compaction() { + let temporary = tempfile::tempdir().expect("telemetry fixture root"); + let (database, store, port) = + fixture_port(&temporary.path().join("project.db"), Duration::from_secs(5)).await; + let context = storage_telemetry_request_context(port.scope.clone()).expect("context"); + + database + .execute_write_batch( + "grow a disposable table before retention", + "WITH RECURSIVE n(i) AS ( + SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 8 + ) INSERT INTO growth_payload SELECT zeroblob(1048576) FROM n;", + ) + .await + .expect("fill the table with real overflow pages"); + let StorageTelemetryReadV1::Observed { sample: before } = + port.store_size(&context, &store).await + else { + panic!("the grown store must expose its page and freelist counts"); + }; + let live_pages_before = before.page_count - before.freelist_pages; + assert!( + live_pages_before > 1024, + "the fixture must have many live pages" + ); + + port.table_growth + .lock() + .expect("table growth state") + .over_budget = Some(OverBudgetTableSizeWalk { + page_count: live_pages_before, + observed_at: now_micros(), + }); + assert!(matches!( + port.table_growth(&context, &store).await, + TableGrowthTelemetryReadV1::Unknown { + reason: TableGrowthUnavailableReasonV1::ExceedsMeasurementBudget, + .. + } + )); + + database + .execute_write_batch( + "reclaim the disposable table without vacuum", + "DELETE FROM growth_payload;", + ) + .await + .expect("reclaim the table's overflow pages"); + let StorageTelemetryReadV1::Observed { sample: after } = + port.store_size(&context, &store).await + else { + panic!("the reclaimed store must expose its page and freelist counts"); + }; + assert_eq!( + after.page_count, before.page_count, + "the database did not shrink, so allocated-page retry suppression would still apply" + ); + assert!(after.freelist_pages > before.freelist_pages); + assert!(after.page_count - after.freelist_pages < live_pages_before); + + let retried = port.table_growth(&context, &store).await; + assert!( + matches!( + retried, + TableGrowthTelemetryReadV1::BaselineEstablished { .. } + ), + "the cheaper post-reclamation census must run without a restart: {retried:?}" + ); + assert_eq!(port.table_growth_state().over_budget, None); } } @@ -1331,10 +1565,15 @@ mod table_growth_tests { mod table_growth_restart_tests { use std::collections::BTreeSet; - use tracedecay_contracts::storage::TableGrowthTelemetryReadV1; + use tracedecay_contracts::now_micros; + use tracedecay_contracts::storage::{ + TableGrowthTelemetryReadV1, TableGrowthUnavailableReasonV1, + }; use tracedecay_runtime_core::cancellation::CancellationToken; - use super::{StoreTelemetrySamplingRegistry, storage_telemetry_request_context}; + use super::{ + OverBudgetTableSizeWalk, StoreTelemetrySamplingRegistry, storage_telemetry_request_context, + }; #[tokio::test] async fn a_restarted_registry_reports_the_persisted_measurement() { @@ -1396,6 +1635,93 @@ mod table_growth_restart_tests { .all(|sample| sample.previous_observed_at == observed_at) ); } + #[tokio::test] + async fn restart_reprobes_a_timed_out_census_without_losing_the_baseline() { + let temporary = tempfile::tempdir().expect("profile root"); + let profile_root = temporary.path().join("profile"); + let database_path = temporary.path().join("project.db"); + let (database, scope) = super::table_growth_tests::growth_fixture(&database_path).await; + let paths = BTreeSet::from([database_path.clone()]); + let before = StoreTelemetrySamplingRegistry::persisted_under(&profile_root); + assert!(before.register_port(&database_path, &scope, || { + database.storage_telemetry_handle() + })); + before + .advance_registered(&paths, &paths, &CancellationToken::new()) + .await; + let context = storage_telemetry_request_context(scope.clone()).expect("reader context"); + let (store, port) = before + .registered_port(&database_path, &scope) + .expect("registered port"); + let TableGrowthTelemetryReadV1::BaselineEstablished { observed_at, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("first maintenance pass must establish a baseline"); + }; + + database + .execute_write_batch( + "grow retained table after baseline", + "INSERT INTO growth_payload VALUES (zeroblob(32768));", + ) + .await + .expect("insert a nonzero measured payload"); + port.table_growth + .lock() + .expect("table growth state") + .over_budget = Some(OverBudgetTableSizeWalk { + page_count: 1, + observed_at: now_micros(), + }); + before + .advance_registered(&paths, &paths, &CancellationToken::new()) + .await; + let skipped = port.latest_table_growth(&context, &store); + assert!(matches!( + skipped, + TableGrowthTelemetryReadV1::Unknown { + reason: TableGrowthUnavailableReasonV1::ExceedsMeasurementBudget, + .. + } + )); + assert!(port.table_growth_state().over_budget.is_some()); + drop(port); + drop(before); + + let after = StoreTelemetrySamplingRegistry::persisted_under(&profile_root); + assert!(after.register_port(&database_path, &scope, || { + database.storage_telemetry_handle() + })); + let (store, port) = after + .registered_port(&database_path, &scope) + .expect("registered port after restart"); + assert_eq!( + port.latest_table_growth(&context, &store), + skipped, + "the last unavailability is retained until a real new measurement" + ); + assert_eq!( + port.table_growth_state().over_budget, + None, + "a previous executable's timed-out census must not suppress the new one" + ); + + after + .advance_registered(&paths, &paths, &CancellationToken::new()) + .await; + let TableGrowthTelemetryReadV1::Observed { samples, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("restart must retry and compare to the retained baseline"); + }; + let grown = samples + .iter() + .find(|sample| sample.table.as_str() == "growth_payload") + .expect("the retried census reports real table bytes"); + assert_eq!(grown.previous_observed_at, observed_at); + assert!(grown.growth_bytes().get() >= 32_768); + assert_eq!(port.table_growth_state().over_budget, None); + } } #[cfg(test)] @@ -1502,8 +1828,8 @@ mod registry_tests { assert!(registry.graph_replay_release_attempt_admitted(project)); assert!(registry.graph_replay_release_attempt_admitted(project)); - // Consecutive unhealthy attempts widen the skip window 1, 2, 4, and - // cap at GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS denied ticks. + // Consecutive unhealthy attempts open a bounded skip window: + // 1, 2, 4, and capped at GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS. for expected_skips in [1_usize, 2, 4, 8, 8] { registry.record_graph_replay_release_unhealthy(project); let mut denied = 0_usize; diff --git a/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs b/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs new file mode 100644 index 00000000000..89df8464370 --- /dev/null +++ b/crates/tracedecay-maintenance/src/telemetry/reclaim_tests.rs @@ -0,0 +1,151 @@ +//! Retry real table measurements after logical reclamation, without compaction. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::collections::BTreeSet; + +use tracedecay_contracts::now_micros; +use tracedecay_contracts::storage::{ + StorageTelemetryReadV1, StoreSizeTelemetryPort, TableGrowthTelemetryReadV1, + TableGrowthUnavailableReasonV1, +}; +use tracedecay_runtime_core::cancellation::CancellationToken; + +use super::table_growth_tests::growth_fixture; +use super::{ + StoreTelemetrySamplingRegistry, TableSizeWalk, settle_table_size_walk, + storage_telemetry_request_context, +}; + +#[tokio::test] +async fn logical_reclamation_retries_a_timed_out_census_without_shrinking_the_file() { + let temporary = tempfile::tempdir().expect("telemetry reclamation fixture"); + let database_path = temporary.path().join("project.db"); + let (database, scope) = growth_fixture(&database_path).await; + database + .execute_write_batch( + "grow reclaimable payload", + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM n WHERE x < 32) \ + INSERT INTO growth_payload SELECT zeroblob(262144) FROM n;", + ) + .await + .expect("grow a real table before its baseline"); + + let registry = StoreTelemetrySamplingRegistry::default(); + assert!(registry.register_port(&database_path, &scope, || { + database.storage_telemetry_handle() + })); + let paths = BTreeSet::from([database_path.clone()]); + let cancellation = CancellationToken::new(); + let baseline = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(baseline.observed, 1); + assert_eq!(baseline.unavailable, 0); + let (store, port) = registry + .registered_port(&database_path, &scope) + .expect("registered measurement port"); + let context = storage_telemetry_request_context(scope.clone()).expect("reader context"); + let TableGrowthTelemetryReadV1::BaselineEstablished { observed_at, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("the first pass must retain a real baseline"); + }; + let StorageTelemetryReadV1::Observed { sample: before } = + port.store_size(&context, &store).await + else { + panic!("the pre-reclamation size must be measured"); + }; + let occupied_before = before.page_count - before.freelist_pages; + assert!(occupied_before > 0); + + // Seed the production timeout transition deterministically; reproducing a + // wall-clock timeout would make this correctness test depend on host speed. + let TableSizeWalk::OverBudget(mark) = settle_table_size_walk( + Err(TableGrowthUnavailableReasonV1::DeadlineExceeded), + occupied_before, + now_micros(), + ) + .unwrap() else { + panic!("a deadline must produce a retry-suppression mark"); + }; + port.table_growth.lock().unwrap().over_budget = Some(mark); + let unchanged = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(unchanged.observed, 0); + assert_eq!(unchanged.unavailable, 1); + assert!(matches!( + port.latest_table_growth(&context, &store), + TableGrowthTelemetryReadV1::Unknown { + reason: TableGrowthUnavailableReasonV1::ExceedsMeasurementBudget, + .. + } + )); + + database + .execute_write_batch( + "reclaim table payload through its writer", + "DELETE FROM growth_payload WHERE rowid > 1;", + ) + .await + .expect("logical deletion without VACUUM or restart"); + let context = storage_telemetry_request_context(scope.clone()).expect("reader context"); + let StorageTelemetryReadV1::Observed { sample: after } = + port.store_size(&context, &store).await + else { + panic!("the post-reclamation size must be measured"); + }; + assert!( + after.page_count >= before.page_count, + "the old allocated-page guard must still suppress this fixture" + ); + assert!(after.freelist_pages > before.freelist_pages); + assert!(after.page_count - after.freelist_pages < occupied_before); + + let retried = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(retried.observed, 1); + assert_eq!(retried.unavailable, 0); + let TableGrowthTelemetryReadV1::Observed { samples, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("retry must compare with the original baseline, not replace it"); + }; + let retained = samples + .iter() + .find(|sample| sample.table.as_str() == "growth_payload") + .expect("the retained table remains measured"); + assert_eq!(retained.previous_observed_at, observed_at); + assert!(retained.previous_bytes.get() >= 8 * 1024 * 1024); + assert!(retained.current_bytes.get() >= 262_144); + assert!(retained.current_bytes.get() < retained.previous_bytes.get()); + let retained_bytes = retained.current_bytes; + assert_eq!(port.table_growth_state().over_budget, None); + + database + .execute_write_batch( + "grow the retained table again", + "INSERT INTO growth_payload VALUES (zeroblob(262144));", + ) + .await + .expect("growth after reclamation"); + let resumed = registry + .advance_registered(&paths, &paths, &cancellation) + .await; + assert_eq!(resumed.observed, 1); + assert_eq!(resumed.unavailable, 0); + let context = storage_telemetry_request_context(scope).expect("reader context"); + let TableGrowthTelemetryReadV1::Observed { samples, .. } = + port.latest_table_growth(&context, &store) + else { + panic!("subsequent growth must remain observable"); + }; + let grown = samples + .iter() + .find(|sample| sample.table.as_str() == "growth_payload") + .expect("the regrown table is measured"); + assert_eq!(grown.previous_bytes, retained_bytes); + assert!(grown.growth_bytes().get() >= 262_144); +} diff --git a/crates/tracedecay-runtime-core/src/operation_task_owner.rs b/crates/tracedecay-runtime-core/src/operation_task_owner.rs index 23ae5090f23..df63c41ba29 100644 --- a/crates/tracedecay-runtime-core/src/operation_task_owner.rs +++ b/crates/tracedecay-runtime-core/src/operation_task_owner.rs @@ -189,23 +189,31 @@ impl RuntimeOperationTaskOwnerV1 { state.lifecycle = RuntimeOperationTaskLifecycleV1::Fenced; let (publish_completion, completion) = watch::channel(None); let tasks = std::mem::take(&mut state.tasks); - self.runtime.spawn(async move { - let mut failures = Vec::new(); - for (task_id, task) in tasks { - if let Err(error) = task.await { - failures.push(RuntimeOperationTaskJoinFailure { - task_id, - error: Arc::new(error), - }); + if tasks.is_empty() { + // Completed owners need no runtime to drain. Cleanup can + // arrive after their admission runtime has stopped; an + // empty reaper spawned there would be cancelled without + // publishing the already-settled result. + publish_completion.send_replace(Some(Ok(()))); + } else { + self.runtime.spawn(async move { + let mut failures = Vec::new(); + for (task_id, task) in tasks { + if let Err(error) = task.await { + failures.push(RuntimeOperationTaskJoinFailure { + task_id, + error: Arc::new(error), + }); + } } - } - let result = if failures.is_empty() { - Ok(()) - } else { - Err(RuntimeOperationShutdownError::JoinFailures(failures)) - }; - publish_completion.send_replace(Some(result)); - }); + let result = if failures.is_empty() { + Ok(()) + } else { + Err(RuntimeOperationShutdownError::JoinFailures(failures)) + }; + publish_completion.send_replace(Some(result)); + }); + } state.lifecycle = RuntimeOperationTaskLifecycleV1::Draining { completion: completion.clone(), }; @@ -248,6 +256,33 @@ mod tests { use std::sync::atomic::AtomicBool; use tokio::sync::oneshot; + #[test] + fn settled_owner_shuts_down_after_its_admission_runtime_stops() { + let admission_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let owner = RuntimeOperationTaskOwnerV1::new(admission_runtime.handle().clone()); + admission_runtime.block_on(async { + owner.retain(|_| async {}).unwrap(); + owner.settled().await; + }); + drop(admission_runtime); + + let cleanup_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + cleanup_runtime.block_on(async { + owner.shutdown().await.unwrap(); + owner.shutdown().await.unwrap(); + assert!(matches!( + owner.retain(|_| async {}), + Err(RuntimeOperationAdmissionError::Closed) + )); + }); + } + #[tokio::test] async fn accepted_settlement_survives_request_and_shutdown_waiter_cancellation() { let owner = Arc::new(RuntimeOperationTaskOwnerV1::new(Handle::current())); diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs index 9edee1472e7..59189d8fdbc 100644 --- a/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker.rs @@ -27,6 +27,8 @@ use crate::telemetry::{ReaderAdmissionRecorder, take_observed_vm}; use super::{ExistingReaderLocator, ReaderStartError}; +mod table_sizes; + const REPLY_POLL_QUANTUM: Duration = Duration::from_millis(5); /// How often an idle snapshot checks whether the checkpoint fence revoked it. @@ -770,29 +772,7 @@ async fn run_snapshot( let _ = reply.send(result); } SnapshotCommand::TableSizes { reply } => { - let read = || -> Result, rusqlite::Error> { - let mut statement = transaction.prepare_cached( - "SELECT schema_entry.name, COALESCE(SUM(dbstat.payload), 0) \ - FROM sqlite_schema AS schema_entry \ - LEFT JOIN dbstat ON dbstat.name = schema_entry.name \ - WHERE schema_entry.type = 'table' \ - AND schema_entry.name NOT LIKE 'sqlite_%' \ - GROUP BY schema_entry.name \ - ORDER BY schema_entry.name", - )?; - statement - .query_map([], |row| { - let bytes = row.get::<_, i64>(1)?; - Ok(TableSizeTelemetrySample { - table_name: row.get(0)?, - bytes: u64::try_from(bytes).map_err(|_| { - rusqlite::Error::IntegralValueOutOfRange(1, bytes) - })?, - }) - })? - .collect() - }; - let result = read().map_err(|error| { + let result = table_sizes::read(transaction).map_err(|error| { ReaderWorkerError::Storage(StorageRuntimeErrorV1::Infrastructure { operation: format!("read table size telemetry: {error}"), }) diff --git a/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs new file mode 100644 index 00000000000..deb7cbd7ba2 --- /dev/null +++ b/crates/tracedecay-rusqlite-runtime/src/reader/worker/table_sizes.rs @@ -0,0 +1,288 @@ +//! Payload census on the reader's existing snapshot, with one page walk. + +use std::collections::BTreeMap; + +use rusqlite::{Connection, types::Type}; + +use super::TableSizeTelemetrySample; + +/// Initialize every user table (including empty and rootless tables) from +/// schema, then visit the dbstat pages exactly once. The prior correlated SUM +/// re-walked the virtual table for each schema table; many-table profiles spent +/// their measurement deadline repeating a census even without a page sorter. +/// +/// Page-mode dbstat preserves the reader's interrupt checks between pages. +/// Index and free-page bytes are not table payload; the separate footprint +/// census measures all physical database/WAL/SHM bytes. +pub(super) fn read(connection: &Connection) -> rusqlite::Result> { + let mut totals = BTreeMap::::new(); + { + let mut statement = connection.prepare_cached( + "SELECT name FROM sqlite_schema \ + WHERE type = 'table' AND name NOT GLOB 'sqlite_*'", + )?; + for name in statement.query_map([], |row| row.get::<_, String>(0))? { + totals.insert(name?, 0); + } + } + + let mut statement = connection.prepare_cached("SELECT name, payload FROM dbstat")?; + let mut pages = statement.query([])?; + while let Some(page) = pages.next()? { + let name_value = page.get_ref(0)?; + let name = name_value.as_str().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(0, Type::Text, Box::new(error)) + })?; + if let Some(total) = totals.get_mut(name) { + let signed_payload = page.get::<_, i64>(1)?; + let payload = u64::try_from(signed_payload) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(1, signed_payload))?; + *total = total + .checked_add(payload) + .ok_or(rusqlite::Error::IntegralValueOutOfRange(1, i64::MAX))?; + } + } + Ok(totals + .into_iter() + .map(|(table_name, bytes)| TableSizeTelemetrySample { table_name, bytes }) + .collect()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + use super::*; + + // Independent, per-page reference for the previous census. Keeping it in + // the test also makes the work regression falsifiable on the same SQLite + // build, without wall-clock thresholds or query-plan string assertions. + const GROUPED_CENSUS: &str = "SELECT schema_entry.name, COALESCE(SUM(dbstat.payload), 0) \ + FROM sqlite_schema AS schema_entry \ + LEFT JOIN dbstat ON dbstat.name = schema_entry.name \ + WHERE schema_entry.type = 'table' \ + AND schema_entry.name NOT LIKE 'sqlite_%' \ + GROUP BY schema_entry.name ORDER BY schema_entry.name"; + + fn fixture() -> (tempfile::TempDir, Connection) { + let directory = tempfile::tempdir().unwrap(); + let connection = Connection::open(directory.path().join("growth.db")).unwrap(); + connection + .execute_batch( + "PRAGMA page_size = 4096; + PRAGMA auto_vacuum = NONE; + CREATE TABLE events(id INTEGER PRIMARY KEY AUTOINCREMENT, body BLOB); + CREATE INDEX event_size ON events(length(body)); + CREATE TABLE empty_table(body BLOB); + CREATE TABLE keyed(name TEXT PRIMARY KEY, body BLOB) WITHOUT ROWID; + CREATE VIRTUAL TABLE page_probe USING dbstat; + INSERT INTO keyed VALUES ('one', zeroblob(32768)); + WITH RECURSIVE n(x) AS ( + SELECT 1 UNION ALL SELECT x + 1 FROM n WHERE x < 16 + ) INSERT INTO events(body) SELECT zeroblob(262144) FROM n;", + ) + .unwrap(); + (directory, connection) + } + + fn reference(connection: &Connection) -> Vec { + let mut statement = connection.prepare(GROUPED_CENSUS).unwrap(); + statement + .query_map([], |row| { + let bytes = row.get::<_, i64>(1)?; + Ok(TableSizeTelemetrySample { + table_name: row.get(0)?, + bytes: u64::try_from(bytes).unwrap(), + }) + }) + .unwrap() + .collect::>() + .unwrap() + } + + fn measure_vm(connection: &Connection, run: impl FnOnce() -> T) -> (T, usize) { + let steps = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&steps); + connection + .progress_handler( + 1, + Some(move || { + observed.fetch_add(1, Ordering::Relaxed); + false + }), + ) + .unwrap(); + let result = run(); + connection + .progress_handler(0, None:: bool>) + .unwrap(); + (result, steps.load(Ordering::Relaxed)) + } + + fn page_count(connection: &Connection) -> i64 { + connection + .pragma_query_value(None, "page_count", |row| row.get(0)) + .unwrap() + } + + #[test] + fn payload_census_tracks_growth_and_reclamation_without_counting_indexes() { + let (_directory, connection) = fixture(); + let before = read(&connection).unwrap(); + assert_eq!(before, reference(&connection)); + let bytes: BTreeMap<_, _> = before + .iter() + .map(|sample| (sample.table_name.as_str(), sample.bytes)) + .collect(); + assert!(bytes["events"] > 16 * 262_144); + assert!(bytes["keyed"] > 32_768); + assert_eq!(bytes["empty_table"], 0); + assert_eq!( + bytes["page_probe"], 0, + "rootless tables still have a sample" + ); + assert!(!bytes.contains_key("event_size")); + assert!(!bytes.contains_key("sqlite_sequence")); + assert!( + before + .windows(2) + .all(|pair| pair[0].table_name < pair[1].table_name) + ); + + connection + .execute("INSERT INTO events(body) VALUES (zeroblob(262144))", []) + .unwrap(); + let enlarged = read(&connection).unwrap(); + assert_eq!(enlarged, reference(&connection)); + assert!( + enlarged + .iter() + .any(|sample| { sample.table_name == "events" && sample.bytes > bytes["events"] }) + ); + + let grown_pages = page_count(&connection); + connection + .execute("DELETE FROM events WHERE id > 1", []) + .unwrap(); + let reclaimed = read(&connection).unwrap(); + assert_eq!(reclaimed, reference(&connection)); + let retained = reclaimed + .iter() + .find(|sample| sample.table_name == "events") + .unwrap(); + assert!(retained.bytes > 262_144); + assert!(retained.bytes < bytes["events"]); + assert_eq!( + page_count(&connection), + grown_pages, + "free pages are not live payload" + ); + + // Only this disposable fixture is vacuumed. Payload accounting must + // not confuse physical compaction with another logical deletion. + connection.execute_batch("VACUUM").unwrap(); + assert!(page_count(&connection) < grown_pages); + assert_eq!(read(&connection).unwrap(), reclaimed); + } + + #[test] + fn payload_census_includes_sqlite_named_user_tables_and_their_growth() { + let (_directory, connection) = fixture(); + connection + .execute_batch( + "CREATE TABLE sqlitemessages(body BLOB); + INSERT INTO sqlitemessages VALUES (zeroblob(1048576));", + ) + .unwrap(); + let before = read(&connection).unwrap(); + let initial = before + .iter() + .find(|sample| sample.table_name == "sqlitemessages") + .expect("a user table without the reserved underscore must be measured"); + assert!(initial.bytes > 1_048_576); + + connection + .execute("INSERT INTO sqlitemessages VALUES (zeroblob(1048576))", []) + .unwrap(); + let after = read(&connection).unwrap(); + let grown = after + .iter() + .find(|sample| sample.table_name == "sqlitemessages") + .unwrap(); + let expected: i64 = connection + .query_row( + "SELECT SUM(payload) FROM dbstat WHERE name = 'sqlitemessages'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(grown.bytes, u64::try_from(expected).unwrap()); + assert!(grown.bytes > initial.bytes); + assert!( + after + .iter() + .all(|sample| sample.table_name != "sqlite_sequence") + ); + } + + #[test] + fn page_census_avoids_group_sort_work_under_the_same_vm_budget() { + let (_directory, connection) = fixture(); + let (expected, grouped_steps) = measure_vm(&connection, || reference(&connection)); + let (actual, streamed_steps) = measure_vm(&connection, || read(&connection).unwrap()); + assert_eq!(actual, expected); + assert!(actual.iter().any(|sample| sample.bytes > 4 * 1024 * 1024)); + assert!(streamed_steps > 0); + assert!( + streamed_steps < grouped_steps / 2, + "page sorting consumed the census budget: streamed={streamed_steps}, grouped={grouped_steps}" + ); + } + + #[test] + fn many_tables_do_not_repeat_the_full_page_census_per_table() { + let directory = tempfile::tempdir().unwrap(); + let connection = Connection::open(directory.path().join("many-tables.db")).unwrap(); + for index in 0..36 { + connection + .execute_batch(&format!( + "CREATE TABLE t{index}(body BLOB); + INSERT INTO t{index} VALUES(zeroblob(32768));" + )) + .unwrap(); + } + + let (expected, grouped_steps) = measure_vm(&connection, || reference(&connection)); + let (actual, single_pass_steps) = measure_vm(&connection, || read(&connection).unwrap()); + assert_eq!(actual, expected); + assert_eq!(actual.len(), 36); + assert!(actual.iter().all(|sample| sample.bytes >= 32768)); + assert!(single_pass_steps > 0); + assert!( + single_pass_steps < grouped_steps / 3, + "many-table payload census repeated work: single_pass={single_pass_steps}, grouped={grouped_steps}" + ); + } + + #[test] + fn interrupted_census_returns_an_error_not_partial_or_zero_sizes_and_can_retry() { + let (_directory, connection) = fixture(); + connection.progress_handler(100, Some(|| true)).unwrap(); + let interrupted = read(&connection); + connection + .progress_handler(0, None:: bool>) + .unwrap(); + assert!(matches!( + interrupted, + Err(rusqlite::Error::SqliteFailure(error, _)) + if error.code == rusqlite::ErrorCode::OperationInterrupted + )); + let retried = read(&connection).unwrap(); + assert_eq!(retried, reference(&connection)); + assert!(retried.iter().any(|sample| sample.bytes > 4 * 1024 * 1024)); + } +} diff --git a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs index a4e66eaca79..949395a7ffd 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs @@ -693,24 +693,39 @@ async fn serve_started_during_daemon_restart_window_proxies_to_restarted_daemon( } let request: Value = serde_json::from_str(request_line.trim()).expect("request json"); - let response = json!({ - "jsonrpc": "2.0", - "id": request["id"], - "result": { - "protocolVersion": "2024-11-05", - "capabilities": { "tools": {} }, - "serverInfo": { - "name": "sentinel-restarted-daemon", - "version": "0.0.1-sentinel" + // Serve also reads its optional hint setting on a background + // connection. That request can reach the restarted listener + // first; only the host's initialize completes this fixture. + let initialize = request["method"] == "initialize"; + let response = if initialize { + json!({ + "jsonrpc": "2.0", + "id": request["id"], + "result": { + "protocolVersion": "2024-11-05", + "capabilities": { "tools": {} }, + "serverInfo": { + "name": "sentinel-restarted-daemon", + "version": "0.0.1-sentinel" + } } - } - }); + }) + } else { + json!({ + "jsonrpc": "2.0", + "id": request["id"], + "error": { + "code": -32601, + "message": "fixture daemon only serves initialize" + } + }) + }; writeln!(stream, "{response}").expect("write fake daemon response"); // Drain until the proxy hangs up so the response is not lost to a // connection reset. let mut scratch = [0_u8; 64]; while matches!(reader.read(&mut scratch), Ok(n) if n > 0) {} - Some(()) + initialize.then_some(()) }, || "timed out waiting for serve to proxy a request to the restarted daemon".to_string(), );