From c8a1effe74cbbc15ae8fcf754cc5817924b58310 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:42:19 -0800 Subject: [PATCH 001/111] fix(graph): reconcile verified retirement and retry state Refs #4616 Combine the proven bounded receipt/ownership cleanup fixes with post-unlink collection-state ordering and compatible filesystem regressions. Replace the old separately reviewed PR history without rewriting either branch, and preserve all exclusion and fail-closed checks. --- .../src/generation_runtime.rs | 65 +- .../tracedecay-graph-db/src/sealed_store.rs | 576 ++++++++++++++++-- .../tests/graph_db_suite/runtime_contract.rs | 41 +- .../src/capability_dir.rs | 355 ++++++++++- .../tracedecay-private-fs/src/framed_log.rs | 9 +- 5 files changed, 976 insertions(+), 70 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation_runtime.rs b/crates/tracedecay-graph-db/src/generation_runtime.rs index d2fdd35dd58..a514bbc76de 100644 --- a/crates/tracedecay-graph-db/src/generation_runtime.rs +++ b/crates/tracedecay-graph-db/src/generation_runtime.rs @@ -1840,6 +1840,11 @@ impl GraphDb { self.retire_sealed_generation_store(locator)?; return Ok(GenerationContentsDeletion::RetentionPending); } + // Native row deletion may already have committed, but the generation + // is not collected until its independently durable sealed artifact is + // gone. If unlink refuses, keep the in-memory retirement state retryable + // instead of advertising collection while bytes still exist on disk. + self.retire_sealed_generation_store(locator)?; let mut state = self.wait_verified_generations_write()?; state.known.remove(locator); state.quarantined.remove(locator); @@ -1858,7 +1863,6 @@ impl GraphDb { state.heads.remove(&locator.projection); } drop(state); - self.retire_sealed_generation_store(locator)?; Ok(GenerationContentsDeletion::Deleted) } @@ -4001,20 +4005,73 @@ mod tests { .delete_generation_contents(&locator, &|| Ok(())) .expect_err("a failed sealed unlink must not complete retention"); assert!( - error - .to_string() - .contains("sealed directory removal failed") + matches!( + &error, + GraphDbError::Unavailable { message } + if message.starts_with("sealed generation store sealed directory removal failed:") + || message.starts_with("sealed generation store sealed generation retirement refused:") + ), + "an unremovable artifact must fail at the sealed-directory unlink boundary: {error}" ); assert_eq!(std::fs::read(&directory).unwrap(), b"not a directory"); + assert!( + !database + .wait_verified_generations_write() + .unwrap() + .collected + .contains(&locator), + "a failed sealed unlink must not mark its generation collected" + ); std::fs::remove_file(&directory).unwrap(); std::fs::create_dir(&directory).unwrap(); + // Retirement requires an exact durable identity from the directory + // being removed. A receipt-free synthetic directory must never confer + // deletion authority when the strict collector is integrated. + let identity = generation.identity(); + let receipt = serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": locator.projection.namespace.as_str(), + "projection": locator.projection.projection.as_str(), + "generation": locator.generation.as_str(), + "physical_namespace": namespace.as_str(), + "recovered_digest": sealed_digest(&generation).as_str(), + "entities": 0, + "relations": 0, + "graph_format": GraphFormatVersion::current().get(), + "row_sum": null, + "base": null, + "source_generation": identity.source_generation.as_str(), + "watermark": identity.watermark.as_str() + }); + std::fs::write( + directory.join("sealed.json"), + serde_json::to_vec(&receipt).unwrap(), + ) + .unwrap(); std::fs::write(directory.join("derived-rows"), b"retired").unwrap(); + let unrelated = directory.parent().unwrap().join("unrelated-entry"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("sentinel"), b"untouched").unwrap(); assert!(matches!( database.delete_generation_contents(&locator, &|| Ok(())), Ok(GenerationContentsDeletion::Deleted) )); + assert_eq!( + std::fs::read(unrelated.join("sentinel")).unwrap(), + b"untouched", + "retiring one sealed generation must not traverse another child" + ); assert!(!directory.exists()); + assert!( + database + .wait_verified_generations_write() + .unwrap() + .collected + .contains(&locator), + "only successful physical retirement may mark the generation collected" + ); // Idempotent replay after the directory is gone still succeeds. assert!(matches!( database.delete_generation_contents(&locator, &|| Ok(())), diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..988c8ccf94e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -112,7 +112,7 @@ pub(crate) fn open_direct_sealed_generation( sealed_generation_directory(&sealed_store_root(database_path), &physical_namespace); let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); let sealed_path = directory.join(SEALED_STORE_DATABASE_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { + let receipt_bytes = match read_sealed_receipt_bytes(&receipt_path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), @@ -207,6 +207,23 @@ pub(crate) fn open_direct_sealed_generation( const SEALED_COPY_GUARD_CHUNK_ROWS: usize = 4096; const SEALED_STORE_RECEIPT_VERSION: u32 = 1; +/// A proof marker contains one canonical recovered digest, never a payload. +const MAX_SEALED_CHECK_BYTES: usize = 256; +/// Identity metadata, never an unbounded content read. +const MAX_SEALED_RECEIPT_BYTES: usize = 64 * 1024; + +fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result> { + // The shared reader opens without following a final symlink, rejects FIFOs + // and other nonregular files, and checks size against the same held handle. + match tracedecay_private_fs::framed_log::read_bounded(path, MAX_SEALED_RECEIPT_BYTES)? { + Some(bytes) => Ok(bytes), + None => Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "sealed generation receipt is missing", + )), + } +} + pub(crate) const SEALED_STORE_DATABASE_FILE: &str = "generation.grafeo"; const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` @@ -477,7 +494,11 @@ impl SealedGenerationStore { /// owner to retry; a sealed-only live head never enters this path. fn discard(&self) -> Result<(), GraphDbError> { let _ = self.close(); - remove_sealed_directory_checked(&self.directory) + retire_sealed_directory_checked( + &self.directory, + &self.locator, + Some(&self.recovered_digest), + ) } } @@ -566,7 +587,7 @@ pub fn census_sealed_store( census.abandoned_staging_bytes += bytes; continue; } - let receipt = std::fs::read(path.join(SEALED_STORE_RECEIPT_FILE)) + let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) .ok() .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); let Some(receipt) = receipt else { @@ -596,7 +617,9 @@ fn directory_bytes( while let Some(directory) = pending.pop() { for entry in std::fs::read_dir(&directory)? { let entry = entry?; - let metadata = entry.metadata()?; + // Count only the link itself: following a foreign directory link + // can mischarge its entire tree or revisit an ancestor forever. + let metadata = std::fs::symlink_metadata(entry.path())?; if metadata.is_dir() { pending.push(entry.path()); continue; @@ -624,40 +647,40 @@ pub(crate) fn discard_superseded_sealed_generations( database_path: &Path, ) -> Result<(), GraphDbError> { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(sealed_store_io_failure("sealed root read failed", error)), - }; - for entry in entries { - let entry = - entry.map_err(|error| sealed_store_io_failure("sealed root read failed", error))?; - let path = entry.path(); - if entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) - { - continue; - } - let current = load_sealed_store_receipt(&path) - .ok() - .flatten() - .is_some_and(|receipt| receipt.graph_format == GraphFormatVersion::current().get()); - if !current { - match std::fs::remove_dir_all(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(sealed_store_io_failure( - "superseded sealed generation removal failed", - error, - )); - } + let reclaimed = tracedecay_private_fs::capability_dir::remove_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), + MAX_SEALED_RECEIPT_BYTES, + |name, bytes| { + let Some(name) = name.to_str() else { + return false; + }; + if name.len() != 64 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return false; } - } + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return false; + }; + let recognized_form = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + recognized_form + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.physical_namespace == format!("generation:{name}") + && receipt.graph_format > 0 + && receipt.graph_format < GraphFormatVersion::current().get() + }, + ); + match reclaimed { + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(sealed_store_io_failure( + "superseded sealed generation removal failed", + error, + )), } - Ok(()) } /// Removes every `.staging-*` directory a seal left behind under the store's @@ -695,9 +718,12 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { continue; } let path = entry.path(); - match std::fs::remove_dir_all(&path) { - Ok(()) => removed += 1, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + match tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + &root, + &entry.file_name(), + ) { + Ok(true) => removed += 1, + Ok(false) => {} Err(error) => tracing::warn!( event = "sealed_staging_sweep_failed", path = %path.display(), @@ -737,9 +763,10 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) { let path = entry.path(); - if let Err(error) = std::fs::remove_dir_all(&path) - && error.kind() != std::io::ErrorKind::NotFound - { + if let Err(error) = tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + &root, + &entry.file_name(), + ) { tracing::warn!( event = "graph_row_spill_sweep_failed", path = %path.display(), @@ -752,14 +779,65 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { - match std::fs::remove_dir_all(directory) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(sealed_store_io_failure( - "sealed directory removal failed", - error, - )), + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed directory has no parent"))?; + let name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed directory has no entry name"))?; + tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(root, name) + .map(|_| ()) + .map_err(|error| sealed_store_io_failure("sealed directory removal failed", error)) +} + +/// Accepted replay retirement is not permission to recursively delete a +/// substituted or unrecognizable artifact. Require the exact durable identity +/// under a held child capability; preserve an unverified directory for retry. +fn retire_sealed_directory_checked( + directory: &Path, + locator: &GenerationLocator, + expected_digest: Option<&str>, +) -> Result<(), GraphDbError> { + let physical = locator.physical_namespace()?; + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed retirement directory has no parent"))?; + if sealed_generation_directory(root, &physical) != directory { + return Err(GraphDbError::unavailable( + "sealed retirement directory does not match generation identity", + )); } + let name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed retirement directory has no entry name"))?; + let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), + MAX_SEALED_RECEIPT_BYTES, + |bytes| { + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return false; + }; + let form_valid = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + form_valid + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.graph_format > 0 + && receipt.graph_format <= GraphFormatVersion::current().get() + && receipt.physical_namespace == physical.as_str() + && receipt.namespace == locator.projection.namespace.as_str() + && receipt.projection == locator.projection.projection.as_str() + && receipt.generation == locator.generation.as_str() + && expected_digest.is_none_or(|digest| receipt.recovered_digest == digest) + }, + ); + result + .map(|_| ()) + .map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error)) } /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted @@ -1694,7 +1772,11 @@ impl GraphDb { }; let physical_namespace = locator.physical_namespace()?; let root = sealed_store_root(database_path); - remove_sealed_directory_checked(&sealed_generation_directory(&root, &physical_namespace)) + retire_sealed_directory_checked( + &sealed_generation_directory(&root, &physical_namespace), + locator, + None, + ) } } @@ -1786,8 +1868,16 @@ fn build_or_open_sealed_store( .and_then(|name| name.to_str()) .unwrap_or("sealed") )); - remove_sealed_directory(&staging); - std::fs::create_dir_all(&staging) + // A failed cleanup must never be followed by create_dir_all: it would + // accept a pre-existing staging symlink and write into its foreign target. + std::fs::create_dir_all(&root) + .map_err(|error| sealed_store_io_failure("sealed root create failed", error))?; + let staging_name = staging + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; + tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(&root, staging_name) + .map_err(|error| sealed_store_io_failure("staging directory cleanup refused", error))?; + std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check) .inspect_err(|_| remove_sealed_directory(&staging))?; @@ -2426,7 +2516,7 @@ fn load_sealed_store_receipt( directory: &Path, ) -> Result, GraphDbError> { let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { + let receipt_bytes = match read_sealed_receipt_bytes(&receipt_path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), @@ -2441,9 +2531,12 @@ fn sealed_store_check_matches( expected_digest: &str, ) -> Result { let path = directory.join(SEALED_STORE_CHECKED_FILE); - match std::fs::read_to_string(&path) { - Ok(digest) => Ok(digest == expected_digest), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + // This is a rebuildable proof hint. Invalid or overlarge bytes cause a + // fresh proof, never an unbounded allocation or a trusted cache hit. + match tracedecay_private_fs::framed_log::read_bounded(&path, MAX_SEALED_CHECK_BYTES) { + Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), + Ok(None) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => Ok(false), Err(error) => Err(sealed_store_io_failure("sealed check read failed", error)), } } @@ -2452,8 +2545,15 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); } - std::fs::write(directory.join(SEALED_STORE_CHECKED_FILE), expected_digest) - .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) + // Never truncate an existing symlink target: the marker is a rebuildable + // cache, so stage a regular file and publish it by atomic replacement. + tracedecay_private_fs::framed_log::atomic_write( + &directory.join(SEALED_STORE_CHECKED_FILE), + "sealed-check", + expected_digest.as_bytes(), + tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, + ) + .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) } fn open_sealed_store_checked( @@ -2730,6 +2830,298 @@ pub(crate) fn sealed_copy_proof( Ok(canonical_bytes) } +#[cfg(test)] +mod receipt_budget_tests { + #[cfg(unix)] + use std::collections::BTreeSet; + use std::path::Path; + + use super::*; + + fn receipt(format: u32, name: &str, generation: &str) -> Vec { + serde_json::to_vec(&SealedStoreReceiptV1 { + version: SEALED_STORE_RECEIPT_VERSION, + form: SEALED_STORE_FORM_COMPACT.to_owned(), + namespace: "test".to_owned(), + projection: "code".to_owned(), + generation: generation.to_owned(), + physical_namespace: format!("generation:{name}"), + recovered_digest: "digest".to_owned(), + entities: 0, + relations: 0, + graph_format: format, + row_sum: None, + base: None, + source_generation: None, + watermark: None, + }) + .unwrap() + } + + fn generation(root: &Path, digit: char, contents: &[u8]) -> PathBuf { + let directory = root.join(digit.to_string().repeat(64)); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write(directory.join(SEALED_STORE_RECEIPT_FILE), contents).unwrap(); + directory + } + + #[test] + fn sealed_receipt_reads_bound_memory_even_for_json_padded_beyond_the_limit() { + let temp = tempfile::tempdir().unwrap(); + let receipt_path = temp.path().join(SEALED_STORE_RECEIPT_FILE); + let normal = receipt(GraphFormatVersion::current().get(), &"a".repeat(64), "head"); + std::fs::write(&receipt_path, &normal).unwrap(); + assert_eq!(read_sealed_receipt_bytes(&receipt_path).unwrap(), normal); + + let oversized = [normal, vec![b' '; MAX_SEALED_RECEIPT_BYTES]].concat(); + assert!(serde_json::from_slice::(&oversized).is_ok()); + std::fs::write(&receipt_path, oversized).unwrap(); + assert_eq!( + read_sealed_receipt_bytes(&receipt_path).unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + // On Unix this is sparse, so the multi-gigabyte bound is exercised + // without allocating a multi-gigabyte test artifact. + #[cfg(unix)] + { + std::fs::File::create(&receipt_path) + .unwrap() + .set_len(15 * 1024 * 1024 * 1024) + .unwrap(); + assert_eq!( + read_sealed_receipt_bytes(&receipt_path).unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + } + } + + #[test] + fn sealed_check_marker_is_bounded_and_never_overwrites_a_foreign_target() { + let temp = tempfile::tempdir().unwrap(); + let directory = temp.path().join("sealed"); + std::fs::create_dir(&directory).unwrap(); + let marker = directory.join(SEALED_STORE_CHECKED_FILE); + let digest = format!("sha256:{}", "a".repeat(64)); + + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + assert_eq!(std::fs::read(&marker).unwrap(), digest.as_bytes()); + + let mut oversized = digest.as_bytes().to_vec(); + oversized.extend(vec![b' '; MAX_SEALED_CHECK_BYTES]); + std::fs::write(&marker, oversized).unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + + #[cfg(unix)] + { + // Sparse bytes establish the large-file refusal without writing + // or allocating a multi-gigabyte test payload. + std::fs::File::create(&marker) + .unwrap() + .set_len(15 * 1024 * 1024 * 1024) + .unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + + let foreign = temp.path().join("foreign"); + std::fs::remove_file(&marker).unwrap(); + std::fs::write(&foreign, digest.as_bytes()).unwrap(); + std::os::unix::fs::symlink(&foreign, &marker).unwrap(); + assert!( + sealed_store_check_matches(&directory, &digest).is_err(), + "a symlink cannot supply proof evidence" + ); + + std::fs::remove_file(&marker).unwrap(); + std::fs::write(&foreign, b"foreign sentinel").unwrap(); + std::os::unix::fs::symlink(&foreign, &marker).unwrap(); + assert!( + persist_sealed_store_check(&directory, &digest).is_err(), + "publishing a proof marker must refuse a foreign symlink" + ); + assert_eq!(std::fs::read(&foreign).unwrap(), b"foreign sentinel"); + } + } + + #[test] + fn obsolete_format_cleanup_keeps_unknown_receipts_and_other_files() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir_all(&root).unwrap(); + let current = GraphFormatVersion::current().get(); + assert!(current > 0); + let old = generation(&root, 'a', &receipt(current - 1, &"a".repeat(64), "old")); + let live = generation(&root, 'b', &receipt(current, &"b".repeat(64), "live")); + let invalid = generation(&root, 'c', b"not a receipt"); + // Zero is not a valid GraphFormatVersion, even when the JSON is well formed. + let zero_format = generation(&root, '0', &receipt(0, &"0".repeat(64), "zero")); + let future = generation( + &root, + 'f', + &receipt(current.saturating_add(1), &"f".repeat(64), "future"), + ); + let padded = [ + receipt(current - 1, &"d".repeat(64), "padded"), + vec![b' '; MAX_SEALED_RECEIPT_BYTES], + ] + .concat(); + let oversized = generation(&root, 'd', &padded); + let missing = root.join("e".repeat(64)); + std::fs::create_dir_all(&missing).unwrap(); + let unrelated = root.join("unrelated.txt"); + std::fs::write(&unrelated, b"foreign").unwrap(); + // A symlink to a syntactically valid old receipt must never confer + // deletion authority over the directory that holds the link. + #[cfg(unix)] + let linked_receipt = { + let directory = root.join("1".repeat(64)); + std::fs::create_dir_all(&directory).unwrap(); + let foreign = temp.path().join("foreign-receipt.json"); + std::fs::write(&foreign, receipt(current - 1, &"1".repeat(64), "foreign")).unwrap(); + std::os::unix::fs::symlink(&foreign, directory.join(SEALED_STORE_RECEIPT_FILE)) + .unwrap(); + std::fs::write(directory.join("sentinel"), b"unowned").unwrap(); + (directory, foreign) + }; + for dir in [ + &old, + &live, + &invalid, + &zero_format, + &oversized, + &missing, + &future, + ] { + std::fs::write(dir.join("sentinel"), b"owned").unwrap(); + } + + discard_superseded_sealed_generations(&database).unwrap(); + assert!(!old.exists(), "verified old-format receipt is reclaimable"); + for dir in [&live, &invalid, &zero_format, &oversized, &missing, &future] { + assert_eq!(std::fs::read(dir.join("sentinel")).unwrap(), b"owned"); + } + assert_eq!(std::fs::read(&unrelated).unwrap(), b"foreign"); + #[cfg(unix)] + { + assert_eq!( + std::fs::read(linked_receipt.0.join("sentinel")).unwrap(), + b"unowned" + ); + assert!(linked_receipt.1.is_file()); + assert_eq!( + read_sealed_receipt_bytes(&linked_receipt.0.join(SEALED_STORE_RECEIPT_FILE)) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidInput + ); + } + } + + #[test] + fn retired_sealed_generation_requires_its_exact_receipt_identity() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new("generation:retirement").unwrap(), + ); + let physical = locator.physical_namespace().unwrap(); + let directory = sealed_generation_directory(&root, &physical); + std::fs::create_dir_all(&directory).unwrap(); + let name = directory.file_name().unwrap().to_str().unwrap(); + let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); + std::fs::write( + &receipt_path, + receipt(GraphFormatVersion::current().get(), name, "generation:other"), + ) + .unwrap(); + std::fs::write(directory.join("sentinel"), b"preserve").unwrap(); + assert!(retire_sealed_directory_checked(&directory, &locator, None).is_err()); + assert_eq!( + std::fs::read(directory.join("sentinel")).unwrap(), + b"preserve" + ); + std::fs::write( + &receipt_path, + receipt( + GraphFormatVersion::current().get(), + name, + locator.generation.as_str(), + ), + ) + .unwrap(); + assert!(retire_sealed_directory_checked(&directory, &locator, Some("wrong")).is_err()); + assert_eq!( + std::fs::read(directory.join("sentinel")).unwrap(), + b"preserve" + ); + retire_sealed_directory_checked(&directory, &locator, Some("digest")).unwrap(); + assert!( + !directory.exists(), + "verified retired generation is physically removed" + ); + retire_sealed_directory_checked(&directory, &locator, Some("digest")).unwrap(); + } + + #[cfg(unix)] + #[test] + fn census_handles_seventy_large_sparse_generations_without_loading_containers() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir_all(&root).unwrap(); + const FILE_BYTES: u64 = 220 * 1024 * 1024; + for index in 0..70 { + let name = format!("{index:064x}"); + let dir = root.join(&name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join(SEALED_STORE_RECEIPT_FILE), + receipt( + GraphFormatVersion::current().get(), + &name, + &format!("g{index}"), + ), + ) + .unwrap(); + std::fs::File::create(dir.join(SEALED_STORE_DATABASE_FILE)) + .unwrap() + .set_len(FILE_BYTES) + .unwrap(); + } + // A sealed directory must never account bytes outside its own tree. + // A missing target must also remain a harmless link, not abort census. + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::File::create(foreign.join("payload")) + .unwrap() + .set_len(20 * 1024 * 1024 * 1024) + .unwrap(); + let oldest = root.join("0".repeat(64)); + std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); + std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); + let census = census_sealed_store(&database, &BTreeSet::from(["g69".to_owned()])).unwrap(); + assert_eq!(census.head_count, 1); + assert_eq!(census.superseded_count, 69); + assert_eq!(census.unrecognized_count, 0); + assert!(census.head_bytes >= FILE_BYTES); + assert!(census.superseded_bytes >= 69 * FILE_BYTES); + assert!( + census.superseded_bytes < 70 * FILE_BYTES, + "foreign link targets must not contribute to sealed generation bytes" + ); + } +} + #[cfg(test)] mod build_tests { use std::collections::{BTreeMap, BTreeSet}; @@ -2995,6 +3387,74 @@ mod build_tests { ); } + /// A leftover staging symlink must not become a writable artifact root. + /// Refusing it preserves the foreign target rather than following it after + /// a best-effort cleanup that did not actually unlink the entry. + #[cfg(unix)] + #[test] + fn sealed_build_refuses_a_foreign_staging_symlink() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let directory = + sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + std::fs::create_dir_all(&root).unwrap(); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"unchanged").unwrap(); + std::os::unix::fs::symlink(&foreign, &staging).unwrap(); + + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!(result.is_err(), "a symlink must refuse the build"); + assert!( + std::fs::symlink_metadata(&staging) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"unchanged" + ); + assert!( + !foreign.join(SEALED_STORE_DATABASE_FILE).exists() + && !foreign.join("sealed.json").exists(), + "the foreign directory must not receive a container or receipt" + ); + assert!(!directory.exists(), "no generation was installed"); + + // Removing the foreign link must restore a successful, verified seal. + // A fixture that always refuses to build would not prove the fix. + std::fs::remove_file(&staging).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("a normal staging directory must still produce a sealed generation"); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!( + directory.join(SEALED_STORE_DATABASE_FILE).is_file(), + "valid staging must install a verified sealed container" + ); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index 123b4e9f3f7..c69dbf3222a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -1026,7 +1026,7 @@ fn wrong_tracedecay_format_requires_reset() { } #[test] -fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generations() { +fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); // Format 4 kept a relation's payload properties on its locator as well as @@ -1063,13 +1063,46 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati ) .unwrap(); raw.close().unwrap(); - let sealed_generation = temp.path().join("graph.sealed").join("0".repeat(64)); + let sealed_generation_name = "0".repeat(64); + let sealed_generation = temp + .path() + .join("graph.sealed") + .join(&sealed_generation_name); std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), b"format 4 bytes", ) .unwrap(); + // A prior-format sealed directory is deletable only when its receipt + // proves both the old format and the matching physical namespace. + std::fs::write( + sealed_generation.join("sealed.json"), + serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": "project", + "projection": "code", + "generation": "g0", + "physical_namespace": format!("generation:{sealed_generation_name}"), + "recovered_digest": "sha256:old", + "entities": 1, + "relations": 0, + "graph_format": previous_format as u32, + }) + .to_string(), + ) + .unwrap(); + + // An unverified artifact may belong to a future build. A format reset + // cannot infer its age or ownership from a directory name alone. + let unproven_generation = temp.path().join("graph.sealed").join("2".repeat(64)); + std::fs::create_dir_all(&unproven_generation).unwrap(); + std::fs::write( + unproven_generation.join("generation.grafeo"), + b"unknown sealed bytes", + ) + .unwrap(); // A generation this build sealed before the staging container was // replaced, as a publication racing the rebuild leaves it. @@ -1096,6 +1129,10 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati let (_registered, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); assert!(!sealed_generation.exists()); + assert!( + unproven_generation.join("generation.grafeo").exists(), + "an unverified generation must survive a format reset" + ); assert!(current_generation.join("sealed.json").exists()); assert_eq!( db.entity(&namespace(), &entity_id("stale"), live()) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c0c846fb8f4..15e7e6cf401 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -5,11 +5,19 @@ //! Every mutation is relative to an open `Dir` handle so a parent path //! swapped for a symlink cannot redirect the operation. +#[cfg(unix)] +use std::ffi::CString; use std::ffi::OsStr; use std::io; +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd}; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; +use std::path::Path; -#[cfg(windows)] use cap_fs_ext::DirExt; +#[cfg(windows)] +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; use cap_std::fs::{Dir, File, OpenOptions}; @@ -72,8 +80,6 @@ pub fn rename_noreplace( ) -> io::Result<()> { #[cfg(all(target_os = "linux", target_env = "gnu"))] { - use std::os::fd::AsRawFd; - crate::rename_noreplace::rename_noreplace_at( from_parent.as_raw_fd(), from, @@ -83,8 +89,6 @@ pub fn rename_noreplace( } #[cfg(target_os = "macos")] { - use std::os::fd::AsRawFd; - crate::rename_noreplace::rename_noreplace_at( from_parent.as_raw_fd(), from, @@ -212,6 +216,179 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { } } +fn single_component(name: &OsStr) -> io::Result<()> { + let mut parts = Path::new(name).components(); + if !matches!(parts.next(), Some(std::path::Component::Normal(_))) + || parts.next().is_some() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "directory operation requires one entry name", + )); + } + Ok(()) +} + +fn open_root_nofollow(root_path: &Path) -> io::Result { + let parent_path = root_path + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "sealed root has no parent"))?; + let root_name = root_path + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "sealed root has no name"))?; + let parent = Dir::open_ambient_dir(parent_path, cap_std::ambient_authority())?; + parent.open_dir_nofollow(root_name) +} + +/// Remove exactly one held child directory without following a substituted +/// sealed-root or child symlink. Missing entries are idempotent; other failures +/// remain errors so callers can retain retry ownership. +pub fn remove_child_directory_nofollow(root_path: &Path, name: &OsStr) -> io::Result { + single_component(name)?; + let root = match open_root_nofollow(root_path) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let child = match root.open_dir_nofollow(name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + remove_open_dir_all_nofollow(child)?; + Ok(true) +} + +/// Retire a named generation only when a bounded receipt read from the SAME +/// opened directory proves the requested identity. Missing is idempotent; +/// unknown receipts are refusals, never permission to discard durable bytes. +pub fn remove_receipted_child_directory_nofollow( + root_path: &Path, + child_name: &OsStr, + receipt_name: &OsStr, + maximum: usize, + authorized: impl FnOnce(&[u8]) -> bool, +) -> io::Result { + single_component(child_name)?; + single_component(receipt_name)?; + let root = match open_root_nofollow(root_path) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let child = match root.open_dir_nofollow(child_name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let receipt = read_bounded_child_receipt(&child, receipt_name, maximum)?; + if !authorized(&receipt) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sealed generation retirement receipt did not prove its ownership", + )); + } + remove_open_dir_all_nofollow(child)?; + Ok(true) +} + +/// Removes only children authorized by a bounded receipt read from the exact +/// directory handle that will be deleted. The sealed root and each child are +/// held capabilities: replacing either pathname after receipt validation +/// cannot redirect recursive removal to a different tree. +/// +/// The caller must own exclusive store maintenance. Unknown, invalid and +/// unreadable child receipts preserve their directory; an actual deletion +/// failure is returned so retirement can retry instead of claiming success. +pub fn remove_receipted_directories_nofollow( + root_path: &Path, + receipt_name: &OsStr, + maximum: usize, + mut authorized: impl FnMut(&OsStr, &[u8]) -> bool, +) -> io::Result { + // Receipt names are always a single sibling: a slash-bearing name could + // otherwise traverse a different directory even with O_NOFOLLOW. + let mut receipt_components = Path::new(receipt_name).components(); + if !matches!( + receipt_components.next(), + Some(std::path::Component::Normal(_)) + ) || receipt_components.next().is_some() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "receipt must be a single filename", + )); + } + let root = open_root_nofollow(root_path)?; + let mut removed = 0usize; + for entry in root.read_dir(".")? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let name = entry.file_name(); + // Do not trust the DirEntry's former pathname kind after a rename. + let child = match root.open_dir_nofollow(&name) { + Ok(child) => child, + Err(error) => { + tracing::warn!(%error, entry = ?name, "sealed child could not be verified"); + continue; + } + }; + let receipt = match read_bounded_child_receipt(&child, receipt_name, maximum) { + Ok(receipt) => receipt, + Err(error) => { + tracing::warn!(%error, entry = ?name, "unverified sealed receipt preserved"); + continue; + } + }; + if authorized(&name, &receipt) { + remove_open_dir_all_nofollow(child)?; + removed = removed.saturating_add(1); + } + } + Ok(removed) +} + +#[cfg(unix)] +fn read_bounded_child_receipt( + directory: &Dir, + name: &OsStr, + maximum: usize, +) -> io::Result> { + let name = CString::new(name.as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL") + })?; + // SAFETY: the held directory fd is valid; name is a NUL-terminated + // sibling component. O_NOFOLLOW prevents link traversal and O_NONBLOCK + // prevents a substituted FIFO from blocking before the regular-file check. + let descriptor = unsafe { + libc::openat( + directory.as_raw_fd(), + name.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, + ) + }; + if descriptor < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: openat returned a new owned fd and this File is its sole owner. + let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; + crate::framed_log::read_bounded_open_file(file, maximum) +} + +#[cfg(windows)] +fn read_bounded_child_receipt( + directory: &Dir, + name: &OsStr, + maximum: usize, +) -> io::Result> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let file = directory.open_with(name, &options)?.into_std(); + crate::framed_log::read_bounded_open_file(file, maximum) +} + /// Removes the held directory without following child symlinks. pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(not(windows))] @@ -373,6 +550,174 @@ mod tests { ); } + + /// A parent-path swap after receipt validation must not redirect cleanup: + /// the already-open child is removed, and matching foreign bytes survive. + #[cfg(unix)] + #[test] + fn verified_cleanup_remains_bound_when_sealed_root_is_substituted() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("owned-displaced"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join("a")).unwrap(); + std::fs::create_dir_all(foreign.join("a")).unwrap(); + std::fs::write(root.join("a/sealed.json"), b"owned receipt").unwrap(); + std::fs::write(root.join("a/owned"), b"owned").unwrap(); + std::fs::write(foreign.join("a/sentinel"), b"foreign").unwrap(); + + let removed = remove_receipted_directories_nofollow( + &root, + OsStr::new("sealed.json"), + 1024, + |name, bytes| { + assert_eq!(name, OsStr::new("a")); + assert_eq!(bytes, b"owned receipt"); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + true + }, + ) + .unwrap(); + assert_eq!(removed, 1); + assert!(!displaced.join("a").exists()); + assert_eq!(std::fs::read(foreign.join("a/sentinel")).unwrap(), b"foreign"); + } + + /// The child also remains the exact opened directory when its original + /// entry name becomes occupied by an unrelated real directory. + #[cfg(unix)] + #[test] + fn verified_cleanup_does_not_remove_a_replacement_real_child() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir_all(root.join("a")).unwrap(); + std::fs::write(root.join("a/sealed.json"), b"owned receipt").unwrap(); + let moved = root.join("moved"); + let removed = remove_receipted_directories_nofollow( + &root, + OsStr::new("sealed.json"), + 1024, + |_, bytes| { + assert_eq!(bytes, b"owned receipt"); + std::fs::rename(root.join("a"), &moved).unwrap(); + std::fs::create_dir(root.join("a")).unwrap(); + std::fs::write(root.join("a/sentinel"), b"foreign").unwrap(); + true + }, + ) + .unwrap(); + assert_eq!(removed, 1); + assert!(!moved.exists()); + assert_eq!(std::fs::read(root.join("a/sentinel")).unwrap(), b"foreign"); + } + + #[test] + fn a_receipt_filename_cannot_escape_its_held_child() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + std::fs::create_dir(&root).unwrap(); + for receipt in ["../outside", "a/b", "."] { + let error = remove_receipted_directories_nofollow( + &root, + OsStr::new(receipt), + 512, + |_, _| true, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } + } + + #[cfg(unix)] + #[test] + fn exact_receipted_retirement_preserves_unknown_and_swapped_real_children() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let child = root.join("a"); + let moved = root.join("moved"); + std::fs::create_dir_all(&child).unwrap(); + std::fs::write(child.join("sealed.json"), b"owned receipt").unwrap(); + std::fs::write(child.join("owned"), b"owned").unwrap(); + assert_eq!( + remove_receipted_child_directory_nofollow( + &root, + OsStr::new("a"), + OsStr::new("sealed.json"), + 1024, + |_| false, + ) + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied, + ); + assert_eq!(std::fs::read(child.join("owned")).unwrap(), b"owned"); + let removed = remove_receipted_child_directory_nofollow( + &root, + OsStr::new("a"), + OsStr::new("sealed.json"), + 1024, + |receipt| { + assert_eq!(receipt, b"owned receipt"); + std::fs::rename(&child, &moved).unwrap(); + std::fs::create_dir(&child).unwrap(); + std::fs::write(child.join("foreign"), b"preserve").unwrap(); + true + }, + ) + .unwrap(); + assert!(removed); + assert!( + !moved.exists(), + "only the receipt-authorized directory is deleted" + ); + assert_eq!( + std::fs::read(child.join("foreign")).unwrap(), + b"preserve" + ); + } + + #[cfg(unix)] + #[test] + fn exact_cleanup_refuses_a_substituted_root_symlink() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("original"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join("a")).unwrap(); + std::fs::create_dir_all(foreign.join("a")).unwrap(); + std::fs::write(root.join("a/sealed.json"), b"owned").unwrap(); + std::fs::write(foreign.join("a/sealed.json"), b"foreign").unwrap(); + std::fs::write(foreign.join("a/sentinel"), b"preserve").unwrap(); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + assert!(remove_child_directory_nofollow(&root, OsStr::new("a")).is_err()); + assert!( + remove_receipted_child_directory_nofollow( + &root, + OsStr::new("a"), + OsStr::new("sealed.json"), + 1024, + |_| true, + ) + .is_err() + ); + assert_eq!( + std::fs::read(foreign.join("a/sentinel")).unwrap(), + b"preserve" + ); + assert_eq!( + std::fs::read(displaced.join("a/sealed.json")).unwrap(), + b"owned" + ); + assert_eq!( + remove_child_directory_nofollow(&displaced, OsStr::new("../a")) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput, + ); + } + #[test] fn a_transient_missing_answer_is_looked_at_again() { let mut attempts = 0; diff --git a/crates/tracedecay-private-fs/src/framed_log.rs b/crates/tracedecay-private-fs/src/framed_log.rs index 005e20779c2..408bc6780e9 100644 --- a/crates/tracedecay-private-fs/src/framed_log.rs +++ b/crates/tracedecay-private-fs/src/framed_log.rs @@ -278,6 +278,13 @@ pub fn read_bounded(path: &Path, maximum: usize) -> io::Result>> Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(error), }; + read_bounded_open_file(file, maximum).map(Some) +} + +/// Applies the same bounded regular-file proof to a file opened relative to a +/// held directory capability. The caller supplies a no-follow, nonblocking +/// handle; this function never resolves another pathname. +pub(crate) fn read_bounded_open_file(file: File, maximum: usize) -> io::Result> { let metadata = file.metadata()?; if !metadata.file_type().is_file() { return Err(io::Error::new( @@ -300,7 +307,7 @@ pub fn read_bounded(path: &Path, maximum: usize) -> io::Result>> "bounded read length mismatch", )); } - Ok(Some(bytes)) + Ok(bytes) } const TEMPORARY_SUFFIX: &str = ".tmp"; From e304bfe22296d1237fd14bdd0ba878c2412361db Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:12:06 -0800 Subject: [PATCH 002/111] style(graph): apply Bazel rustfmt to sealed retirement Refs #4616 Reconcile all eleven formatting deltas from the exact-head Bazel repository-gates failure without modifying retirement behavior or tests. --- .../tracedecay-graph-db/src/sealed_store.rs | 25 ++++++++------ .../src/capability_dir.rs | 34 +++++++------------ 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 988c8ccf94e..f1f014c6eb1 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -763,10 +763,12 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) { let path = entry.path(); - if let Err(error) = tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( - &root, - &entry.file_name(), - ) { + if let Err(error) = + tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + &root, + &entry.file_name(), + ) + { tracing::warn!( event = "graph_row_spill_sweep_failed", path = %path.display(), @@ -807,9 +809,9 @@ fn retire_sealed_directory_checked( "sealed retirement directory does not match generation identity", )); } - let name = directory - .file_name() - .ok_or_else(|| GraphDbError::unavailable("sealed retirement directory has no entry name"))?; + let name = directory.file_name().ok_or_else(|| { + GraphDbError::unavailable("sealed retirement directory has no entry name") + })?; let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( root, name, @@ -3041,7 +3043,11 @@ mod receipt_budget_tests { let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); std::fs::write( &receipt_path, - receipt(GraphFormatVersion::current().get(), name, "generation:other"), + receipt( + GraphFormatVersion::current().get(), + name, + "generation:other", + ), ) .unwrap(); std::fs::write(directory.join("sentinel"), b"preserve").unwrap(); @@ -3400,8 +3406,7 @@ mod build_tests { let identity = manifest.identity(); let expected = manifest.expected_recovered_digest(check).unwrap(); let root = sealed_store_root(&database_path); - let directory = - sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); let staging = root.join(format!( ".staging-{}", directory.file_name().unwrap().to_str().unwrap() diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 15e7e6cf401..4b4013345fb 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -16,10 +16,10 @@ use std::os::unix::ffi::OsStrExt; use std::path::Path; use cap_fs_ext::DirExt; -#[cfg(windows)] -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; +#[cfg(windows)] +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, File, OpenOptions}; /// How many times a create-or-open that reports the entry missing is asked @@ -218,9 +218,7 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { fn single_component(name: &OsStr) -> io::Result<()> { let mut parts = Path::new(name).components(); - if !matches!(parts.next(), Some(std::path::Component::Normal(_))) - || parts.next().is_some() - { + if !matches!(parts.next(), Some(std::path::Component::Normal(_))) || parts.next().is_some() { return Err(io::Error::new( io::ErrorKind::InvalidInput, "directory operation requires one entry name", @@ -356,9 +354,8 @@ fn read_bounded_child_receipt( name: &OsStr, maximum: usize, ) -> io::Result> { - let name = CString::new(name.as_bytes()).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL") - })?; + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL"))?; // SAFETY: the held directory fd is valid; name is a NUL-terminated // sibling component. O_NOFOLLOW prevents link traversal and O_NONBLOCK // prevents a substituted FIFO from blocking before the regular-file check. @@ -550,7 +547,6 @@ mod tests { ); } - /// A parent-path swap after receipt validation must not redirect cleanup: /// the already-open child is removed, and matching foreign bytes survive. #[cfg(unix)] @@ -581,7 +577,10 @@ mod tests { .unwrap(); assert_eq!(removed, 1); assert!(!displaced.join("a").exists()); - assert_eq!(std::fs::read(foreign.join("a/sentinel")).unwrap(), b"foreign"); + assert_eq!( + std::fs::read(foreign.join("a/sentinel")).unwrap(), + b"foreign" + ); } /// The child also remains the exact opened directory when its original @@ -618,13 +617,9 @@ mod tests { let root = temp.path().join("sealed"); std::fs::create_dir(&root).unwrap(); for receipt in ["../outside", "a/b", "."] { - let error = remove_receipted_directories_nofollow( - &root, - OsStr::new(receipt), - 512, - |_, _| true, - ) - .unwrap_err(); + let error = + remove_receipted_directories_nofollow(&root, OsStr::new(receipt), 512, |_, _| true) + .unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); } } @@ -671,10 +666,7 @@ mod tests { !moved.exists(), "only the receipt-authorized directory is deleted" ); - assert_eq!( - std::fs::read(child.join("foreign")).unwrap(), - b"preserve" - ); + assert_eq!(std::fs::read(child.join("foreign")).unwrap(), b"preserve"); } #[cfg(unix)] From 3c66144e84acc390542d3c7dd9f1e0eabfb3a86c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:13:44 -0800 Subject: [PATCH 003/111] fix(graph): preserve unrelated directories during store recovery --- .../tracedecay-graph-db/src/sealed_store.rs | 27 ++++++++++++++----- .../sealed_store.rs | 14 +++++++++- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index f1f014c6eb1..d20e41665de 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -579,10 +579,7 @@ pub fn census_sealed_store( let path = entry.path(); let bytes = directory_bytes(&path, &mut counted)?; let name = entry.file_name(); - if name - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) - { + if name.to_str().is_some_and(sealed_staging_name) { census.abandoned_staging_count += 1; census.abandoned_staging_bytes += bytes; continue; @@ -683,6 +680,15 @@ pub(crate) fn discard_superseded_sealed_generations( } } +fn sealed_staging_name(name: &str) -> bool { + name.strip_prefix(".staging-").is_some_and(|suffix| { + suffix.len() == 64 + && suffix + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) +} + /// Removes every `.staging-*` directory a seal left behind under the store's /// sealed root. /// @@ -714,7 +720,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Some(name) = name.to_str() else { continue; }; - if !name.starts_with(".staging-") { + if !sealed_staging_name(name) { continue; } let path = entry.path(); @@ -755,12 +761,19 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let Ok(entries) = std::fs::read_dir(&root) else { return; }; - let own = format!("{ROW_SPILL_PREFIX}{}-", std::process::id()); + let own_pid = std::process::id(); for entry in entries.flatten() { if entry .file_name() .to_str() - .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) + .and_then(|name| name.strip_prefix(ROW_SPILL_PREFIX)) + .and_then(|suffix| suffix.split_once('-')) + .is_some_and(|(pid, sequence)| { + let (Ok(owner), Ok(counter)) = (pid.parse::(), sequence.parse::()) else { + return false; + }; + owner != own_pid && owner.to_string() == pid && counter.to_string() == sequence + }) { let path = entry.path(); if let Err(error) = diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 3bb165ee0f6..7d360753e71 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1263,6 +1263,9 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { .map(|entry| entry.path()) .find(|path| path.join("sealed.json").is_file()) .expect("the sealed artifact directory must exist"); + let unrelated = root.join(".staging-notes"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("notes"), b"keep").unwrap(); let abandoned = [ root.join(format!(".staging-{}", "c".repeat(64))), root.join(format!( @@ -1287,6 +1290,7 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { &record.publication.key.projection, ) .unwrap(); + assert_eq!(std::fs::read(unrelated.join("notes")).unwrap(), b"keep"); for staging in &abandoned { assert!( !staging.exists(), @@ -2371,7 +2375,15 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { let own = format!(".rows-{}-999999", std::process::id()); std::fs::create_dir_all(root.join(&own)).unwrap(); + let foreign = ".rows-notes"; + std::fs::create_dir(root.join(foreign)).unwrap(); + std::fs::write(root.join(foreign).join("notes"), b"keep").unwrap(); let _registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); - assert_eq!(row_spills(temp.path()), vec![own]); + assert_eq!( + std::fs::read(root.join(foreign).join("notes")).unwrap(), + b"keep" + ); + assert!(root.join(own).is_dir()); + assert!(!root.join(".rows-4194305-0").exists()); } From 65bd98f5b4a49250ecf9a2d3094f06d5fd3ca0b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:16:38 -0800 Subject: [PATCH 004/111] fix(graph): refuse unverified sealed paths during rebuild Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 130 +++++++++++++++++- 1 file changed, 125 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index d20e41665de..a4608aa6258 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1862,6 +1862,7 @@ fn build_or_open_sealed_store( // Idempotent replay: an artifact from an earlier seal of this exact // generation is adopted if its receipt binds the same digest. Adoption // never enumerates `source`'s rows, so it yields no staging proof. + let locator = GenerationLocator::new(identity.projection.clone(), identity.generation.clone()); match open_sealed_store_checked( &directory, identity, @@ -1870,11 +1871,26 @@ fn build_or_open_sealed_store( trusted_base.as_ref(), ) { Ok(Some(store)) => return Ok((store, None)), - Ok(None) => {} - Err(_) => remove_sealed_directory(&directory), - } - if directory.exists() { - remove_sealed_directory(&directory); + Ok(None) => match std::fs::symlink_metadata(&directory) { + Ok(_) => { + return Err(GraphDbError::unavailable( + "unverified sealed generation occupies the artifact directory", + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(sealed_store_io_failure( + "sealed artifact metadata read failed", + error, + )); + } + }, + Err(GraphDbError::Cancelled) => return Err(GraphDbError::Cancelled), + Err(_) => { + // A corrupt owned artifact may be rebuilt, but an unknown receipt + // (or a replacement directory) never grants deletion authority. + retire_sealed_directory_checked(&directory, &locator, Some(expected.as_str()))?; + } } let staging = root.join(format!( ".staging-{}", @@ -1887,6 +1903,15 @@ fn build_or_open_sealed_store( // accept a pre-existing staging symlink and write into its foreign target. std::fs::create_dir_all(&root) .map_err(|error| sealed_store_io_failure("sealed root create failed", error))?; + // create_dir_all accepts symlinked directory components. Refuse a + // preexisting substituted sealed root before writing staging bytes. + let root_metadata = std::fs::symlink_metadata(&root) + .map_err(|error| sealed_store_io_failure("sealed root metadata failed", error))?; + if !root_metadata.is_dir() { + return Err(GraphDbError::unavailable( + "sealed root must be a real directory", + )); + } let staging_name = staging .file_name() .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; @@ -3406,6 +3431,101 @@ mod build_tests { ); } + /// An unverified existing generation directory cannot be treated as + /// disposable scratch space. Rebuilding is allowed only after its owner + /// removes the unknown data. + #[test] + fn sealed_build_preserves_an_unverified_occupied_generation() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + std::fs::create_dir_all(&directory).unwrap(); + let sentinel = directory.join("foreign-sentinel"); + std::fs::write(&sentinel, b"not owned").unwrap(); + + for foreign_receipt in [false, true] { + if foreign_receipt { + std::fs::write(directory.join(SEALED_STORE_RECEIPT_FILE), b"invalid receipt") + .unwrap(); + } + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!( + result.is_err(), + "unreceipted or malformed foreign data must refuse rebuilding" + ); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"not owned"); + assert!( + !directory.join(SEALED_STORE_DATABASE_FILE).exists(), + "refused rebuilding must never write a container into foreign data" + ); + } + + std::fs::remove_dir_all(&directory).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("clearing the unknown artifact allows a verified sealed build"); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + } + + #[cfg(unix)] + #[test] + fn sealed_build_refuses_a_foreign_root_symlink() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"not owned").unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!(result.is_err(), "a symlinked sealed root must refuse writes"); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"not owned"); + assert_eq!( + std::fs::read_dir(&foreign).unwrap().count(), + 1, + "no staged container or receipt may be written into the foreign root" + ); + + std::fs::remove_file(&root).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("a genuine sealed root still supports building"); + assert_eq!(store.recovered_digest(), expected.as_str()); + } + /// A leftover staging symlink must not become a writable artifact root. /// Refusing it preserves the foreign target rather than following it after /// a best-effort cleanup that did not actually unlink the entry. From 3d9892140f4d33b40b6bc087a41bdca7e1e8d1a9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:43:07 -0800 Subject: [PATCH 005/111] style(graph): format sealed generation retirement tests --- crates/tracedecay-graph-db/src/sealed_store.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a4608aa6258..7f063d87f50 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3450,8 +3450,11 @@ mod build_tests { for foreign_receipt in [false, true] { if foreign_receipt { - std::fs::write(directory.join(SEALED_STORE_RECEIPT_FILE), b"invalid receipt") - .unwrap(); + std::fs::write( + directory.join(SEALED_STORE_RECEIPT_FILE), + b"invalid receipt", + ) + .unwrap(); } let result = build_or_open_sealed_store( SealedRowSource::Manifest(&manifest), @@ -3506,8 +3509,14 @@ mod build_tests { &database_path, check, ); - assert!(result.is_err(), "a symlinked sealed root must refuse writes"); - assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"not owned"); + assert!( + result.is_err(), + "a symlinked sealed root must refuse writes" + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"not owned" + ); assert_eq!( std::fs::read_dir(&foreign).unwrap().count(), 1, From c5b89807d057724812f700c8078ec082c1f4cfdd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 006/111] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From c22406e1840698932f7e5f9f1fc7b6117865f88d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:02:17 -0800 Subject: [PATCH 007/111] fix(graph): verify failed-adoption cleanup before unlink Refs #4616 Use the strict generation/digest-bound retirement authority on failed recovery adoption and post-install reopen failures. Preserve unverified artifacts instead of path-deleting them. Cover missing, corrupt, wrong- digest and owned receipts plus unrelated sibling preservation. --- .../tracedecay-graph-db/src/sealed_store.rs | 84 +++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7f063d87f50..2422c4f5d6d 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -858,6 +858,24 @@ fn retire_sealed_directory_checked( /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted /// replay instead uses the checked path, so a failed unlink cannot be reported /// as a completed retirement and lose its durable retry ownership. +/// Best-effort disposal after failed adoption/reopen. Only an exact durable +/// generation receipt authorizes removal; foreign bytes remain for diagnosis. +fn discard_failed_sealed_adoption( + directory: &Path, + locator: &GenerationLocator, + expected_digest: &str, +) { + if let Err(error) = + retire_sealed_directory_checked(directory, locator, Some(expected_digest)) + { + tracing::warn!( + %error, + path = %directory.display(), + "failed sealed adoption left an unverified artifact for inspection" + ); + } +} + fn remove_sealed_directory(directory: &Path) { if let Err(error) = remove_sealed_directory_checked(directory) { tracing::warn!( @@ -1644,10 +1662,16 @@ impl GraphDb { Ok(Some(store)) => self.install_sealed_generation_store(locator, store), Ok(None) => Ok(()), Err(error @ (GraphDbError::Cancelled | GraphDbError::DeadlineExceeded)) => Err(error), - Err(_) => { - // A stale or corrupt artifact never outranks the verified - // staging rows; discard it so a later seal can rebuild. - remove_sealed_directory(&directory); + Err(error) => { + // A failed adoption does not prove this directory belongs to + // the generation. Preserve unknown bytes rather than removing + // an occupied path without a matching durable receipt. + discard_failed_sealed_adoption(&directory, &locator, expected.as_str()); + tracing::warn!( + %error, + path = %directory.display(), + "sealed adoption failed; serving falls back to staging rows" + ); Ok(()) } } @@ -1954,13 +1978,13 @@ fn build_or_open_sealed_store( Ok((store, staging_proof)) } Ok(None) => { - remove_sealed_directory(&directory); + discard_failed_sealed_adoption(&directory, &locator, expected.as_str()); Err(GraphDbError::unavailable( "sealed generation store disappeared between install and reopen".to_owned(), )) } Err(error) => { - remove_sealed_directory(&directory); + discard_failed_sealed_adoption(&directory, &locator, expected.as_str()); Err(error) } } @@ -3062,6 +3086,54 @@ mod receipt_budget_tests { } } + #[test] + fn failed_adoption_cannot_discard_unverified_or_foreign_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new("generation:retirement").unwrap(), + ); + let directory = + sealed_generation_directory(&root, &locator.physical_namespace().unwrap()); + std::fs::create_dir_all(&directory).unwrap(); + let name = directory.file_name().unwrap().to_str().unwrap(); + let sentinel = directory.join("foreign-sentinel"); + std::fs::write(&sentinel, b"preserve").unwrap(); + let sibling = root.join("unrelated"); + std::fs::create_dir(&sibling).unwrap(); + std::fs::write(sibling.join("sentinel"), b"other").unwrap(); + + // Neither a missing nor a malformed receipt proves ownership. + discard_failed_sealed_adoption(&directory, &locator, "digest"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); + let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); + std::fs::write(&receipt_path, b"invalid receipt").unwrap(); + discard_failed_sealed_adoption(&directory, &locator, "digest"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); + + // An otherwise recognizable receipt still cannot retire a different + // expected digest. The exact matching receipt allows cleanup. + std::fs::write( + &receipt_path, + receipt( + GraphFormatVersion::current().get(), + name, + "generation:retirement", + ), + ) + .unwrap(); + discard_failed_sealed_adoption(&directory, &locator, "other-digest"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); + discard_failed_sealed_adoption(&directory, &locator, "digest"); + assert!(!directory.exists(), "owned invalid artifacts can be reclaimed"); + assert_eq!(std::fs::read(sibling.join("sentinel")).unwrap(), b"other"); + } + #[test] fn retired_sealed_generation_requires_its_exact_receipt_identity() { let temp = tempfile::tempdir().unwrap(); From 7c9a6f479c7804d70520d2b7313921a28a494eb5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:22:45 -0800 Subject: [PATCH 008/111] style(graph): format failed adoption cleanup --- crates/tracedecay-graph-db/src/sealed_store.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 2422c4f5d6d..46137d5a209 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -865,9 +865,7 @@ fn discard_failed_sealed_adoption( locator: &GenerationLocator, expected_digest: &str, ) { - if let Err(error) = - retire_sealed_directory_checked(directory, locator, Some(expected_digest)) - { + if let Err(error) = retire_sealed_directory_checked(directory, locator, Some(expected_digest)) { tracing::warn!( %error, path = %directory.display(), @@ -3098,8 +3096,7 @@ mod receipt_budget_tests { ), crate::GraphGenerationId::new("generation:retirement").unwrap(), ); - let directory = - sealed_generation_directory(&root, &locator.physical_namespace().unwrap()); + let directory = sealed_generation_directory(&root, &locator.physical_namespace().unwrap()); std::fs::create_dir_all(&directory).unwrap(); let name = directory.file_name().unwrap().to_str().unwrap(); let sentinel = directory.join("foreign-sentinel"); @@ -3130,7 +3127,10 @@ mod receipt_budget_tests { discard_failed_sealed_adoption(&directory, &locator, "other-digest"); assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); discard_failed_sealed_adoption(&directory, &locator, "digest"); - assert!(!directory.exists(), "owned invalid artifacts can be reclaimed"); + assert!( + !directory.exists(), + "owned invalid artifacts can be reclaimed" + ); assert_eq!(std::fs::read(sibling.join("sentinel")).unwrap(), b"other"); } From 882d64297c3a98c0a0d032957aed817bab84ed4f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:39:50 -0800 Subject: [PATCH 009/111] fix(graph): refuse occupied sealed generation installs Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 7 +- .../src/capability_dir.rs | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 46137d5a209..207d54fe378 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -54,6 +54,7 @@ use grafeo_core::graph::compact::IncrementalCompactStoreBuilder; use grafeo_engine::GrafeoDB; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nofollow; use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ @@ -1948,7 +1949,11 @@ fn build_or_open_sealed_store( std::fs::write(staging.join(SEALED_STORE_RECEIPT_FILE), encoded) .inspect_err(|_| remove_sealed_directory(&staging)) .map_err(|error| sealed_store_io_failure("receipt write failed", error))?; - if let Err(error) = std::fs::rename(&staging, &directory) { + let artifact_name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact name missing"))?; + let install = install_child_directory_noreplace_nofollow(&root, staging_name, artifact_name); + if let Err(error) = install { remove_sealed_directory(&staging); // A concurrent seal of the same generation may have installed the // directory first; adopting it below keeps this path idempotent. diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 4b4013345fb..56983fb4eaf 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -238,6 +238,28 @@ fn open_root_nofollow(root_path: &Path) -> io::Result { parent.open_dir_nofollow(root_name) } +/// Install a staged generation without replacing even an empty destination +/// directory. The final component of the sealed root and the staging entry +/// must be real directories; a link or an occupied generation is not a +/// recoverable staging collision. +/// +/// The caller still owns the responsibility for the identity of the store +/// ancestors, which this pathname API cannot establish. +pub fn install_child_directory_noreplace_nofollow( + root_path: &Path, + staging_name: &OsStr, + generation_name: &OsStr, +) -> io::Result<()> { + single_component(staging_name)?; + single_component(generation_name)?; + let root = open_root_nofollow(root_path)?; + let staging = root.open_dir_nofollow(staging_name)?; + // Reject a linked staging entry, then close the extra directory handle + // before publication for platforms that disallow renaming open children. + drop(staging); + rename_noreplace(&root, staging_name, &root, generation_name) +} + /// Remove exactly one held child directory without following a substituted /// sealed-root or child symlink. Missing entries are idempotent; other failures /// remain errors so callers can retain retry ownership. @@ -490,6 +512,54 @@ mod tests { Dir::open_ambient_dir(path, ambient_authority()).expect("open test directory") } + #[test] + fn staged_generation_install_preserves_an_occupied_empty_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + std::fs::create_dir(&root).unwrap(); + let staged_name = OsStr::new(".staging-generation"); + let installed_name = OsStr::new("generation"); + let staged = root.join(staged_name); + let installed = root.join(installed_name); + std::fs::create_dir(&staged).unwrap(); + std::fs::write(staged.join("sealed.json"), b"built").unwrap(); + std::fs::create_dir(&installed).unwrap(); + + let result = + install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::AlreadyExists); + assert!(installed.is_dir(), "an empty foreign target is not expendable"); + assert_eq!(std::fs::read(staged.join("sealed.json")).unwrap(), b"built"); + + std::fs::remove_dir(&installed).unwrap(); + install_child_directory_noreplace_nofollow(&root, staged_name, installed_name).unwrap(); + assert!(!staged.exists()); + assert_eq!(std::fs::read(installed.join("sealed.json")).unwrap(), b"built"); + } + + #[cfg(unix)] + #[test] + fn staged_generation_install_refuses_linked_root_and_staging_entries() { + let temp = tempfile::tempdir().unwrap(); + let foreign = temp.path().join("foreign"); + let real = temp.path().join("real"); + let link = temp.path().join("link"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::create_dir(&real).unwrap(); + std::fs::write(foreign.join("sentinel"), b"preserve").unwrap(); + std::fs::create_dir(foreign.join("staged")).unwrap(); + std::os::unix::fs::symlink(&foreign, &link).unwrap(); + let stage = OsStr::new("staged"); + let published = OsStr::new("installed"); + assert!(install_child_directory_noreplace_nofollow(&link, stage, published).is_err()); + + std::os::unix::fs::symlink(&foreign, real.join("staged")).unwrap(); + assert!(install_child_directory_noreplace_nofollow(&real, stage, published).is_err()); + assert!(!foreign.join("installed").exists()); + assert!(!real.join("installed").exists()); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"preserve"); + } + #[test] fn rename_noreplace_moves_between_parents_and_refuses_occupied_destinations() { let root = tempfile::tempdir().expect("create rename fixture"); From dacd4a0e1e86f79300ac3c92c79e773ea66f8dbe Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:05:18 -0800 Subject: [PATCH 010/111] style(storage): format no-replace directory installation --- .../src/capability_dir.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 56983fb4eaf..d04bea7d3fe 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -525,16 +525,21 @@ mod tests { std::fs::write(staged.join("sealed.json"), b"built").unwrap(); std::fs::create_dir(&installed).unwrap(); - let result = - install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); + let result = install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); assert_eq!(result.unwrap_err().kind(), io::ErrorKind::AlreadyExists); - assert!(installed.is_dir(), "an empty foreign target is not expendable"); + assert!( + installed.is_dir(), + "an empty foreign target is not expendable" + ); assert_eq!(std::fs::read(staged.join("sealed.json")).unwrap(), b"built"); std::fs::remove_dir(&installed).unwrap(); install_child_directory_noreplace_nofollow(&root, staged_name, installed_name).unwrap(); assert!(!staged.exists()); - assert_eq!(std::fs::read(installed.join("sealed.json")).unwrap(), b"built"); + assert_eq!( + std::fs::read(installed.join("sealed.json")).unwrap(), + b"built" + ); } #[cfg(unix)] @@ -557,7 +562,10 @@ mod tests { assert!(install_child_directory_noreplace_nofollow(&real, stage, published).is_err()); assert!(!foreign.join("installed").exists()); assert!(!real.join("installed").exists()); - assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"preserve"); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"preserve" + ); } #[test] From e2f3371c632323e580be465ffa5fee27f4b4b7cf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:31:08 -0800 Subject: [PATCH 011/111] test(graph): preserve unverified sealed artifacts on recovery Refs #4616 --- .../verified_generation_contract/sealed_store.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 7d360753e71..c2f11b96dc8 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1187,8 +1187,8 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { assert!(registered.close().unwrap()); drop(registered); - // Tamper with the receipt: recovery must discard the artifact and serve - // from the staging database. + // Tamper with the receipt: recovery must refuse adoption and serve from + // staging, without deleting bytes that no longer prove our ownership. let root = sealed_store_root(temp.path()); let mut tampered = None; for entry in std::fs::read_dir(&root).unwrap().map(Result::unwrap) { @@ -1200,6 +1200,7 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { } } let tampered = tampered.expect("the sealed artifact directory must exist"); + let tampered_receipt = std::fs::read(tampered.join("sealed.json")).unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let (control, probe) = control_and_probe(); let context = GraphPublicationOperationContextV1::new(&control, &probe).unwrap(); @@ -1217,8 +1218,13 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { "a tampered artifact must not be adopted" ); assert!( - !tampered.exists(), - "a tampered artifact must be discarded from disk" + tampered.is_dir(), + "recovery must preserve an artifact with an unverified receipt" + ); + assert_eq!( + std::fs::read(tampered.join("sealed.json")).unwrap(), + tampered_receipt, + "recovery must not modify the unverified receipt" ); assert_snapshot_reads(&snapshot, &identity, "durable"); } From b568770d301cabd1337549834e57b92b6aa62a8d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:36:00 -0800 Subject: [PATCH 012/111] fix(graph): refuse occupied sealed staging on rebuild Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 63 ++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 207d54fe378..6d3903d7165 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1922,8 +1922,8 @@ fn build_or_open_sealed_store( .and_then(|name| name.to_str()) .unwrap_or("sealed") )); - // A failed cleanup must never be followed by create_dir_all: it would - // accept a pre-existing staging symlink and write into its foreign target. + // create_dir_all may accept a symlinked sealed root, so verify the actual + // root before creating the staging entry or writing a container. std::fs::create_dir_all(&root) .map_err(|error| sealed_store_io_failure("sealed root create failed", error))?; // create_dir_all accepts symlinked directory components. Refuse a @@ -1938,8 +1938,9 @@ fn build_or_open_sealed_store( let staging_name = staging .file_name() .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; - tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(&root, staging_name) - .map_err(|error| sealed_store_io_failure("staging directory cleanup refused", error))?; + // A deterministic staging name is not proof of ownership. An earlier + // interrupted build or a foreign directory must not be deleted on retry. + // create_dir is exclusive, so an occupied name fails closed. std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check) @@ -3612,9 +3613,8 @@ mod build_tests { assert_eq!(store.recovered_digest(), expected.as_str()); } - /// A leftover staging symlink must not become a writable artifact root. - /// Refusing it preserves the foreign target rather than following it after - /// a best-effort cleanup that did not actually unlink the entry. + /// A pre-existing staging symlink must not become a writable artifact + /// root or be unlinked as disposable scratch space. #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_staging_symlink() { @@ -3679,6 +3679,55 @@ mod build_tests { ); } + /// A pre-existing real staging directory is not disposable merely + /// because its name matches the generation being built. + #[test] + fn sealed_build_preserves_a_foreign_real_staging_directory() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + std::fs::create_dir_all(&staging).unwrap(); + let sentinel = staging.join("foreign-sentinel"); + std::fs::write(&sentinel, b"must survive").unwrap(); + + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!(result.is_err(), "an occupied staging entry must refuse sealing"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"must survive"); + assert!( + !staging.join(SEALED_STORE_DATABASE_FILE).exists() + && !staging.join(SEALED_STORE_RECEIPT_FILE).exists(), + "refused sealing must not modify foreign staging data" + ); + assert!(!directory.exists(), "no generation may be installed"); + + std::fs::remove_dir_all(&staging).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("clearing the unknown stage allows a verified seal"); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves From 5de9d92cb05f93e4c3324e291ed5ac387bca0897 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:42:34 -0800 Subject: [PATCH 013/111] fix(graph): require receipts for abandoned staging cleanup Refs #4616 Do not treat a staging-shaped name as deletion authority. Retire only staged children with bounded matching sealed-generation receipts through held no-follow capabilities, preserving unknown or substituted contents. Exercise real store reopen for owned cleanup and unverified preservation. --- .../tracedecay-graph-db/src/sealed_store.rs | 102 +++++++++--------- .../sealed_store.rs | 92 ++++++++++++---- 2 files changed, 121 insertions(+), 73 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 6d3903d7165..08b192f268b 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -690,62 +690,64 @@ fn sealed_staging_name(name: &str) -> bool { }) } -/// Removes every `.staging-*` directory a seal left behind under the store's -/// sealed root. +/// Reclaims interrupted seals only when the bounded receipt from the same +/// opened staging directory proves its exact generation identity. The staging +/// filename alone is not deletion authority: a missing, malformed, oversized, +/// or foreign receipt leaves the directory intact for inspection. /// -/// A seal builds into `.staging-` and installs by rename, so a staging -/// directory that survives to the next open of the same store belongs to a -/// build the process never finished, a crash, a kill, or an OOM between -/// container write and rename. Nothing reads it and the next seal of that -/// generation starts over, so on the live profile these accumulated to 7.4 GB -/// under one store. Run at open: the exclusive store lock means no seal of -/// this store is in flight. +/// Store open owns exclusive maintenance admission. Capability-relative +/// traversal holds the root and child across verification and recursive removal, +/// so a pathname replacement cannot redirect deletion into a different tree. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, - Err(error) => { - tracing::warn!( - event = "sealed_staging_sweep_unreadable", - root = %root.display(), - error = %error, - "sealed store root could not be enumerated; abandoned staging stays" - ); - return; - } - }; - let mut removed = 0usize; - for entry in entries.flatten() { - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !sealed_staging_name(name) { - continue; - } - let path = entry.path(); - match tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( - &root, - &entry.file_name(), - ) { - Ok(true) => removed += 1, - Ok(false) => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - path = %path.display(), - error = %error, - "abandoned sealed staging directory could not be removed" - ), - } - } - if removed > 0 { - tracing::info!( + let reclaimed = tracedecay_private_fs::capability_dir::remove_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), + MAX_SEALED_RECEIPT_BYTES, + |name, bytes| { + let Some(name) = name.to_str() else { + return false; + }; + if !sealed_staging_name(name) { + return false; + } + let Some(generation_hex) = name.strip_prefix(".staging-") else { + return false; + }; + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return false; + }; + let recognized_form = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + recognized_form + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.graph_format > 0 + && receipt.graph_format <= GraphFormatVersion::current().get() + && receipt.physical_namespace == format!("generation:{generation_hex}") + && !receipt.namespace.is_empty() + && !receipt.projection.is_empty() + && !receipt.generation.is_empty() + && !receipt.recovered_digest.is_empty() + }, + ); + match reclaimed { + Ok(removed) if removed > 0 => tracing::info!( event = "sealed_staging_swept", root = %root.display(), removed, - "removed abandoned sealed staging directories left by interrupted seals" - ); + "removed receipt-verified staging from interrupted seals" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned staging stays for inspection" + ), } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index c2f11b96dc8..a965d9f9a15 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1229,12 +1229,11 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { assert_snapshot_reads(&snapshot, &identity, "durable"); } -/// A seal that died between container write and rename leaves -/// `.staging-` under the sealed root. Nothing reads it, so the next -/// open of the store removes every such directory while leaving installed -/// artifacts untouched. +/// A completed interrupted seal carries an exact generation receipt and is +/// reclaimed when reopening the store. Unreceipted, mismatched, oversized and +/// linked lookalikes must survive instead of being deleted by filename. #[test] -fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { +fn reopening_the_store_reclaims_only_receipted_staging() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let mut authority = RelationalAuthority::default(); @@ -1260,28 +1259,47 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { assert!(registered.close().unwrap()); drop(registered); - // Two interrupted seals: one of a generation that never installed and one - // whose digest matches the installed artifact but was started again. let root = sealed_store_root(temp.path()); let installed = std::fs::read_dir(&root) .unwrap() .map(Result::unwrap) .map(|entry| entry.path()) .find(|path| path.join("sealed.json").is_file()) - .expect("the sealed artifact directory must exist"); + .expect("the installed sealed artifact directory must exist"); + let receipt = std::fs::read(installed.join("sealed.json")).unwrap(); + let installed_name = installed.file_name().unwrap().to_str().unwrap(); + let verified = root.join(format!(".staging-{installed_name}")); + std::fs::create_dir(&verified).unwrap(); + std::fs::copy( + installed.join("generation.grafeo"), + verified.join("generation.grafeo"), + ) + .unwrap(); + std::fs::write(verified.join("sealed.json"), &receipt).unwrap(); + + let unowned = root.join(format!(".staging-{}", "c".repeat(64))); + std::fs::create_dir(&unowned).unwrap(); + std::fs::write(unowned.join("generation.grafeo"), b"foreign bytes").unwrap(); + let mismatched = root.join(format!(".staging-{}", "d".repeat(64))); + std::fs::create_dir(&mismatched).unwrap(); + std::fs::write(mismatched.join("sealed.json"), &receipt).unwrap(); + std::fs::write(mismatched.join("sentinel"), b"wrong identity").unwrap(); + let oversized = root.join(format!(".staging-{}", "e".repeat(64))); + std::fs::create_dir(&oversized).unwrap(); + std::fs::write(oversized.join("sealed.json"), vec![b'x'; 128 * 1024]).unwrap(); + std::fs::write(oversized.join("sentinel"), b"too large").unwrap(); let unrelated = root.join(".staging-notes"); std::fs::create_dir(&unrelated).unwrap(); std::fs::write(unrelated.join("notes"), b"keep").unwrap(); - let abandoned = [ - root.join(format!(".staging-{}", "c".repeat(64))), - root.join(format!( - ".staging-{}", - installed.file_name().unwrap().to_str().unwrap() - )), - ]; - for staging in &abandoned { - std::fs::create_dir_all(staging).unwrap(); - std::fs::write(staging.join("generation.grafeo"), b"partial container").unwrap(); + + #[cfg(unix)] + let outside = TempDir::new().unwrap(); + #[cfg(unix)] + let linked = root.join(format!(".staging-{}", "f".repeat(64))); + #[cfg(unix)] + { + std::fs::write(outside.path().join("sentinel"), b"outside").unwrap(); + std::os::unix::fs::symlink(outside.path(), &linked).unwrap(); } let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); @@ -1296,17 +1314,45 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { &record.publication.key.projection, ) .unwrap(); + + assert!(!verified.exists(), "a verified abandoned seal must be reclaimed"); + assert_eq!( + std::fs::read(unowned.join("generation.grafeo")).unwrap(), + b"foreign bytes", + "missing receipt is not deletion authority" + ); + assert_eq!( + std::fs::read(mismatched.join("sealed.json")).unwrap(), + receipt, + "a receipt for a different generation cannot authorize cleanup" + ); + assert_eq!( + std::fs::read(mismatched.join("sentinel")).unwrap(), + b"wrong identity" + ); + assert_eq!( + std::fs::read(oversized.join("sentinel")).unwrap(), + b"too large", + "oversized receipts must be refused without recursive removal" + ); assert_eq!(std::fs::read(unrelated.join("notes")).unwrap(), b"keep"); - for staging in &abandoned { + #[cfg(unix)] + { assert!( - !staging.exists(), - "opening the store removes abandoned staging {}", - staging.display() + std::fs::symlink_metadata(&linked) + .unwrap() + .file_type() + .is_symlink(), + "a linked staging entry must not be followed or removed" + ); + assert_eq!( + std::fs::read(outside.path().join("sentinel")).unwrap(), + b"outside" ); } assert!( installed.join("sealed.json").is_file(), - "the installed sealed artifact is untouched by the sweep" + "the installed generation must remain untouched" ); assert!(snapshot.serves_from_sealed_store()); assert_snapshot_reads(&snapshot, &identity, "durable"); From 0bf8a86853f66920ef8544b5cee26b773ac53a92 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:45:12 -0800 Subject: [PATCH 014/111] fix(graph): bind abandoned receipts to canonical generation identity Refs #4616 A copied receipt with rewritten physical namespace must not authorize recursive cleanup of an unrelated staging directory. Exercise the forgery and malformed-receipt refusals through real store reopen. --- .../tracedecay-graph-db/src/sealed_store.rs | 21 ++++++++++++-- .../sealed_store.rs | 28 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 08b192f268b..0f10cedbf37 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -717,6 +717,23 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Ok(receipt) = serde_json::from_slice::(bytes) else { return false; }; + // Bind the receipt's logical identity back to the canonical + // physical namespace. A copied receipt with a rewritten physical + // name is not proof of ownership of another staged generation. + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace.clone()), + GraphProjectionId::new(receipt.projection.clone()), + crate::GraphGenerationId::new(receipt.generation.clone()), + ) else { + return false; + }; + let locator = GenerationLocator::new( + GraphProjectionIdentity::new(namespace, projection), + generation, + ); + let Ok(physical_namespace) = locator.physical_namespace() else { + return false; + }; let recognized_form = match receipt.form.as_str() { SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), @@ -727,9 +744,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { && receipt.graph_format > 0 && receipt.graph_format <= GraphFormatVersion::current().get() && receipt.physical_namespace == format!("generation:{generation_hex}") - && !receipt.namespace.is_empty() - && !receipt.projection.is_empty() - && !receipt.generation.is_empty() + && receipt.physical_namespace == physical_namespace.as_str() && !receipt.recovered_digest.is_empty() }, ); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index a965d9f9a15..f983483cd4b 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1284,6 +1284,21 @@ fn reopening_the_store_reclaims_only_receipted_staging() { std::fs::create_dir(&mismatched).unwrap(); std::fs::write(mismatched.join("sealed.json"), &receipt).unwrap(); std::fs::write(mismatched.join("sentinel"), b"wrong identity").unwrap(); + // A copied receipt whose physical namespace was rewritten to fit the + // staging filename still does not bind the original logical generation. + let forged_name = "a".repeat(64); + let forged = root.join(format!(".staging-{forged_name}")); + std::fs::create_dir(&forged).unwrap(); + let mut forged_receipt: serde_json::Value = serde_json::from_slice(&receipt).unwrap(); + forged_receipt["physical_namespace"] = + serde_json::Value::String(format!("generation:{forged_name}")); + let forged_bytes = serde_json::to_vec(&forged_receipt).unwrap(); + std::fs::write(forged.join("sealed.json"), &forged_bytes).unwrap(); + std::fs::write(forged.join("sentinel"), b"forged receipt").unwrap(); + let malformed = root.join(format!(".staging-{}", "b".repeat(64))); + std::fs::create_dir(&malformed).unwrap(); + std::fs::write(malformed.join("sealed.json"), b"not-json").unwrap(); + std::fs::write(malformed.join("sentinel"), b"malformed receipt").unwrap(); let oversized = root.join(format!(".staging-{}", "e".repeat(64))); std::fs::create_dir(&oversized).unwrap(); std::fs::write(oversized.join("sealed.json"), vec![b'x'; 128 * 1024]).unwrap(); @@ -1330,6 +1345,19 @@ fn reopening_the_store_reclaims_only_receipted_staging() { std::fs::read(mismatched.join("sentinel")).unwrap(), b"wrong identity" ); + assert_eq!( + std::fs::read(forged.join("sealed.json")).unwrap(), + forged_bytes, + "a rewritten physical namespace is not generation ownership" + ); + assert_eq!( + std::fs::read(forged.join("sentinel")).unwrap(), + b"forged receipt" + ); + assert_eq!( + std::fs::read(malformed.join("sentinel")).unwrap(), + b"malformed receipt" + ); assert_eq!( std::fs::read(oversized.join("sentinel")).unwrap(), b"too large", From bd4f39eb6b85516919d9cbb3686f1a4dd7e4cea8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:58:20 -0800 Subject: [PATCH 015/111] test(graph): preserve unreceipted staging in crash regression Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 0f10cedbf37..687b3ce09f3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3977,11 +3977,32 @@ mod build_tests { "an interrupted direct build must leave neither its staging directory nor an artifact" ); - // A crash image: the process died after creating the staging - // directory and part of a container. The next attempt owns that - // directory and rebuilds from the manifest. + // A crash before the receipt exists leaves no deletion authority. + // A retry must preserve the unverified staging bytes, not silently + // treat a name collision as proof that the directory belongs to us. std::fs::create_dir_all(&staging).unwrap(); - std::fs::write(staging.join(SEALED_STORE_DATABASE_FILE), b"torn container").unwrap(); + let torn_container = staging.join(SEALED_STORE_DATABASE_FILE); + std::fs::write(&torn_container, b"torn container").unwrap(); + let refused = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!( + matches!(&refused, Err(GraphDbError::Unavailable { .. })), + "an unreceipted staging collision must fail closed: {refused:?}" + ); + assert_eq!(std::fs::read(&torn_container).unwrap(), b"torn container"); + assert!( + !directory.exists(), + "refused build must not publish a generation" + ); + + // The test fixture explicitly removes its unowned crash image. + // Only then may the same build succeed and prove its normal payload. + std::fs::remove_dir_all(&staging).unwrap(); let (direct, staging_proof) = build_or_open_sealed_store( SealedRowSource::Manifest(&manifest), &identity, @@ -3998,7 +4019,7 @@ mod build_tests { assert_eq!((direct.entity_count, direct.relation_count), (2_000, 3_000)); assert!( !staging.exists(), - "the crash image must be replaced, not kept" + "successful build must consume its staging directory" ); let _ = direct.database().close(); let direct_bytes = std::fs::read(&artifact).unwrap(); From 35abfc8c0b3a6e19d6b5a57ddb207ec400bc4b3c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:04:01 -0800 Subject: [PATCH 016/111] test(graph-db): import the canonical sealed receipt filename --- crates/tracedecay-graph-db/src/sealed_store.rs | 11 +++++++---- .../verified_generation_contract/sealed_store.rs | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 687b3ce09f3..8396576fabe 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3270,9 +3270,9 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - SEALED_STORE_DATABASE_FILE, SealedRowSource, build_or_open_sealed_store, - build_sealed_container, sealed_artifact_database_options, sealed_copy_proof, - sealed_generation_directory, sealed_store_root, + SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, + build_or_open_sealed_store, build_sealed_container, sealed_artifact_database_options, + sealed_copy_proof, sealed_generation_directory, sealed_store_root, }; use crate::location::PersistentGraphStoreState; use crate::{ @@ -3723,7 +3723,10 @@ mod build_tests { &database_path, check, ); - assert!(result.is_err(), "an occupied staging entry must refuse sealing"); + assert!( + result.is_err(), + "an occupied staging entry must refuse sealing" + ); assert_eq!(std::fs::read(&sentinel).unwrap(), b"must survive"); assert!( !staging.join(SEALED_STORE_DATABASE_FILE).exists() diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index f983483cd4b..05bd84f29a2 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1330,7 +1330,10 @@ fn reopening_the_store_reclaims_only_receipted_staging() { ) .unwrap(); - assert!(!verified.exists(), "a verified abandoned seal must be reclaimed"); + assert!( + !verified.exists(), + "a verified abandoned seal must be reclaimed" + ); assert_eq!( std::fs::read(unowned.join("generation.grafeo")).unwrap(), b"foreign bytes", From f3d8f4eecdabc42115ce89e1cba4e8087d2a8598 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:30:10 -0800 Subject: [PATCH 017/111] fix(graph): verify row spill ownership before reclaim Refs #4616 Preserve foreign row-spill directories on store-open sweeping and drop. Persist bounded owner receipts before writing rows; reclaim only from receipt-verified held directories, with native crash/replacement tests. --- crates/tracedecay-graph-db/src/generation.rs | 5 +- .../src/generation/spill.rs | 94 ++++++++++++++++++- .../tracedecay-graph-db/src/sealed_store.rs | 16 +++- 3 files changed, 106 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation.rs b/crates/tracedecay-graph-db/src/generation.rs index 7a78013cdc9..8f8b912aa49 100644 --- a/crates/tracedecay-graph-db/src/generation.rs +++ b/crates/tracedecay-graph-db/src/generation.rs @@ -59,7 +59,10 @@ pub use replay::{ pub(crate) use replay::{ checked_decode_replay_source, metadata_manifest_from_source, validate_supplied_rows_binding, }; -pub(crate) use spill::{ENTITIES_FILE, read_spilled_entity}; +pub(crate) use spill::{ + ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, + read_spilled_entity, row_spill_owner_receipt, +}; pub use spill::{ GRAPH_ROW_SPILL_RUN_BYTES, GraphGenerationRowSpill, GraphGenerationRows, GraphSpillRowFootprint, SpilledGraphGeneration, diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 92ff10db944..3ffdb687859 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -17,6 +17,7 @@ use std::cmp::{Ordering as CmpOrdering, Reverse}; use std::collections::BinaryHeap; +use std::ffi::OsStr; use std::fs::File; use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; @@ -59,13 +60,41 @@ const RELATIONS_FILE: &str = "relations.rows"; /// A producer-owned file that travels with the sealed generation, see /// [`GraphGenerationRowSpill::attachment_path`]. const ATTACHMENT_FILE: &str = "attachment"; +/// An abandoned row-spill name is not authority to erase an occupied directory. +/// Persist this marker before any rows; a crash before the marker leaves an +/// unverified directory for inspection rather than authorizing deletion. +pub(crate) const ROW_SPILL_OWNER_RECEIPT_FILE: &str = ".row-spill-owner-v1"; +pub(crate) const MAX_ROW_SPILL_OWNER_RECEIPT_BYTES: usize = 512; + +pub(crate) fn row_spill_owner_receipt(name: &OsStr) -> Option> { + let name = name.to_str()?; + let receipt = format!("tracedecay.graph-row-spill-v1:{name}").into_bytes(); + (receipt.len() <= MAX_ROW_SPILL_OWNER_RECEIPT_BYTES).then_some(receipt) +} /// Scratch directory owned by one spill; removed with its owner. struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { + let name = path + .file_name() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; + let receipt = row_spill_owner_receipt(name) + .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; std::fs::create_dir(&path).map_err(|error| spill_io("directory create", error))?; + // Exclusive marker creation refuses a substituted preexisting file. + // The marker is synced before row writes so crash recovery can verify + // this exact directory instead of deleting by its filename. + let marker = path.join(ROW_SPILL_OWNER_RECEIPT_FILE); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker) + .map_err(|error| spill_io("owner marker create", error))?; + file.write_all(&receipt) + .and_then(|()| file.sync_all()) + .map_err(|error| spill_io("owner marker persist", error))?; Ok(Self(path)) } @@ -76,14 +105,27 @@ impl SpillDirectory { impl Drop for SpillDirectory { fn drop(&mut self) { - match std::fs::remove_dir_all(&self.0) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + let Some((root, name)) = self.0.parent().zip(self.0.file_name()) else { + return; + }; + let Some(receipt) = row_spill_owner_receipt(name) else { + return; + }; + // The pathname may have been replaced since this spill was created. + // Read the proof and remove relative to the same held child capability. + match tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |bytes| bytes == receipt, + ) { + Ok(_) => {} Err(error) => tracing::warn!( event = "graph_row_spill_remove_failed", path = %self.0.display(), error = %error, - "graph row spill directory could not be removed; the next store open sweeps it" + "unverified graph row spill preserved for inspection" ), } } @@ -1037,3 +1079,47 @@ impl GraphGenerationManifestIdentity { .map_err(|error| GraphDbError::invalid(error.to_string())) } } + +#[cfg(test)] +mod owner_receipt_tests { + use super::{SpillDirectory, ROW_SPILL_OWNER_RECEIPT_FILE}; + + #[test] + fn row_spill_recovery_reclaims_only_proven_owners_and_drop_is_fail_closed() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let other_pid = std::process::id().wrapping_add(1); + let owned = root.join(format!(".rows-{other_pid}-1")); + let completed = SpillDirectory::create(owned.clone()).unwrap(); + std::fs::write(owned.join("entities-0.run"), b"owned").unwrap(); + std::mem::forget(completed); // Simulate a producer killed before Drop. + + let unreceipted = root.join(format!(".rows-{other_pid}-2")); + std::fs::create_dir(&unreceipted).unwrap(); + std::fs::write(unreceipted.join("foreign"), b"unowned").unwrap(); + let forged = root.join(format!(".rows-{other_pid}-3")); + let interrupted = SpillDirectory::create(forged.clone()).unwrap(); + std::fs::write(forged.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"wrong owner").unwrap(); + std::fs::write(forged.join("foreign"), b"preserve").unwrap(); + std::mem::forget(interrupted); + + crate::sealed_store::sweep_abandoned_row_spills(&database); + assert!(!owned.exists(), "the completed, marker-bound spill is reclaimed"); + assert_eq!(std::fs::read(unreceipted.join("foreign")).unwrap(), b"unowned"); + assert_eq!(std::fs::read(forged.join("foreign")).unwrap(), b"preserve"); + + let active = root.join(format!(".rows-{}-4", std::process::id())); + let spill = SpillDirectory::create(active.clone()).unwrap(); + drop(spill); + assert!(!active.exists(), "a current, proven spill cleans up on drop"); + + let substituted = root.join(format!(".rows-{}-5", std::process::id())); + let spill = SpillDirectory::create(substituted.clone()).unwrap(); + std::fs::write(substituted.join("foreign"), b"still here").unwrap(); + std::fs::write(substituted.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"substituted").unwrap(); + drop(spill); + assert_eq!(std::fs::read(substituted.join("foreign")).unwrap(), b"still here"); + } +} diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 8396576fabe..ec70c1fab78 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -58,7 +58,8 @@ use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nof use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ - GraphRowDigestSum, physical_namespace_projection_map, recovered_digest_from_row_sum, + GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, + physical_namespace_projection_map, recovered_digest_from_row_sum, row_spill_owner_receipt, verify_sealed_copy_generation, }; use crate::lease::GenerationLocator; @@ -793,18 +794,25 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { owner != own_pid && owner.to_string() == pid && counter.to_string() == sequence }) { + let name = entry.file_name(); + let Some(receipt) = row_spill_owner_receipt(&name) else { + continue; + }; let path = entry.path(); if let Err(error) = - tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( &root, - &entry.file_name(), + &name, + std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |bytes| bytes == receipt, ) { tracing::warn!( event = "graph_row_spill_sweep_failed", path = %path.display(), error = %error, - "abandoned graph row spill could not be removed" + "unverified abandoned graph row spill preserved for inspection" ); } } From 05c18a3c708eaa6c4b9d295c5bd57f480535a55d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:50:37 -0800 Subject: [PATCH 018/111] fix(graph): publish row spills only with a complete owner receipt --- .../src/generation/spill.rs | 93 ++++++++++++++++--- .../sealed_store.rs | 16 ++-- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 3ffdb687859..c83a5f36341 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -82,11 +82,14 @@ impl SpillDirectory { .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; let receipt = row_spill_owner_receipt(name) .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; - std::fs::create_dir(&path).map_err(|error| spill_io("directory create", error))?; - // Exclusive marker creation refuses a substituted preexisting file. - // The marker is synced before row writes so crash recovery can verify - // this exact directory instead of deleting by its filename. - let marker = path.join(ROW_SPILL_OWNER_RECEIPT_FILE); + let parent = path + .parent() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; + // Receipt failures must leave the requested name available to retry. + // Keep the partial marker private until the complete spill can be installed. + let mut staging = + tempfile::tempdir_in(parent).map_err(|error| spill_io("directory create", error))?; + let marker = staging.path().join(ROW_SPILL_OWNER_RECEIPT_FILE); let mut file = std::fs::OpenOptions::new() .write(true) .create_new(true) @@ -95,6 +98,10 @@ impl SpillDirectory { file.write_all(&receipt) .and_then(|()| file.sync_all()) .map_err(|error| spill_io("owner marker persist", error))?; + drop(file); + tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) + .map_err(|error| spill_io("directory install", error))?; + staging.disable_cleanup(true); Ok(Self(path)) } @@ -1082,7 +1089,55 @@ impl GraphGenerationManifestIdentity { #[cfg(test)] mod owner_receipt_tests { - use super::{SpillDirectory, ROW_SPILL_OWNER_RECEIPT_FILE}; + use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory}; + + #[cfg(unix)] + #[test] + fn receipt_write_failure_leaves_no_spill_directory() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_LIMITED_SPILL_ROOT"; + const WRITE_REFUSED: i32 = 42; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let destination = std::path::PathBuf::from(root).join(".rows-1-0"); + let error = SpillDirectory::create(destination) + .err() + .expect("zero file-size limit"); + assert!( + error.to_string().contains("owner marker persist"), + "{error}" + ); + std::process::exit(WRITE_REFUSED); + } + + // Limit only the child process. Its directory creation succeeds and + // the real filesystem refuses the first receipt write with EFBIG. + let root = tempfile::tempdir().unwrap(); + let child = std::process::Command::new("/bin/sh") + .args(["-c", r#"trap "" XFSZ && ulimit -f 0 && exec "$@""#, "sh"]) + .arg(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, root.path()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(WRITE_REFUSED), "{child:?}"); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } + + #[test] + fn occupied_spill_publication_preserves_destination_and_discards_staging() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join(".rows-1-0"); + std::fs::create_dir(&destination).unwrap(); + assert!(SpillDirectory::create(destination.clone()).is_err()); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 1); + assert_eq!(std::fs::read_dir(&destination).unwrap().count(), 0); + + std::fs::remove_dir(&destination).unwrap(); + let spill = SpillDirectory::create(destination.clone()).unwrap(); + assert!(destination.join(ROW_SPILL_OWNER_RECEIPT_FILE).is_file()); + drop(spill); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } #[test] fn row_spill_recovery_reclaims_only_proven_owners_and_drop_is_fail_closed() { @@ -1106,20 +1161,36 @@ mod owner_receipt_tests { std::mem::forget(interrupted); crate::sealed_store::sweep_abandoned_row_spills(&database); - assert!(!owned.exists(), "the completed, marker-bound spill is reclaimed"); - assert_eq!(std::fs::read(unreceipted.join("foreign")).unwrap(), b"unowned"); + assert!( + !owned.exists(), + "the completed, marker-bound spill is reclaimed" + ); + assert_eq!( + std::fs::read(unreceipted.join("foreign")).unwrap(), + b"unowned" + ); assert_eq!(std::fs::read(forged.join("foreign")).unwrap(), b"preserve"); let active = root.join(format!(".rows-{}-4", std::process::id())); let spill = SpillDirectory::create(active.clone()).unwrap(); drop(spill); - assert!(!active.exists(), "a current, proven spill cleans up on drop"); + assert!( + !active.exists(), + "a current, proven spill cleans up on drop" + ); let substituted = root.join(format!(".rows-{}-5", std::process::id())); let spill = SpillDirectory::create(substituted.clone()).unwrap(); std::fs::write(substituted.join("foreign"), b"still here").unwrap(); - std::fs::write(substituted.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"substituted").unwrap(); + std::fs::write( + substituted.join(ROW_SPILL_OWNER_RECEIPT_FILE), + b"substituted", + ) + .unwrap(); drop(spill); - assert_eq!(std::fs::read(substituted.join("foreign")).unwrap(), b"still here"); + assert_eq!( + std::fs::read(substituted.join("foreign")).unwrap(), + b"still here" + ); } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 05bd84f29a2..0f43b56e3a3 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -2453,10 +2453,14 @@ fn spilled_rows_refuse_conflicting_repeats_and_dangling_endpoints() { fn store_open_sweeps_row_spills_abandoned_by_another_process() { let temp = TempDir::new().unwrap(); let root = sealed_store_root(temp.path()); - std::fs::create_dir_all(root.join(".rows-4194305-0")).unwrap(); - std::fs::write(root.join(".rows-4194305-0/entities-0.run"), b"abandoned").unwrap(); - let own = format!(".rows-{}-999999", std::process::id()); - std::fs::create_dir_all(root.join(&own)).unwrap(); + std::fs::create_dir_all(&root).unwrap(); + let abandoned = root.join(".rows-4194305-0"); + let identity = projection("code", "store-open-spill"); + let spill = GraphGenerationRowSpill::create(abandoned.clone(), identity.clone()).unwrap(); + std::fs::write(abandoned.join("entities-0.run"), b"abandoned").unwrap(); + std::mem::forget(spill); // Simulate a producer that exited before Drop. + let own = root.join(format!(".rows-{}-999999", std::process::id())); + let _live_spill = GraphGenerationRowSpill::create(own.clone(), identity).unwrap(); let foreign = ".rows-notes"; std::fs::create_dir(root.join(foreign)).unwrap(); @@ -2467,6 +2471,6 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::read(root.join(foreign).join("notes")).unwrap(), b"keep" ); - assert!(root.join(own).is_dir()); - assert!(!root.join(".rows-4194305-0").exists()); + assert!(own.is_dir()); + assert!(!abandoned.exists()); } From d13622cb45887b7d8688a7b87185ec0217b4e2f4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:07:28 -0800 Subject: [PATCH 019/111] fix(graph): recover pending row spills after producer exit --- crates/tracedecay-graph-db/src/generation.rs | 4 +- .../src/generation/spill.rs | 130 ++++++++++++++++-- .../tracedecay-graph-db/src/sealed_store.rs | 29 +--- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation.rs b/crates/tracedecay-graph-db/src/generation.rs index 8f8b912aa49..2db4c376fb7 100644 --- a/crates/tracedecay-graph-db/src/generation.rs +++ b/crates/tracedecay-graph-db/src/generation.rs @@ -60,8 +60,8 @@ pub(crate) use replay::{ checked_decode_replay_source, metadata_manifest_from_source, validate_supplied_rows_binding, }; pub(crate) use spill::{ - ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, - read_spilled_entity, row_spill_owner_receipt, + AbandonedRowSpill, ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, read_spilled_entity, }; pub use spill::{ GRAPH_ROW_SPILL_RUN_BYTES, GraphGenerationRowSpill, GraphGenerationRows, diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index c83a5f36341..90949aef3f5 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -34,6 +34,8 @@ use tracedecay_store::runtime::{ StoreShardIdV1, }; +use tracedecay_domain::canonical_text::{canonical_framed_sha256, is_lowercase_hex}; + use crate::limits::{MAX_VERIFIED_GENERATION_ENTITIES, MAX_VERIFIED_GENERATION_RELATIONS}; use crate::row_index::{ ENTITY_ROW_OFFSETS_FILE, EntityRowOffsetsWriter, ROW_INDEX_FILE, RowIndexBuilder, @@ -66,17 +68,78 @@ const ATTACHMENT_FILE: &str = "attachment"; pub(crate) const ROW_SPILL_OWNER_RECEIPT_FILE: &str = ".row-spill-owner-v1"; pub(crate) const MAX_ROW_SPILL_OWNER_RECEIPT_BYTES: usize = 512; -pub(crate) fn row_spill_owner_receipt(name: &OsStr) -> Option> { +pub(crate) const ROW_SPILL_PREFIX: &str = ".rows-"; +const RECEIPT_PREFIX: &str = "tracedecay.graph-row-spill-v1:"; + +fn row_spill_owner_receipt(name: &OsStr) -> Option> { let name = name.to_str()?; - let receipt = format!("tracedecay.graph-row-spill-v1:{name}").into_bytes(); + let receipt = format!("{RECEIPT_PREFIX}{name}").into_bytes(); (receipt.len() <= MAX_ROW_SPILL_OWNER_RECEIPT_BYTES).then_some(receipt) } +fn staging_name(owner: u32, receipt: &[u8]) -> String { + let digest = canonical_framed_sha256( + b"tracedecay.graph-row-spill.pending.v1", + &[&owner.to_be_bytes(), receipt], + ); + format!("{ROW_SPILL_PREFIX}{owner}-pending-{digest}") +} + +/// A foreign publisher may have stopped immediately before or after rename. +/// Reclaim either name only when the receipt on the held child proves it. +pub(crate) enum AbandonedRowSpill<'a> { + Published(Vec), + Pending { owner: u32, name: &'a str }, +} + +impl<'a> AbandonedRowSpill<'a> { + pub(crate) fn from_name(name: &'a OsStr, current: u32) -> Option { + let text = name.to_str()?; + let (pid, suffix) = text.strip_prefix(ROW_SPILL_PREFIX)?.split_once('-')?; + let owner = pid.parse::().ok()?; + if current == owner || owner.to_string() != pid { + return None; + } + if let Some(digest) = suffix.strip_prefix("pending-") { + return is_lowercase_hex(digest, 64).then_some(Self::Pending { owner, name: text }); + } + let counter = suffix.parse::().ok()?; + if counter.to_string() != suffix { + return None; + } + row_spill_owner_receipt(name).map(Self::Published) + } + + pub(crate) fn owns(&self, receipt: &[u8]) -> bool { + match self { + Self::Published(expected) => receipt == expected, + Self::Pending { owner, name } => { + let Some(bound_name) = receipt + .strip_prefix(RECEIPT_PREFIX.as_bytes()) + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + else { + return false; + }; + Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) + && staging_name(*owner, receipt) == *name + } + } + } +} + /// Scratch directory owned by one spill; removed with its owner. struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { + let mut staging = Self::prepare(&path)?; + tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) + .map_err(|error| spill_io("directory install", error))?; + staging.disable_cleanup(true); + Ok(Self(path)) + } + + fn prepare(path: &Path) -> Result { let name = path .file_name() .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; @@ -87,8 +150,12 @@ impl SpillDirectory { .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; // Receipt failures must leave the requested name available to retry. // Keep the partial marker private until the complete spill can be installed. - let mut staging = - tempfile::tempdir_in(parent).map_err(|error| spill_io("directory create", error))?; + let temporary = staging_name(std::process::id(), &receipt); + let staging = tempfile::Builder::new() + .prefix(&temporary) + .rand_bytes(0) + .tempdir_in(parent) + .map_err(|error| spill_io("directory create", error))?; let marker = staging.path().join(ROW_SPILL_OWNER_RECEIPT_FILE); let mut file = std::fs::OpenOptions::new() .write(true) @@ -99,10 +166,7 @@ impl SpillDirectory { .and_then(|()| file.sync_all()) .map_err(|error| spill_io("owner marker persist", error))?; drop(file); - tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) - .map_err(|error| spill_io("directory install", error))?; - staging.disable_cleanup(true); - Ok(Self(path)) + Ok(staging) } fn path(&self) -> &Path { @@ -1123,6 +1187,56 @@ mod owner_receipt_tests { assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); } + #[test] + fn store_open_reclaims_receipted_staging_after_producer_exits_before_rename() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_PENDING_SPILL_ROOT"; + const PREPARED: i32 = 42; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let name = format!(".rows-{}-1", std::process::id()); + let _staged = + SpillDirectory::prepare(&std::path::PathBuf::from(root).join(name)).unwrap(); + std::process::exit(PREPARED); + } + + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, &root) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(PREPARED), "{child:?}"); + let interrupted = std::fs::read_dir(&root) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let (owner, _) = interrupted + .file_name() + .unwrap() + .to_str() + .unwrap() + .split_once("-pending-") + .unwrap(); + let mismatched = root.join(format!("{owner}-pending-{}", "0".repeat(64))); + std::fs::create_dir(&mismatched).unwrap(); + let receipt = std::fs::read(interrupted.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(); + std::fs::write(mismatched.join(ROW_SPILL_OWNER_RECEIPT_FILE), &receipt).unwrap(); + + let active = SpillDirectory::prepare(&root.join(".rows-active")).unwrap(); + crate::sealed_store::sweep_abandoned_row_spills(&database); + assert!(!interrupted.exists()); + assert!(active.path().is_dir()); + assert_eq!( + std::fs::read(mismatched.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(), + receipt + ); + } + #[test] fn occupied_spill_publication_preserves_destination_and_discards_staging() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ec70c1fab78..86330fcc7e4 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -58,9 +58,9 @@ use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nof use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ - GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, - physical_namespace_projection_map, recovered_digest_from_row_sum, row_spill_owner_receipt, - verify_sealed_copy_generation, + AbandonedRowSpill, GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, physical_namespace_projection_map, + recovered_digest_from_row_sum, verify_sealed_copy_generation, }; use crate::lease::GenerationLocator; use crate::location::PersistentGraphStoreState; @@ -767,9 +767,6 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { } } -/// Prefix of a batch producer's row spill under the sealed root. -const ROW_SPILL_PREFIX: &str = ".rows-"; - /// Removes every row spill another process left under the store's sealed /// root. Run at the eager open, whose exclusive store lock means no other /// process is producing rows for this store. Spills named for this process @@ -782,22 +779,8 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { }; let own_pid = std::process::id(); for entry in entries.flatten() { - if entry - .file_name() - .to_str() - .and_then(|name| name.strip_prefix(ROW_SPILL_PREFIX)) - .and_then(|suffix| suffix.split_once('-')) - .is_some_and(|(pid, sequence)| { - let (Ok(owner), Ok(counter)) = (pid.parse::(), sequence.parse::()) else { - return false; - }; - owner != own_pid && owner.to_string() == pid && counter.to_string() == sequence - }) - { - let name = entry.file_name(); - let Some(receipt) = row_spill_owner_receipt(&name) else { - continue; - }; + let name = entry.file_name(); + if let Some(proof) = AbandonedRowSpill::from_name(&name, own_pid) { let path = entry.path(); if let Err(error) = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( @@ -805,7 +788,7 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { &name, std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, - |bytes| bytes == receipt, + |bytes| proof.owns(bytes), ) { tracing::warn!( From ded88e87c54cde9c989ef3edd035242dcd1b8b6e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:28:29 -0800 Subject: [PATCH 020/111] fix(graph): distinguish spill owners across reused process IDs --- crates/tracedecay-graph-db/src/generation.rs | 2 +- .../src/generation/spill.rs | 98 +++++++++++++------ .../tracedecay-graph-db/src/sealed_store.rs | 8 +- .../sealed_store.rs | 47 +++++++-- .../tracedecay-private-fs/src/coarse_time.rs | 6 +- crates/tracedecay-private-fs/src/lib.rs | 11 +++ 6 files changed, 124 insertions(+), 48 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation.rs b/crates/tracedecay-graph-db/src/generation.rs index 2db4c376fb7..ba64685fd60 100644 --- a/crates/tracedecay-graph-db/src/generation.rs +++ b/crates/tracedecay-graph-db/src/generation.rs @@ -61,7 +61,7 @@ pub(crate) use replay::{ }; pub(crate) use spill::{ AbandonedRowSpill, ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, - ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, read_spilled_entity, + ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, read_spilled_entity, row_spill_owner, }; pub use spill::{ GRAPH_ROW_SPILL_RUN_BYTES, GraphGenerationRowSpill, GraphGenerationRows, diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 90949aef3f5..57a76eb1238 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -71,13 +71,20 @@ pub(crate) const MAX_ROW_SPILL_OWNER_RECEIPT_BYTES: usize = 512; pub(crate) const ROW_SPILL_PREFIX: &str = ".rows-"; const RECEIPT_PREFIX: &str = "tracedecay.graph-row-spill-v1:"; -fn row_spill_owner_receipt(name: &OsStr) -> Option> { +fn row_spill_owner_receipt(name: &OsStr, owner: u128) -> Option> { let name = name.to_str()?; - let receipt = format!("{RECEIPT_PREFIX}{name}").into_bytes(); + let receipt = format!("{RECEIPT_PREFIX}{owner}:{name}").into_bytes(); (receipt.len() <= MAX_ROW_SPILL_OWNER_RECEIPT_BYTES).then_some(receipt) } -fn staging_name(owner: u32, receipt: &[u8]) -> String { +/// The PID changes at fork. The process nonce changes on restart, even when +/// the kernel reuses that PID and the allocation counter starts over. +pub(crate) fn row_spill_owner() -> u128 { + (u128::from(std::process::id()) << 64) + | u128::from(tracedecay_private_fs::process_instance_nonce()) +} + +fn staging_name(owner: u128, receipt: &[u8]) -> String { let digest = canonical_framed_sha256( b"tracedecay.graph-row-spill.pending.v1", &[&owner.to_be_bytes(), receipt], @@ -89,15 +96,15 @@ fn staging_name(owner: u32, receipt: &[u8]) -> String { /// Reclaim either name only when the receipt on the held child proves it. pub(crate) enum AbandonedRowSpill<'a> { Published(Vec), - Pending { owner: u32, name: &'a str }, + Pending { owner: u128, name: &'a str }, } impl<'a> AbandonedRowSpill<'a> { - pub(crate) fn from_name(name: &'a OsStr, current: u32) -> Option { + pub(crate) fn from_name(name: &'a OsStr, current: u128) -> Option { let text = name.to_str()?; - let (pid, suffix) = text.strip_prefix(ROW_SPILL_PREFIX)?.split_once('-')?; - let owner = pid.parse::().ok()?; - if current == owner || owner.to_string() != pid { + let (instance, suffix) = text.strip_prefix(ROW_SPILL_PREFIX)?.split_once('-')?; + let owner = instance.parse::().ok()?; + if current == owner || owner.to_string() != instance { return None; } if let Some(digest) = suffix.strip_prefix("pending-") { @@ -107,7 +114,7 @@ impl<'a> AbandonedRowSpill<'a> { if counter.to_string() != suffix { return None; } - row_spill_owner_receipt(name).map(Self::Published) + row_spill_owner_receipt(name, owner).map(Self::Published) } pub(crate) fn owns(&self, receipt: &[u8]) -> bool { @@ -120,7 +127,11 @@ impl<'a> AbandonedRowSpill<'a> { else { return false; }; - Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) + let Some((declared_owner, bound_name)) = bound_name.split_once(':') else { + return false; + }; + declared_owner == owner.to_string() + && Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) && staging_name(*owner, receipt) == *name } } @@ -132,25 +143,25 @@ struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { - let mut staging = Self::prepare(&path)?; + let mut staging = Self::prepare(&path, row_spill_owner())?; tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) .map_err(|error| spill_io("directory install", error))?; staging.disable_cleanup(true); Ok(Self(path)) } - fn prepare(path: &Path) -> Result { + fn prepare(path: &Path, owner: u128) -> Result { let name = path .file_name() .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; - let receipt = row_spill_owner_receipt(name) + let receipt = row_spill_owner_receipt(name, owner) .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; let parent = path .parent() .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; // Receipt failures must leave the requested name available to retry. // Keep the partial marker private until the complete spill can be installed. - let temporary = staging_name(std::process::id(), &receipt); + let temporary = staging_name(owner, &receipt); let staging = tempfile::Builder::new() .prefix(&temporary) .rand_bytes(0) @@ -179,7 +190,7 @@ impl Drop for SpillDirectory { let Some((root, name)) = self.0.parent().zip(self.0.file_name()) else { return; }; - let Some(receipt) = row_spill_owner_receipt(name) else { + let Some(receipt) = row_spill_owner_receipt(name, row_spill_owner()) else { return; }; // The pathname may have been replaced since this spill was created. @@ -1153,7 +1164,7 @@ impl GraphGenerationManifestIdentity { #[cfg(test)] mod owner_receipt_tests { - use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory}; + use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory, row_spill_owner}; #[cfg(unix)] #[test] @@ -1192,9 +1203,12 @@ mod owner_receipt_tests { const CHILD_ROOT: &str = "TRACEDECAY_TEST_PENDING_SPILL_ROOT"; const PREPARED: i32 = 42; if let Some(root) = std::env::var_os(CHILD_ROOT) { - let name = format!(".rows-{}-1", std::process::id()); - let _staged = - SpillDirectory::prepare(&std::path::PathBuf::from(root).join(name)).unwrap(); + let name = format!(".rows-{}-1", row_spill_owner()); + let _staged = SpillDirectory::prepare( + &std::path::PathBuf::from(root).join(name), + row_spill_owner(), + ) + .unwrap(); std::process::exit(PREPARED); } @@ -1227,7 +1241,8 @@ mod owner_receipt_tests { let receipt = std::fs::read(interrupted.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(); std::fs::write(mismatched.join(ROW_SPILL_OWNER_RECEIPT_FILE), &receipt).unwrap(); - let active = SpillDirectory::prepare(&root.join(".rows-active")).unwrap(); + let active = + SpillDirectory::prepare(&root.join(".rows-active"), row_spill_owner()).unwrap(); crate::sealed_store::sweep_abandoned_row_spills(&database); assert!(!interrupted.exists()); assert!(active.path().is_dir()); @@ -1237,6 +1252,32 @@ mod owner_receipt_tests { ); } + #[test] + fn a_reused_pid_reclaims_the_prior_incarnation_and_preserves_live_spills() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let current = row_spill_owner(); + let previous = current ^ 1; + assert_eq!(current >> 64, previous >> 64, "same numeric process ID"); + let final_path = root.join(format!(".rows-{current}-0")); + let stale = SpillDirectory::prepare(&final_path, previous) + .unwrap() + .keep(); + let live = SpillDirectory::prepare(&final_path, current).unwrap(); + let peer_path = root.join(format!(".rows-{current}-1")); + let _peer = SpillDirectory::create(peer_path.clone()).unwrap(); + + crate::sealed_store::sweep_abandoned_row_spills(&database); + assert!(!stale.exists()); + assert!(live.path().is_dir()); + assert!(peer_path.is_dir()); + drop(live); + let _published = SpillDirectory::create(final_path.clone()).unwrap(); + assert!(final_path.is_dir()); + } + #[test] fn occupied_spill_publication_preserves_destination_and_discards_staging() { let root = tempfile::tempdir().unwrap(); @@ -1259,16 +1300,17 @@ mod owner_receipt_tests { let database = temp.path().join("graph.grafeo"); let root = temp.path().join("graph.sealed"); std::fs::create_dir(&root).unwrap(); - let other_pid = std::process::id().wrapping_add(1); - let owned = root.join(format!(".rows-{other_pid}-1")); - let completed = SpillDirectory::create(owned.clone()).unwrap(); + let other_owner = row_spill_owner() ^ 1; + let owned = root.join(format!(".rows-{other_owner}-1")); + let mut completed = SpillDirectory::prepare(&owned, other_owner).unwrap(); + tracedecay_private_fs::framed_log::rename_noreplace(completed.path(), &owned).unwrap(); + completed.disable_cleanup(true); std::fs::write(owned.join("entities-0.run"), b"owned").unwrap(); - std::mem::forget(completed); // Simulate a producer killed before Drop. - let unreceipted = root.join(format!(".rows-{other_pid}-2")); + let unreceipted = root.join(format!(".rows-{other_owner}-2")); std::fs::create_dir(&unreceipted).unwrap(); std::fs::write(unreceipted.join("foreign"), b"unowned").unwrap(); - let forged = root.join(format!(".rows-{other_pid}-3")); + let forged = root.join(format!(".rows-{other_owner}-3")); let interrupted = SpillDirectory::create(forged.clone()).unwrap(); std::fs::write(forged.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"wrong owner").unwrap(); std::fs::write(forged.join("foreign"), b"preserve").unwrap(); @@ -1285,7 +1327,7 @@ mod owner_receipt_tests { ); assert_eq!(std::fs::read(forged.join("foreign")).unwrap(), b"preserve"); - let active = root.join(format!(".rows-{}-4", std::process::id())); + let active = root.join(format!(".rows-{}-4", row_spill_owner())); let spill = SpillDirectory::create(active.clone()).unwrap(); drop(spill); assert!( @@ -1293,7 +1335,7 @@ mod owner_receipt_tests { "a current, proven spill cleans up on drop" ); - let substituted = root.join(format!(".rows-{}-5", std::process::id())); + let substituted = root.join(format!(".rows-{}-5", row_spill_owner())); let spill = SpillDirectory::create(substituted.clone()).unwrap(); std::fs::write(substituted.join("foreign"), b"still here").unwrap(); std::fs::write( diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 86330fcc7e4..e22b590885d 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -60,7 +60,7 @@ use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ AbandonedRowSpill, GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, physical_namespace_projection_map, - recovered_digest_from_row_sum, verify_sealed_copy_generation, + recovered_digest_from_row_sum, row_spill_owner, verify_sealed_copy_generation, }; use crate::lease::GenerationLocator; use crate::location::PersistentGraphStoreState; @@ -777,10 +777,10 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let Ok(entries) = std::fs::read_dir(&root) else { return; }; - let own_pid = std::process::id(); + let current_owner = row_spill_owner(); for entry in entries.flatten() { let name = entry.file_name(); - if let Some(proof) = AbandonedRowSpill::from_name(&name, own_pid) { + if let Some(proof) = AbandonedRowSpill::from_name(&name, current_owner) { let path = entry.path(); if let Err(error) = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( @@ -1469,7 +1469,7 @@ impl GraphDb { .map_err(|error| sealed_store_io_failure("row spill root create failed", error))?; Ok(root.join(format!( "{ROW_SPILL_PREFIX}{}-{}", - std::process::id(), + row_spill_owner(), NEXT_SPILL.fetch_add(1, AtomicOrdering::Relaxed) ))) } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 0f43b56e3a3..046861645a8 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -2451,16 +2451,44 @@ fn spilled_rows_refuse_conflicting_repeats_and_dangling_endpoints() { /// own live publisher's and stays. #[test] fn store_open_sweeps_row_spills_abandoned_by_another_process() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_ABANDONED_GRAPH_SPILL"; + const SPILLED: i32 = 42; + let identity = projection("code", "store-open-spill"); + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let root = PathBuf::from(root); + let registered = RegisteredGraph::new_mounted(&root).unwrap(); + let spill = registered + .registry + .generation_row_spill(registration(registered.binding.clone(), &root), identity) + .unwrap(); + std::fs::write(spill.attachment_path(), b"abandoned").unwrap(); + std::process::exit(SPILLED); + } + let temp = TempDir::new().unwrap(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, temp.path()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(SPILLED), "{child:?}"); let root = sealed_store_root(temp.path()); - std::fs::create_dir_all(&root).unwrap(); - let abandoned = root.join(".rows-4194305-0"); - let identity = projection("code", "store-open-spill"); - let spill = GraphGenerationRowSpill::create(abandoned.clone(), identity.clone()).unwrap(); - std::fs::write(abandoned.join("entities-0.run"), b"abandoned").unwrap(); - std::mem::forget(spill); // Simulate a producer that exited before Drop. - let own = root.join(format!(".rows-{}-999999", std::process::id())); - let _live_spill = GraphGenerationRowSpill::create(own.clone(), identity).unwrap(); + let abandoned = row_spills(temp.path()); + assert_eq!(abandoned.len(), 1); + let abandoned = root.join(&abandoned[0]); + let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); + assert!(!abandoned.exists()); + let live_spill = registered + .registry + .generation_row_spill( + registration(registered.binding.clone(), temp.path()), + identity, + ) + .unwrap(); + let attachment = live_spill.attachment_path(); + std::fs::write(&attachment, b"live peer attachment").unwrap(); + drop(registered); let foreign = ".rows-notes"; std::fs::create_dir(root.join(foreign)).unwrap(); @@ -2471,6 +2499,5 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::read(root.join(foreign).join("notes")).unwrap(), b"keep" ); - assert!(own.is_dir()); - assert!(!abandoned.exists()); + assert_eq!(std::fs::read(attachment).unwrap(), b"live peer attachment"); } diff --git a/crates/tracedecay-private-fs/src/coarse_time.rs b/crates/tracedecay-private-fs/src/coarse_time.rs index 8978f1b928a..8b8ccc1f0a3 100644 --- a/crates/tracedecay-private-fs/src/coarse_time.rs +++ b/crates/tracedecay-private-fs/src/coarse_time.rs @@ -6,13 +6,10 @@ //! timestamp quantum can share a stamp, so a change time is proof the bytes //! are stable only once the clock has left the quantum it was stamped in. -use std::collections::hash_map::RandomState; use std::fs::Metadata; -use std::hash::BuildHasher; #[cfg(unix)] use std::os::unix::fs::MetadataExt; use std::path::Path; -use std::sync::LazyLock; use std::sync::atomic::{AtomicU64, Ordering}; const NANOS_PER_SECOND: i128 = 1_000_000_000; @@ -120,10 +117,9 @@ impl ChangeStamp { fn unvouched() -> Self { // The random half keeps samples from different processes apart where // an identity is persisted as a digest. - static PROCESS: LazyLock = LazyLock::new(|| RandomState::new().hash_one(0_u8)); static SAMPLE: AtomicU64 = AtomicU64::new(0); let sample = SAMPLE.fetch_add(1, Ordering::Relaxed); - Self::Unvouched((u128::from(*PROCESS) << 64) | u128::from(sample)) + Self::Unvouched((u128::from(crate::process_instance_nonce()) << 64) | u128::from(sample)) } #[must_use] diff --git a/crates/tracedecay-private-fs/src/lib.rs b/crates/tracedecay-private-fs/src/lib.rs index 7e7298a08db..52c99ebe108 100644 --- a/crates/tracedecay-private-fs/src/lib.rs +++ b/crates/tracedecay-private-fs/src/lib.rs @@ -2,7 +2,10 @@ //! plus the Windows file-handle identity primitive that shares this crate's //! cold, dependency-light position. +use std::collections::hash_map::RandomState; +use std::hash::BuildHasher; use std::io; +use std::sync::LazyLock; pub mod capability_dir; mod coarse_time; @@ -15,6 +18,14 @@ mod rename_noreplace; pub use coarse_time::{ChangeClockReading, ChangeStamp, RewriteWitness, change_time_settled}; pub use inode_generation::inode_generation; +/// A process-local random witness for persisted identities that must distinguish +/// successive processes reusing a PID. Pair it with the PID for forked children. +#[must_use] +pub fn process_instance_nonce() -> u64 { + static NONCE: LazyLock = LazyLock::new(|| RandomState::new().hash_one(0_u8)); + *NONCE +} + pub use file_lease::FileLease; pub use lock_admission::{LockAdmissionError, lock_shared_until, lock_until}; #[cfg(windows)] From 59b5cd33c56ea30c730a12303b95d2261b15cb60 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:35:16 -0800 Subject: [PATCH 021/111] fix(graph): keep row-spill sweep bound to one sealed-root handle Refs #4616 Reuse the receipt-verified held-root collector for abandoned row spills. Filter candidate names before receipt IO so sealed generations are not probed as spills. Cover root path replacement during verified collection and preserve wrong-receipt foreign rows on a real store reopen. --- .../tracedecay-graph-db/src/sealed_store.rs | 52 +++++++------- .../sealed_store.rs | 10 +++ .../src/capability_dir.rs | 67 +++++++++++++++++++ 3 files changed, 105 insertions(+), 24 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e22b590885d..9c32619154e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -774,31 +774,35 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { /// whose spill removes itself when it drops. pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let root = sealed_store_root(database_path); - let Ok(entries) = std::fs::read_dir(&root) else { - return; - }; let current_owner = row_spill_owner(); - for entry in entries.flatten() { - let name = entry.file_name(); - if let Some(proof) = AbandonedRowSpill::from_name(&name, current_owner) { - let path = entry.path(); - if let Err(error) = - tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( - &root, - &name, - std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), - MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, - |bytes| proof.owns(bytes), - ) - { - tracing::warn!( - event = "graph_row_spill_sweep_failed", - path = %path.display(), - error = %error, - "unverified abandoned graph row spill preserved for inspection" - ); - } - } + // Enumerate, verify and unlink through one held sealed-root capability: + // a root rename between discovery and cleanup must not redirect deletion. + let result = + tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |name| AbandonedRowSpill::from_name(name, current_owner).is_some(), + |name, receipt| { + AbandonedRowSpill::from_name(name, current_owner) + .is_some_and(|proof| proof.owns(receipt)) + }, + ); + match result { + Ok(removed) if removed > 0 => tracing::info!( + event = "graph_row_spill_swept", + root = %root.display(), + removed, + "reclaimed receipt-verified abandoned graph row spills" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "graph_row_spill_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned graph row spills preserved for inspection" + ), } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 046861645a8..1fd3f0ada90 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -2490,6 +2490,12 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::write(&attachment, b"live peer attachment").unwrap(); drop(registered); + // A different producer's name is not authority to reclaim foreign bytes. + // Even a well-shaped abandoned-spill entry with a wrong receipt survives + // store opening, while the authentic abandoned spill above was reclaimed. + std::fs::create_dir(&abandoned).unwrap(); + std::fs::write(abandoned.join(".row-spill-owner-v1"), b"wrong owner").unwrap(); + std::fs::write(abandoned.join("foreign"), b"preserve").unwrap(); let foreign = ".rows-notes"; std::fs::create_dir(root.join(foreign)).unwrap(); std::fs::write(root.join(foreign).join("notes"), b"keep").unwrap(); @@ -2499,5 +2505,9 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::read(root.join(foreign).join("notes")).unwrap(), b"keep" ); + assert_eq!( + std::fs::read(abandoned.join("foreign")).unwrap(), + b"preserve" + ); assert_eq!(std::fs::read(attachment).unwrap(), b"live peer attachment"); } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index d04bea7d3fe..a504c587af0 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -324,6 +324,26 @@ pub fn remove_receipted_directories_nofollow( root_path: &Path, receipt_name: &OsStr, maximum: usize, + authorized: impl FnMut(&OsStr, &[u8]) -> bool, +) -> io::Result { + remove_matching_receipted_directories_nofollow( + root_path, + receipt_name, + maximum, + |_| true, + authorized, + ) +} + +/// Scan and reclaim selected children beneath one held root. Filter names +/// before receipt I/O so unrelated installed generations are not treated as +/// abandoned staging or spills. Receipt verification and removal remain +/// anchored to the same opened child, even if the root pathname changes. +pub fn remove_matching_receipted_directories_nofollow( + root_path: &Path, + receipt_name: &OsStr, + maximum: usize, + mut candidate: impl FnMut(&OsStr) -> bool, mut authorized: impl FnMut(&OsStr, &[u8]) -> bool, ) -> io::Result { // Receipt names are always a single sibling: a slash-bearing name could @@ -347,6 +367,9 @@ pub fn remove_receipted_directories_nofollow( continue; } let name = entry.file_name(); + if !candidate(&name) { + continue; + } // Do not trust the DirEntry's former pathname kind after a rename. let child = match root.open_dir_nofollow(&name) { Ok(child) => child, @@ -625,6 +648,50 @@ mod tests { ); } + /// Candidate filtering must not reopen a potentially substituted root: + /// only the receipted owned child goes away; unrelated and foreign trees survive. + #[cfg(unix)] + #[test] + fn filtered_receipt_sweep_holds_root_through_path_substitution() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("owned-displaced"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join(".rows-old")).unwrap(); + std::fs::create_dir_all(root.join("generation")).unwrap(); + std::fs::create_dir_all(foreign.join(".rows-old")).unwrap(); + std::fs::write(root.join(".rows-old/.owner"), b"verified").unwrap(); + std::fs::write(root.join("generation/live"), b"live").unwrap(); + std::fs::write(foreign.join(".rows-old/.owner"), b"verified").unwrap(); + std::fs::write(foreign.join(".rows-old/sentinel"), b"foreign").unwrap(); + + let removed = remove_matching_receipted_directories_nofollow( + &root, + OsStr::new(".owner"), + 512, + |name| name == OsStr::new(".rows-old"), + |name, receipt| { + assert_eq!(name, OsStr::new(".rows-old")); + assert_eq!(receipt, b"verified"); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + true + }, + ) + .unwrap(); + + assert_eq!(removed, 1); + assert!(!displaced.join(".rows-old").exists()); + assert_eq!( + std::fs::read(displaced.join("generation/live")).unwrap(), + b"live" + ); + assert_eq!( + std::fs::read(foreign.join(".rows-old/sentinel")).unwrap(), + b"foreign" + ); + } + /// A parent-path swap after receipt validation must not redirect cleanup: /// the already-open child is removed, and matching foreign bytes survive. #[cfg(unix)] From 552b5c7fc6e82ff9248f5d0743f6ee7866eb2eb7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:51:59 -0800 Subject: [PATCH 022/111] fix(storage): continue cleanup after a verified child unlink fails --- .../src/capability_dir.rs | 66 ++++++++++++++++--- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index a504c587af0..72b9eda88b3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -339,6 +339,7 @@ pub fn remove_receipted_directories_nofollow( /// before receipt I/O so unrelated installed generations are not treated as /// abandoned staging or spills. Receipt verification and removal remain /// anchored to the same opened child, even if the root pathname changes. +/// A failure on one entry is reported after its independent siblings are processed. pub fn remove_matching_receipted_directories_nofollow( root_path: &Path, receipt_name: &OsStr, @@ -360,37 +361,50 @@ pub fn remove_matching_receipted_directories_nofollow( )); } let root = open_root_nofollow(root_path)?; - let mut removed = 0usize; - for entry in root.read_dir(".")? { + let outcomes = root.read_dir(".")?.map(|entry| -> io::Result { let entry = entry?; if !entry.file_type()?.is_dir() { - continue; + return Ok(false); } let name = entry.file_name(); if !candidate(&name) { - continue; + return Ok(false); } // Do not trust the DirEntry's former pathname kind after a rename. let child = match root.open_dir_nofollow(&name) { Ok(child) => child, Err(error) => { tracing::warn!(%error, entry = ?name, "sealed child could not be verified"); - continue; + return Ok(false); } }; let receipt = match read_bounded_child_receipt(&child, receipt_name, maximum) { Ok(receipt) => receipt, Err(error) => { tracing::warn!(%error, entry = ?name, "unverified sealed receipt preserved"); - continue; + return Ok(false); } }; - if authorized(&name, &receipt) { - remove_open_dir_all_nofollow(child)?; - removed = removed.saturating_add(1); + if !authorized(&name, &receipt) { + return Ok(false); + } + remove_open_dir_all_nofollow(child).inspect_err(|error| { + tracing::warn!(%error, entry = ?name, "verified child directory could not be removed"); + })?; + Ok(true) + }); + let mut removed = 0; + let mut first_error = None; + for outcome in outcomes { + match outcome { + Ok(true) => removed += 1, + Ok(false) => {} + Err(error) => { + first_error.get_or_insert(error); + } } } - Ok(removed) + first_error.map_or(Ok(removed), Err) } #[cfg(unix)] @@ -692,6 +706,38 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn a_failed_verified_unlink_still_reclaims_independent_siblings() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + for name in ["a", "b"] { + std::fs::create_dir_all(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("owner"), name).unwrap(); + } + let mut verified = Vec::new(); + let error = remove_matching_receipted_directories_nofollow( + &root, + OsStr::new("owner"), + 64, + |_| true, + |name, receipt| { + assert_eq!(name.as_encoded_bytes(), receipt); + if verified.is_empty() { + // A concurrent cleanup wins after this child was verified. + // Its now-unlinked capability cannot be removed a second time. + std::fs::remove_dir_all(root.join(name)).unwrap(); + } + verified.push(name.to_owned()); + true + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::NotFound, "{error}"); + assert_eq!(verified.len(), 2); + assert!(!root.join(&verified[1]).exists()); + } + /// A parent-path swap after receipt validation must not redirect cleanup: /// the already-open child is removed, and matching foreign bytes survive. #[cfg(unix)] From 6eadebb7e39913b0a1b1355ef3f9bd41510184c5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:14:06 -0800 Subject: [PATCH 023/111] fix(graph): verify logical owner before old-format cleanup --- .../tracedecay-graph-db/src/sealed_store.rs | 83 ++++++++++++++----- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 9c32619154e..fca6cb2ccc9 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -667,7 +667,8 @@ pub(crate) fn discard_superseded_sealed_generations( }; recognized_form && receipt.version == SEALED_STORE_RECEIPT_VERSION - && receipt.physical_namespace == format!("generation:{name}") + && sealed_receipt_matches_generation_entry(&receipt, name) + && !receipt.recovered_digest.is_empty() && receipt.graph_format > 0 && receipt.graph_format < GraphFormatVersion::current().get() }, @@ -682,6 +683,30 @@ pub(crate) fn discard_superseded_sealed_generations( } } +/// A physical generation entry is deletable only if the receipt's logical +/// locator derives that exact entry name. A copied receipt with a fabricated +/// physical namespace is not evidence of ownership. +fn sealed_receipt_matches_generation_entry( + receipt: &SealedStoreReceiptV1, + generation_hex: &str, +) -> bool { + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace.clone()), + GraphProjectionId::new(receipt.projection.clone()), + crate::GraphGenerationId::new(receipt.generation.clone()), + ) else { + return false; + }; + let locator = GenerationLocator::new( + GraphProjectionIdentity::new(namespace, projection), + generation, + ); + locator.physical_namespace().is_ok_and(|physical| { + physical.as_str() == receipt.physical_namespace + && receipt.physical_namespace == format!("generation:{generation_hex}") + }) +} + fn sealed_staging_name(name: &str) -> bool { name.strip_prefix(".staging-").is_some_and(|suffix| { suffix.len() == 64 @@ -718,23 +743,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Ok(receipt) = serde_json::from_slice::(bytes) else { return false; }; - // Bind the receipt's logical identity back to the canonical - // physical namespace. A copied receipt with a rewritten physical - // name is not proof of ownership of another staged generation. - let (Ok(namespace), Ok(projection), Ok(generation)) = ( - GraphNamespace::new(receipt.namespace.clone()), - GraphProjectionId::new(receipt.projection.clone()), - crate::GraphGenerationId::new(receipt.generation.clone()), - ) else { - return false; - }; - let locator = GenerationLocator::new( - GraphProjectionIdentity::new(namespace, projection), - generation, - ); - let Ok(physical_namespace) = locator.physical_namespace() else { - return false; - }; + // Reuse the same identity proof as obsolete-format retirement. let recognized_form = match receipt.form.as_str() { SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), @@ -744,8 +753,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { && receipt.version == SEALED_STORE_RECEIPT_VERSION && receipt.graph_format > 0 && receipt.graph_format <= GraphFormatVersion::current().get() - && receipt.physical_namespace == format!("generation:{generation_hex}") - && receipt.physical_namespace == physical_namespace.as_str() + && sealed_receipt_matches_generation_entry(&receipt, generation_hex) && !receipt.recovered_digest.is_empty() }, ); @@ -3035,7 +3043,29 @@ mod receipt_budget_tests { std::fs::create_dir_all(&root).unwrap(); let current = GraphFormatVersion::current().get(); assert!(current > 0); - let old = generation(&root, 'a', &receipt(current - 1, &"a".repeat(64), "old")); + let old_locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new("generation:obsolete").unwrap(), + ); + let old_physical = old_locator.physical_namespace().unwrap(); + let old_name = old_physical.as_str().strip_prefix("generation:").unwrap(); + let old = root.join(old_name); + std::fs::create_dir(&old).unwrap(); + std::fs::write( + old.join(SEALED_STORE_RECEIPT_FILE), + receipt(current - 1, old_name, "generation:obsolete"), + ) + .unwrap(); + // A syntactically plausible old receipt with a different logical + // generation cannot authorize deletion of this occupied directory. + let forged = generation( + &root, + 'a', + &receipt(current - 1, &"a".repeat(64), "generation:foreign"), + ); let live = generation(&root, 'b', &receipt(current, &"b".repeat(64), "live")); let invalid = generation(&root, 'c', b"not a receipt"); // Zero is not a valid GraphFormatVersion, even when the JSON is well formed. @@ -3070,6 +3100,7 @@ mod receipt_budget_tests { }; for dir in [ &old, + &forged, &live, &invalid, &zero_format, @@ -3082,7 +3113,15 @@ mod receipt_budget_tests { discard_superseded_sealed_generations(&database).unwrap(); assert!(!old.exists(), "verified old-format receipt is reclaimable"); - for dir in [&live, &invalid, &zero_format, &oversized, &missing, &future] { + for dir in [ + &forged, + &live, + &invalid, + &zero_format, + &oversized, + &missing, + &future, + ] { assert_eq!(std::fs::read(dir.join("sentinel")).unwrap(), b"owned"); } assert_eq!(std::fs::read(&unrelated).unwrap(), b"foreign"); From 53457e4bead9be2fb4ad80811e8cb3897451adc2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:31:13 -0800 Subject: [PATCH 024/111] fix(graph): bind pending spill receipts to canonical names Refs #4616 --- .../src/generation/spill.rs | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 57a76eb1238..c62d9873ebb 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -130,8 +130,20 @@ impl<'a> AbandonedRowSpill<'a> { let Some((declared_owner, bound_name)) = bound_name.split_once(':') else { return false; }; + // A pending receipt must bind a real published spill name. + // Hashing arbitrary receipt text is not proof that the + // directory was ever produced by a graph row-spill writer. + let Some((published_owner, counter)) = bound_name + .strip_prefix(ROW_SPILL_PREFIX) + .and_then(|suffix| suffix.split_once('-')) + else { + return false; + }; declared_owner == owner.to_string() - && Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) + && published_owner == declared_owner + && counter + .parse::() + .is_ok_and(|parsed| parsed.to_string() == counter) && staging_name(*owner, receipt) == *name } } @@ -1164,7 +1176,10 @@ impl GraphGenerationManifestIdentity { #[cfg(test)] mod owner_receipt_tests { - use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory, row_spill_owner}; + use super::{ + RECEIPT_PREFIX, ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, SpillDirectory, + row_spill_owner, staging_name, + }; #[cfg(unix)] #[test] @@ -1252,6 +1267,58 @@ mod owner_receipt_tests { ); } + #[test] + fn store_open_preserves_pending_spills_with_noncanonical_published_receipts() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let previous_owner = row_spill_owner() ^ 1; + let canonical = root.join(format!("{ROW_SPILL_PREFIX}{previous_owner}-7")); + let owned = SpillDirectory::prepare(&canonical, previous_owner) + .unwrap() + .keep(); + + let forged_bound_names = [ + "unrelated".to_owned(), + format!("{ROW_SPILL_PREFIX}{previous_owner}-00"), + format!("{ROW_SPILL_PREFIX}{previous_owner}-not-a-counter"), + format!("{ROW_SPILL_PREFIX}{previous_owner}-18446744073709551616"), + format!("{ROW_SPILL_PREFIX}{}-7", previous_owner ^ 2), + ]; + let mut foreign = Vec::new(); + for (index, bound_name) in forged_bound_names.iter().enumerate() { + // Even a digest-consistent staging name is not ownership proof + // when its claimed published row-spill name is not canonical. + let receipt = format!("{RECEIPT_PREFIX}{previous_owner}:{bound_name}").into_bytes(); + let directory = root.join(staging_name(previous_owner, &receipt)); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join(ROW_SPILL_OWNER_RECEIPT_FILE), &receipt).unwrap(); + let sentinel = format!("foreign-{index}").into_bytes(); + std::fs::write(directory.join("sentinel"), &sentinel).unwrap(); + foreign.push((directory, receipt, sentinel)); + } + let unrelated = root.join("live-generation"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("sentinel"), b"live").unwrap(); + + crate::sealed_store::sweep_abandoned_row_spills(&database); + + assert!( + !owned.exists(), + "a genuine interrupted pending spill must still be reclaimed" + ); + for (directory, receipt, sentinel) in foreign { + assert_eq!( + std::fs::read(directory.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(), + receipt, + "a forged receipt cannot authorize pending-spill cleanup" + ); + assert_eq!(std::fs::read(directory.join("sentinel")).unwrap(), sentinel); + } + assert_eq!(std::fs::read(unrelated.join("sentinel")).unwrap(), b"live"); + } + #[test] fn a_reused_pid_reclaims_the_prior_incarnation_and_preserves_live_spills() { let temp = tempfile::tempdir().unwrap(); From e60c8940c5ea215548f60275529dfdae1ad7295f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:59:08 -0800 Subject: [PATCH 025/111] test(storage): use genuine spill and seal ownership identities --- crates/tracedecay-graph-db/src/generation/spill.rs | 3 ++- .../tests/graph_db_suite/runtime_contract.rs | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index c62d9873ebb..8add23cd36d 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -1329,7 +1329,8 @@ mod owner_receipt_tests { let previous = current ^ 1; assert_eq!(current >> 64, previous >> 64, "same numeric process ID"); let final_path = root.join(format!(".rows-{current}-0")); - let stale = SpillDirectory::prepare(&final_path, previous) + let previous_path = root.join(format!(".rows-{previous}-0")); + let stale = SpillDirectory::prepare(&previous_path, previous) .unwrap() .keep(); let live = SpillDirectory::prepare(&final_path, current).unwrap(); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index c69dbf3222a..a34e05465fd 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -5,6 +5,7 @@ use std::time::Duration; use grafeo_common::types::Value; use tempfile::TempDir; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_graph_db::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphDbLeaseV1, GraphDbOwner, GraphEntity, GraphEntityId, GraphFormatVersion, GraphIdempotencyKey, GraphLabel, GraphMutation, @@ -1063,7 +1064,8 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() ) .unwrap(); raw.close().unwrap(); - let sealed_generation_name = "0".repeat(64); + // The on-disk name is the digest of the receipt's JSON identity tuple. + let sealed_generation_name = sha256_hex(br#"["project","code","g0"]"#); let sealed_generation = temp .path() .join("graph.sealed") @@ -1106,7 +1108,8 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() // A generation this build sealed before the staging container was // replaced, as a publication racing the rebuild leaves it. - let current_generation = temp.path().join("graph.sealed").join("1".repeat(64)); + let current_name = sha256_hex(br#"["project","code","g1"]"#); + let current_generation = temp.path().join("graph.sealed").join(¤t_name); std::fs::create_dir_all(¤t_generation).unwrap(); std::fs::write( current_generation.join("sealed.json"), @@ -1116,7 +1119,7 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() "namespace": "project", "projection": "code", "generation": "g1", - "physical_namespace": format!("generation:{}", "1".repeat(64)), + "physical_namespace": format!("generation:{current_name}"), "recovered_digest": "sha256:1", "entities": 1, "relations": 0, From 6287274abb5f6ac46160d5c628e3962de6506654 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:32:46 -0800 Subject: [PATCH 026/111] fix(graph): reclaim crashed sealed stages with owner receipts Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 342 +++++++++++++++--- 1 file changed, 290 insertions(+), 52 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index fca6cb2ccc9..a95a32ffe2a 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,6 +44,7 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; +use std::io::Write; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -228,6 +229,9 @@ fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result> { pub(crate) const SEALED_STORE_DATABASE_FILE: &str = "generation.grafeo"; const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; +/// Small, generation-bound deletion authority persisted before any staged rows. +/// Unlike sealed.json, this is never evidence that a container is complete. +const SEALED_STAGING_OWNER_FILE: &str = ".sealed-staging-owner-v1"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` /// written before that proof is not release authority. const SEALED_STORE_CHECKED_FILE: &str = "sealed.checked"; @@ -716,6 +720,103 @@ fn sealed_staging_name(name: &str) -> bool { }) } +fn initial_sealed_staging_owner( + identity: &GraphGenerationManifestIdentity, + expected: &GraphRecoveredGenerationDigestV1, + physical_namespace: &str, +) -> SealedStoreReceiptV1 { + // This receipt authorizes only abandoned-staging cleanup, never serving. + // The completed container receives a separate sealed.json after writing. + SealedStoreReceiptV1 { + version: SEALED_STORE_RECEIPT_VERSION, + form: SEALED_STORE_FORM_COMPACT.to_owned(), + namespace: identity.projection.namespace.as_str().to_owned(), + projection: identity.projection.projection.as_str().to_owned(), + generation: identity.generation.as_str().to_owned(), + physical_namespace: physical_namespace.to_owned(), + recovered_digest: expected.as_str().to_owned(), + entities: 0, + relations: 0, + graph_format: GraphFormatVersion::current().get(), + row_sum: None, + base: None, + source_generation: None, + watermark: None, + } +} + +fn verified_staging_receipt( + name: &std::ffi::OsStr, + bytes: &[u8], +) -> Option { + let name = name.to_str()?; + if !sealed_staging_name(name) { + return None; + } + let generation_hex = name.strip_prefix(".staging-")?; + let receipt = serde_json::from_slice::(bytes).ok()?; + let recognized_form = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + (recognized_form + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.graph_format > 0 + && receipt.graph_format <= GraphFormatVersion::current().get() + && sealed_receipt_matches_generation_entry(&receipt, generation_hex) + && !receipt.recovered_digest.is_empty()) + .then_some(receipt) +} + +/// A failed builder must never recursively delete a substituted directory +/// merely because its pathname is the deterministic staging name. The early +/// owner marker covers unfinished builds; sealed.json covers the short window +/// after final receipt publication and before installation. +fn discard_failed_sealed_staging( + staging: &Path, + locator: &GenerationLocator, + expected_digest: &str, +) { + let Some((root, name)) = staging.parent().zip(staging.file_name()) else { + return; + }; + for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { + let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + std::ffi::OsStr::new(marker), + MAX_SEALED_RECEIPT_BYTES, + |bytes| { + verified_staging_receipt(name, bytes).is_some_and(|receipt| { + receipt.namespace == locator.projection.namespace.as_str() + && receipt.projection == locator.projection.projection.as_str() + && receipt.generation == locator.generation.as_str() + && receipt.recovered_digest == expected_digest + }) + }, + ); + match result { + Ok(_) => return, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + tracing::warn!( + event = "sealed_staging_cleanup_deferred", + path = %staging.display(), + %error, + "unverified staging preserved instead of deleting an unknown directory" + ); + return; + } + } + } + tracing::warn!( + event = "sealed_staging_cleanup_deferred", + path = %staging.display(), + "staging has no verified owner receipt; preserving unknown bytes" + ); +} + /// Reclaims interrupted seals only when the bounded receipt from the same /// opened staging directory proves its exact generation identity. The staging /// filename alone is not deletion authority: a missing, malformed, oversized, @@ -726,52 +827,33 @@ fn sealed_staging_name(name: &str) -> bool { /// so a pathname replacement cannot redirect deletion into a different tree. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let reclaimed = tracedecay_private_fs::capability_dir::remove_receipted_directories_nofollow( - &root, - std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), - MAX_SEALED_RECEIPT_BYTES, - |name, bytes| { - let Some(name) = name.to_str() else { - return false; - }; - if !sealed_staging_name(name) { - return false; - } - let Some(generation_hex) = name.strip_prefix(".staging-") else { - return false; - }; - let Ok(receipt) = serde_json::from_slice::(bytes) else { - return false; - }; - // Reuse the same identity proof as obsolete-format retirement. - let recognized_form = match receipt.form.as_str() { - SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), - SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), - _ => false, - }; - recognized_form - && receipt.version == SEALED_STORE_RECEIPT_VERSION - && receipt.graph_format > 0 - && receipt.graph_format <= GraphFormatVersion::current().get() - && sealed_receipt_matches_generation_entry(&receipt, generation_hex) - && !receipt.recovered_digest.is_empty() - }, - ); - match reclaimed { - Ok(removed) if removed > 0 => tracing::info!( - event = "sealed_staging_swept", - root = %root.display(), - removed, - "removed receipt-verified staging from interrupted seals" - ), - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - root = %root.display(), - %error, - "unverified abandoned staging stays for inspection" - ), + // A crash during row copying has no completed sealed.json yet, but it + // does have the initial owner marker. A crash just before installation + // can be reclaimed using either marker. Never sweep an unreceipted name. + for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { + let reclaimed = tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(marker), + MAX_SEALED_RECEIPT_BYTES, + |name| name.to_str().is_some_and(sealed_staging_name), + |name, bytes| verified_staging_receipt(name, bytes).is_some(), + ); + match reclaimed { + Ok(removed) if removed > 0 => tracing::info!( + event = "sealed_staging_swept", + root = %root.display(), + removed, + "removed receipt-verified staging from interrupted seals" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned staging stays for inspection" + ), + } } } @@ -1963,19 +2045,46 @@ fn build_or_open_sealed_store( // create_dir is exclusive, so an occupied name fails closed. std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; + let owner = initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()); + let owner_bytes = serde_json::to_vec(&owner) + .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; + // Exclusive creation refuses a pre-existing marker. Flush before writing + // rows so a killed producer leaves verifiable deletion authority. + let marker_path = staging.join(SEALED_STAGING_OWNER_FILE); + let mut marker = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker_path) + .map_err(|error| sealed_store_io_failure("staging owner create failed", error))?; + marker + .write_all(&owner_bytes) + .and_then(|()| marker.sync_all()) + .map_err(|error| sealed_store_io_failure("staging owner persist failed", error))?; + drop(marker); let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check) - .inspect_err(|_| remove_sealed_directory(&staging))?; + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; let encoded = serde_json::to_vec_pretty(&receipt) - .map_err(|error| GraphDbError::unavailable(format!("sealed receipt encode: {error}")))?; - std::fs::write(staging.join(SEALED_STORE_RECEIPT_FILE), encoded) - .inspect_err(|_| remove_sealed_directory(&staging)) - .map_err(|error| sealed_store_io_failure("receipt write failed", error))?; + .map_err(|error| GraphDbError::unavailable(format!("sealed receipt encode: {error}"))) + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; + let mut receipt_file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(staging.join(SEALED_STORE_RECEIPT_FILE)) + .map_err(|error| sealed_store_io_failure("receipt create failed", error)) + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; + let receipt_write = receipt_file + .write_all(&encoded) + .and_then(|()| receipt_file.sync_all()); + drop(receipt_file); + receipt_write + .map_err(|error| sealed_store_io_failure("receipt persist failed", error)) + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; let artifact_name = directory .file_name() .ok_or_else(|| GraphDbError::unavailable("sealed artifact name missing"))?; let install = install_child_directory_noreplace_nofollow(&root, staging_name, artifact_name); if let Err(error) = install { - remove_sealed_directory(&staging); + discard_failed_sealed_staging(&staging, &locator, expected.as_str()); // A concurrent seal of the same generation may have installed the // directory first; adopting it below keeps this path idempotent. if !directory.exists() { @@ -3782,6 +3891,135 @@ mod build_tests { assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); } + /// Kill the real builder during row preparation, before sealed.json is + /// published. Its early receipt must let a later store-open sweep reclaim + /// the crashed stage without touching foreign staging data. + #[test] + fn crashed_seal_before_final_receipt_is_reclaimed_and_can_retry() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_CRASHED_SEAL_ROOT"; + const CRASHED: i32 = 43; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let database_path = std::path::PathBuf::from(root).join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let exit_during_copy = || -> Result<(), GraphDbError> { + std::process::exit(CRASHED); + }; + let _ = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &exit_during_copy, + ); + panic!("the row stream did not invoke its cancellation callback"); + } + + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, temp.path()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(CRASHED), "{child:?}"); + assert!(staging.join(SEALED_STAGING_OWNER_FILE).is_file()); + assert!(!staging.join(SEALED_STORE_RECEIPT_FILE).exists()); + + let foreign = root.join(format!(".staging-{}", "0".repeat(64))); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"foreign").unwrap(); + sweep_abandoned_sealed_staging(&database_path); + assert!( + !staging.exists(), + "the crashed but proven stage must be reclaimed" + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"foreign" + ); + + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + } + + /// The failed-builder cleanup must not delete a foreign replacement of a + /// staging pathname after the owner marker was written. + #[cfg(unix)] + #[test] + fn interrupted_seal_preserves_a_substituted_staging_directory() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let moved = root.join("original-stage"); + let swapped = std::cell::Cell::new(false); + let interrupt = || { + if !swapped.replace(true) { + std::fs::rename(&staging, &moved).unwrap(); + std::fs::create_dir(&staging).unwrap(); + std::fs::write(staging.join("sentinel"), b"foreign replacement").unwrap(); + } + Err(GraphDbError::Cancelled) + }; + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &interrupt, + ); + assert!( + matches!(&result, Err(GraphDbError::Cancelled)), + "{result:?}" + ); + assert_eq!( + std::fs::read(staging.join("sentinel")).unwrap(), + b"foreign replacement", + "cleanup must not erase the substituted directory" + ); + assert!(moved.join(SEALED_STAGING_OWNER_FILE).is_file()); + assert!(!directory.exists()); + + std::fs::remove_dir_all(&staging).unwrap(); + std::fs::remove_dir_all(&moved).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + assert_eq!(store.recovered_digest(), expected.as_str()); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves From 76ac5bd5145f7847c76d40276a46ee160a9d5b7c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:34:34 -0800 Subject: [PATCH 027/111] test(graph): import sealed staging recovery helpers Refs #4616 --- crates/tracedecay-graph-db/src/sealed_store.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a95a32ffe2a..8f908e65603 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3413,9 +3413,10 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, - build_or_open_sealed_store, build_sealed_container, sealed_artifact_database_options, - sealed_copy_proof, sealed_generation_directory, sealed_store_root, + SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, + SealedRowSource, build_or_open_sealed_store, build_sealed_container, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ From 95a66960f8cd451385b1e25f049b76d2992451fb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:03:47 -0800 Subject: [PATCH 028/111] fix(graph): preserve live sealed staging across remounts Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 178 +++++++++++------- 1 file changed, 115 insertions(+), 63 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 8f908e65603..3b221f75fe5 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -277,6 +277,14 @@ struct SealedStoreReceiptV1 { watermark: Option, } +/// An incomplete sealed container belongs to one producer incarnation. +/// A remount in that process must not reclaim its in-flight row stream. +#[derive(Debug, Deserialize, Serialize)] +struct SealedStagingOwnerV1 { + producer: String, + receipt: SealedStoreReceiptV1, +} + impl SealedStoreReceiptV1 { fn binds( &self, @@ -748,13 +756,18 @@ fn initial_sealed_staging_owner( fn verified_staging_receipt( name: &std::ffi::OsStr, bytes: &[u8], -) -> Option { +) -> Option { let name = name.to_str()?; if !sealed_staging_name(name) { return None; } let generation_hex = name.strip_prefix(".staging-")?; - let receipt = serde_json::from_slice::(bytes).ok()?; + let owner = serde_json::from_slice::(bytes).ok()?; + let instance = owner.producer.parse::().ok()?; + if instance.to_string() != owner.producer { + return None; + } + let receipt = &owner.receipt; let recognized_form = match receipt.form.as_str() { SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), @@ -764,15 +777,15 @@ fn verified_staging_receipt( && receipt.version == SEALED_STORE_RECEIPT_VERSION && receipt.graph_format > 0 && receipt.graph_format <= GraphFormatVersion::current().get() - && sealed_receipt_matches_generation_entry(&receipt, generation_hex) + && sealed_receipt_matches_generation_entry(receipt, generation_hex) && !receipt.recovered_digest.is_empty()) - .then_some(receipt) + .then_some(owner) } /// A failed builder must never recursively delete a substituted directory -/// merely because its pathname is the deterministic staging name. The early -/// owner marker covers unfinished builds; sealed.json covers the short window -/// after final receipt publication and before installation. +/// merely because its pathname is the deterministic staging name. Only the +/// producer-bound early marker authorizes cleanup, including after sealed.json +/// publication and before installation. fn discard_failed_sealed_staging( staging: &Path, locator: &GenerationLocator, @@ -781,40 +794,30 @@ fn discard_failed_sealed_staging( let Some((root, name)) = staging.parent().zip(staging.file_name()) else { return; }; - for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { - let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( - root, - name, - std::ffi::OsStr::new(marker), - MAX_SEALED_RECEIPT_BYTES, - |bytes| { - verified_staging_receipt(name, bytes).is_some_and(|receipt| { - receipt.namespace == locator.projection.namespace.as_str() - && receipt.projection == locator.projection.projection.as_str() - && receipt.generation == locator.generation.as_str() - && receipt.recovered_digest == expected_digest - }) - }, + let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + std::ffi::OsStr::new(SEALED_STAGING_OWNER_FILE), + MAX_SEALED_RECEIPT_BYTES, + |bytes| { + verified_staging_receipt(name, bytes).is_some_and(|owner| { + let receipt = &owner.receipt; + owner.producer == row_spill_owner().to_string() + && receipt.namespace == locator.projection.namespace.as_str() + && receipt.projection == locator.projection.projection.as_str() + && receipt.generation == locator.generation.as_str() + && receipt.recovered_digest == expected_digest + }) + }, + ); + if !matches!(&result, Ok(true)) { + tracing::warn!( + event = "sealed_staging_cleanup_deferred", + path = %staging.display(), + ?result, + "unverified staging preserved instead of deleting an unknown directory" ); - match result { - Ok(_) => return, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - tracing::warn!( - event = "sealed_staging_cleanup_deferred", - path = %staging.display(), - %error, - "unverified staging preserved instead of deleting an unknown directory" - ); - return; - } - } } - tracing::warn!( - event = "sealed_staging_cleanup_deferred", - path = %staging.display(), - "staging has no verified owner receipt; preserving unknown bytes" - ); } /// Reclaims interrupted seals only when the bounded receipt from the same @@ -827,33 +830,35 @@ fn discard_failed_sealed_staging( /// so a pathname replacement cannot redirect deletion into a different tree. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - // A crash during row copying has no completed sealed.json yet, but it - // does have the initial owner marker. A crash just before installation - // can be reclaimed using either marker. Never sweep an unreceipted name. - for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { - let reclaimed = tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( + // Every new sealed build writes this marker before rows and retains it + // through installation. The final sealed.json is not a separate cleanup + // authority: it cannot identify whether a concurrent build is still live. + let reclaimed = + tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( &root, - std::ffi::OsStr::new(marker), + std::ffi::OsStr::new(SEALED_STAGING_OWNER_FILE), MAX_SEALED_RECEIPT_BYTES, |name| name.to_str().is_some_and(sealed_staging_name), - |name, bytes| verified_staging_receipt(name, bytes).is_some(), + |name, bytes| { + verified_staging_receipt(name, bytes) + .is_some_and(|owner| owner.producer != row_spill_owner().to_string()) + }, ); - match reclaimed { - Ok(removed) if removed > 0 => tracing::info!( - event = "sealed_staging_swept", - root = %root.display(), - removed, - "removed receipt-verified staging from interrupted seals" - ), - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - root = %root.display(), - %error, - "unverified abandoned staging stays for inspection" - ), - } + match reclaimed { + Ok(removed) if removed > 0 => tracing::info!( + event = "sealed_staging_swept", + root = %root.display(), + removed, + "removed receipt-verified staging from exited producers" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned staging stays for inspection" + ), } } @@ -2045,7 +2050,10 @@ fn build_or_open_sealed_store( // create_dir is exclusive, so an occupied name fails closed. std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; - let owner = initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()); + let owner = SealedStagingOwnerV1 { + producer: row_spill_owner().to_string(), + receipt: initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()), + }; let owner_bytes = serde_json::to_vec(&owner) .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; // Exclusive creation refuses a pre-existing marker. Flush before writing @@ -3892,6 +3900,50 @@ mod build_tests { assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); } + /// A store remount invokes the real startup sweep while a producer is + /// still streaming rows. The sweep must preserve its owner-bound staging + /// directory, or the builder can no longer install the verified artifact. + #[test] + fn remount_preserves_live_sealed_staging_and_seal_completes() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(48, 24); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = + sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let remounted = std::cell::Cell::new(false); + let check = || { + if !remounted.replace(true) { + assert!(staging.join(SEALED_STAGING_OWNER_FILE).is_file()); + let remount = open_source(&database_path); + drop(remount); + assert!( + staging.join(SEALED_STAGING_OWNER_FILE).is_file(), + "remount must not reclaim a live producer's staged container" + ); + } + Ok(()) + }; + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &check, + ) + .expect("live seal must finish after a store remount"); + assert!(remounted.get(), "remount must run inside the row-copy callback"); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(!staging.exists()); + } + /// Kill the real builder during row preparation, before sealed.json is /// published. Its early receipt must let a later store-open sweep reclaim /// the crashed stage without touching foreign staging data. From 8d97999f422480e7dc6baefecfc71c3544290fbe Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:07:12 -0800 Subject: [PATCH 029/111] fix(graph): reject unowned receipts in sealed census Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 114 ++++++++++++++++-- 1 file changed, 102 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 3b221f75fe5..84327defcf7 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -591,13 +591,16 @@ pub fn census_sealed_store( continue; } let path = entry.path(); - let bytes = directory_bytes(&path, &mut counted)?; let name = entry.file_name(); if name.to_str().is_some_and(sealed_staging_name) { + let bytes = directory_bytes(&path, &mut counted)?; census.abandoned_staging_count += 1; census.abandoned_staging_bytes += bytes; continue; } + // Receipt existence is not sufficient: a copied receipt can name a + // verified head while residing in an unrelated physical directory. + // Classify unproven entries before walking or charging their contents. let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) .ok() .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); @@ -605,12 +608,32 @@ pub fn census_sealed_store( census.unrecognized_count += 1; continue; }; + let Some(name) = name.to_str() else { + census.unrecognized_count += 1; + continue; + }; + let form_valid = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + if !form_valid + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.graph_format == 0 + || receipt.graph_format > GraphFormatVersion::current().get() + || receipt.recovered_digest.is_empty() + || !sealed_receipt_matches_generation_entry(&receipt, name) + { + census.unrecognized_count += 1; + continue; + } + let bytes = directory_bytes(&path, &mut counted)?; if head_generations.contains(&receipt.generation) { census.head_count += 1; - census.head_bytes += bytes; + census.head_bytes = census.head_bytes.saturating_add(bytes); } else { census.superseded_count += 1; - census.superseded_bytes += bytes; + census.superseded_bytes = census.superseded_bytes.saturating_add(bytes); } } Ok(census) @@ -3370,17 +3393,26 @@ mod receipt_budget_tests { let root = sealed_store_root(&database); std::fs::create_dir_all(&root).unwrap(); const FILE_BYTES: u64 = 220 * 1024 * 1024; + let mut oldest = None; for index in 0..70 { - let name = format!("{index:064x}"); - let dir = root.join(&name); + let generation = format!("generation:g{index}"); + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new(generation.clone()).unwrap(), + ); + let physical = locator.physical_namespace().unwrap(); + let name = physical.as_str().strip_prefix("generation:").unwrap(); + let dir = root.join(name); + if index == 0 { + oldest = Some(dir.clone()); + } std::fs::create_dir_all(&dir).unwrap(); std::fs::write( dir.join(SEALED_STORE_RECEIPT_FILE), - receipt( - GraphFormatVersion::current().get(), - &name, - &format!("g{index}"), - ), + receipt(GraphFormatVersion::current().get(), name, &generation), ) .unwrap(); std::fs::File::create(dir.join(SEALED_STORE_DATABASE_FILE)) @@ -3396,10 +3428,14 @@ mod receipt_budget_tests { .unwrap() .set_len(20 * 1024 * 1024 * 1024) .unwrap(); - let oldest = root.join("0".repeat(64)); + let oldest = oldest.unwrap(); std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); - let census = census_sealed_store(&database, &BTreeSet::from(["g69".to_owned()])).unwrap(); + let census = census_sealed_store( + &database, + &BTreeSet::from(["generation:g69".to_owned()]), + ) + .unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 69); assert_eq!(census.unrecognized_count, 0); @@ -3410,6 +3446,60 @@ mod receipt_budget_tests { "foreign link targets must not contribute to sealed generation bytes" ); } + + #[test] + fn census_does_not_classify_copied_or_foreign_receipts_as_live_heads() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir_all(&root).unwrap(); + let generation = "generation:head"; + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new(generation).unwrap(), + ); + let physical = locator.physical_namespace().unwrap(); + let owned_name = physical.as_str().strip_prefix("generation:").unwrap(); + let owned = root.join(owned_name); + std::fs::create_dir(&owned).unwrap(); + let owned_receipt = receipt( + GraphFormatVersion::current().get(), + owned_name, + generation, + ); + std::fs::write(owned.join(SEALED_STORE_RECEIPT_FILE), &owned_receipt).unwrap(); + std::fs::write(owned.join(SEALED_STORE_DATABASE_FILE), b"owned payload").unwrap(); + + // The foreign directory carries the *exact* valid head receipt, but + // its physical entry is not the one derived from the logical identity. + let foreign_name = if owned_name == "f".repeat(64) { + "e".repeat(64) + } else { + "f".repeat(64) + }; + let foreign = root.join(foreign_name); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join(SEALED_STORE_RECEIPT_FILE), &owned_receipt).unwrap(); + std::fs::File::create(foreign.join(SEALED_STORE_DATABASE_FILE)) + .unwrap() + .set_len(5 * 1024 * 1024 * 1024) + .unwrap(); + + let census = census_sealed_store( + &database, + &BTreeSet::from([generation.to_owned()]), + ) + .unwrap(); + assert_eq!(census.head_count, 1); + assert_eq!(census.superseded_count, 0); + assert_eq!(census.unrecognized_count, 1); + assert_eq!(census.head_bytes, owned_receipt.len() as u64 + 13); + assert!(foreign.join(SEALED_STORE_DATABASE_FILE).exists()); + } + } #[cfg(test)] From c87aa2fecbbbe6d10cbfb31f8dc250c9a6317d41 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:08:29 -0800 Subject: [PATCH 030/111] test(graph): keep sparse census regression Unix-only Refs #4616 --- crates/tracedecay-graph-db/src/sealed_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 84327defcf7..d8de2bbacfe 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3447,6 +3447,7 @@ mod receipt_budget_tests { ); } + #[cfg(unix)] #[test] fn census_does_not_classify_copied_or_foreign_receipts_as_live_heads() { let temp = tempfile::tempdir().unwrap(); @@ -3499,7 +3500,6 @@ mod receipt_budget_tests { assert_eq!(census.head_bytes, owned_receipt.len() as u64 + 13); assert!(foreign.join(SEALED_STORE_DATABASE_FILE).exists()); } - } #[cfg(test)] From 54b3b0a1aded24fcd5b404780d70455114ef33a7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:16:06 -0800 Subject: [PATCH 031/111] fix(graph): exclude unproven staging from sealed census Refs #4616 Only a receipt-verified staging directory from an exited producer may count as abandoned. Preserve unknown, copied, oversized and current-producer staging without traversing its contents. Cover positive and negative census behavior with native Rust regressions. --- .../tracedecay-graph-db/src/sealed_store.rs | 104 ++++++++++++++++-- .../sealed_store.rs | 15 ++- 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index d8de2bbacfe..8127870b838 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -556,11 +556,12 @@ pub struct SealedStoreCensusV1 { /// retirement has not run since the last publication. pub superseded_count: u64, pub superseded_bytes: u64, - /// `.staging-*` directories left by seals that never installed. + /// Receipt-verified `.staging-*` directories belonging to earlier producer + /// incarnations; current or unproven staging cannot be called abandoned. pub abandoned_staging_count: u64, pub abandoned_staging_bytes: u64, - /// Entries that are neither a readable sealed receipt nor staging: an - /// unrecognized layout is reported, never silently classed as dead. + /// Unrecognized or unproven entries (including live staging) are reported + /// without walking their contents or claiming that they can be reclaimed. pub unrecognized_count: u64, } @@ -593,9 +594,25 @@ pub fn census_sealed_store( let path = entry.path(); let name = entry.file_name(); if name.to_str().is_some_and(sealed_staging_name) { + // A matching filename is not proof that the directory is + // abandoned: remounts can overlap a seal in this process, and + // foreign or interrupted pre-receipt entries may occupy its name. + // Do not walk unverified trees (which may be arbitrarily large). + let verified = tracedecay_private_fs::framed_log::read_bounded( + &path.join(SEALED_STAGING_OWNER_FILE), + MAX_SEALED_RECEIPT_BYTES, + ) + .ok() + .flatten() + .and_then(|bytes| verified_staging_receipt(&name, &bytes)) + .is_some_and(|owner| owner.producer != row_spill_owner().to_string()); + if !verified { + census.unrecognized_count += 1; + continue; + } let bytes = directory_bytes(&path, &mut counted)?; census.abandoned_staging_count += 1; - census.abandoned_staging_bytes += bytes; + census.abandoned_staging_bytes = census.abandoned_staging_bytes.saturating_add(bytes); continue; } // Receipt existence is not sufficient: a copied receipt can name a @@ -3511,10 +3528,11 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, - SealedRowSource, build_or_open_sealed_store, build_sealed_container, - sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, - sealed_store_root, sweep_abandoned_sealed_staging, + MAX_SEALED_RECEIPT_BYTES, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, + SEALED_STORE_RECEIPT_FILE, SealedRowSource, SealedStagingOwnerV1, + build_or_open_sealed_store, build_sealed_container, census_sealed_store, + initial_sealed_staging_owner, sealed_artifact_database_options, sealed_copy_proof, + sealed_generation_directory, sealed_store_root, sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ @@ -3582,6 +3600,76 @@ mod build_tests { .unwrap() } + #[cfg(unix)] + #[test] + fn sealed_census_only_attributes_verified_abandoned_staging() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("source.grafeo"); + let root = sealed_store_root(&database); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let physical = identity.physical_namespace().unwrap(); + let name = physical.as_str().strip_prefix("generation:").unwrap(); + let staging = root.join(format!(".staging-{name}")); + std::fs::create_dir_all(&staging).unwrap(); + + let exited = SealedStagingOwnerV1 { + producer: (crate::generation::row_spill_owner() ^ 1).to_string(), + receipt: initial_sealed_staging_owner(&identity, &expected, physical.as_str()), + }; + let bytes = serde_json::to_vec(&exited).unwrap(); + let owner_path = staging.join(SEALED_STAGING_OWNER_FILE); + std::fs::write(&owner_path, &bytes).unwrap(); + std::fs::write(staging.join("payload"), vec![7; 4096]).unwrap(); + + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 1); + assert_eq!(census.abandoned_staging_bytes, bytes.len() as u64 + 4096); + assert_eq!(census.unrecognized_count, 0); + + // This same completed marker may belong to a seal currently running + // in this process. A remount must not report its bytes as garbage. + let live = SealedStagingOwnerV1 { + producer: crate::generation::row_spill_owner().to_string(), + receipt: initial_sealed_staging_owner(&identity, &expected, physical.as_str()), + }; + std::fs::write(&owner_path, serde_json::to_vec(&live).unwrap()).unwrap(); + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.abandoned_staging_bytes, 0); + assert_eq!(census.unrecognized_count, 1); + + // A copied authentic receipt in a different physical staging name + // does not authorize either reclamation or an unbounded size walk. + let foreign_name = if name == "f".repeat(64) { + "e".repeat(64) + } else { + "f".repeat(64) + }; + let foreign = root.join(format!(".staging-{foreign_name}")); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join(SEALED_STAGING_OWNER_FILE), &bytes).unwrap(); + std::fs::File::create(foreign.join("sparse-foreign-payload")) + .unwrap() + .set_len(5 * 1024 * 1024 * 1024) + .unwrap(); + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.abandoned_staging_bytes, 0); + assert_eq!(census.unrecognized_count, 2); + assert!(foreign.join("sparse-foreign-payload").is_file()); + + // Even an owned directory whose marker exceeds the fixed bound must + // not cause its contents to be traversed or counted as abandoned. + let oversized = vec![b'x'; MAX_SEALED_RECEIPT_BYTES + 1]; + std::fs::write(&owner_path, &oversized).unwrap(); + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.unrecognized_count, 2); + assert_eq!(std::fs::read(&owner_path).unwrap(), oversized); + } + fn open_source(database_path: &std::path::Path) -> crate::GraphDbLeaseV1 { let owner = GraphDbOwner::open(GraphDbOpenOptions { location: GraphDbLocation::Persistent(database_path.to_path_buf()), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 1fd3f0ada90..e0371137939 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1389,12 +1389,11 @@ fn reopening_the_store_reclaims_only_receipted_staging() { assert_snapshot_reads(&snapshot, &identity, "durable"); } -/// The sealed census splits the sealed root into what serves and what does -/// not: the head's artifact, a superseded generation's artifact, and staging -/// an interrupted seal left behind are each counted where they belong, and a -/// directory without a readable receipt is reported rather than classed. +/// The sealed census splits the sealed root into verified head and +/// superseded artifacts. Unreceipted staging and unrelated directories remain +/// unrecognized rather than falsely counted as reclaimable garbage. #[test] -fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { +fn sealed_census_separates_heads_and_preserves_unproven_staging() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let mut authority = RelationalAuthority::default(); @@ -1450,9 +1449,9 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { census.superseded_count, 1, "g1 is sealed but no longer the head" ); - assert_eq!(census.abandoned_staging_count, 1); - assert_eq!(census.abandoned_staging_bytes, 4096); - assert_eq!(census.unrecognized_count, 1); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.abandoned_staging_bytes, 0); + assert_eq!(census.unrecognized_count, 2); assert!(census.head_bytes > 0 && census.superseded_bytes > 0); // With no heads journaled every sealed artifact reads as superseded, and From f192a01ab3efb448275c2d10fbf25212b6101901 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:54:29 -0800 Subject: [PATCH 032/111] style(graph): format sealed census and recovery regressions --- .../tracedecay-graph-db/src/sealed_store.rs | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 8127870b838..6f6ecc4af64 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -793,10 +793,7 @@ fn initial_sealed_staging_owner( } } -fn verified_staging_receipt( - name: &std::ffi::OsStr, - bytes: &[u8], -) -> Option { +fn verified_staging_receipt(name: &std::ffi::OsStr, bytes: &[u8]) -> Option { let name = name.to_str()?; if !sealed_staging_name(name) { return None; @@ -3448,11 +3445,8 @@ mod receipt_budget_tests { let oldest = oldest.unwrap(); std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); - let census = census_sealed_store( - &database, - &BTreeSet::from(["generation:g69".to_owned()]), - ) - .unwrap(); + let census = + census_sealed_store(&database, &BTreeSet::from(["generation:g69".to_owned()])).unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 69); assert_eq!(census.unrecognized_count, 0); @@ -3483,11 +3477,7 @@ mod receipt_budget_tests { let owned_name = physical.as_str().strip_prefix("generation:").unwrap(); let owned = root.join(owned_name); std::fs::create_dir(&owned).unwrap(); - let owned_receipt = receipt( - GraphFormatVersion::current().get(), - owned_name, - generation, - ); + let owned_receipt = receipt(GraphFormatVersion::current().get(), owned_name, generation); std::fs::write(owned.join(SEALED_STORE_RECEIPT_FILE), &owned_receipt).unwrap(); std::fs::write(owned.join(SEALED_STORE_DATABASE_FILE), b"owned payload").unwrap(); @@ -3506,11 +3496,8 @@ mod receipt_budget_tests { .set_len(5 * 1024 * 1024 * 1024) .unwrap(); - let census = census_sealed_store( - &database, - &BTreeSet::from([generation.to_owned()]), - ) - .unwrap(); + let census = + census_sealed_store(&database, &BTreeSet::from([generation.to_owned()])).unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 0); assert_eq!(census.unrecognized_count, 1); @@ -4089,8 +4076,7 @@ mod build_tests { let identity = manifest.identity(); let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); let root = sealed_store_root(&database_path); - let directory = - sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); let staging = root.join(format!( ".staging-{}", directory.file_name().unwrap().to_str().unwrap() @@ -4116,7 +4102,10 @@ mod build_tests { &check, ) .expect("live seal must finish after a store remount"); - assert!(remounted.get(), "remount must run inside the row-copy callback"); + assert!( + remounted.get(), + "remount must run inside the row-copy callback" + ); assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); assert_eq!(store.recovered_digest(), expected.as_str()); assert!(!staging.exists()); @@ -4176,10 +4165,7 @@ mod build_tests { !staging.exists(), "the crashed but proven stage must be reclaimed" ); - assert_eq!( - std::fs::read(foreign.join("sentinel")).unwrap(), - b"foreign" - ); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"foreign"); let (store, _) = build_or_open_sealed_store( SealedRowSource::Manifest(&manifest), From e006c620df35ce6c4048f5f16ff28034467ec7af Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:56:58 -0800 Subject: [PATCH 033/111] test(graph): retire five sealed replay generations --- .../sealed_store.rs | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index e0371137939..94e656c34db 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1603,6 +1603,116 @@ fn retirement_deletes_the_superseded_sealed_artifact() { ); } +/// A real sequence of five published sealed replay generations must retire +/// all four superseded artifacts independently, not merely the oldest one. +/// Each retirement goes through the same production registry API used by +/// code-generation retention, and the newest reader stays usable throughout. +#[test] +fn five_sealed_replay_generations_retire_to_the_live_head() { + let temp = TempDir::new().unwrap(); + let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); + let mut authority = RelationalAuthority::default(); + let identity = projection("sealed-store:five-generation-retire", "code"); + let mut prior_head = None; + let mut superseded = Vec::new(); + let mut serving = None; + + for (ordinal, digit) in (1..=5).zip(['1', '2', '3', '4', '5']) { + let generation = format!("five-retire-g{ordinal}"); + let manifest = rich_manifest( + identity.clone(), + &generation, + &format!("marker-{ordinal}"), + ); + let record = stage_sealed_manifest( + &mut authority, + ®istered.binding, + &manifest, + &format!("publish:{generation}"), + prior_head.take(), + digit, + ); + let commit = publish_sealed( + ®istered, + temp.path(), + &mut authority, + &record, + &manifest, + ); + prior_head = Some(commit.head.clone()); + assert!( + receipt_for_generation(temp.path(), &generation).is_some(), + "each published generation must have its own sealed artifact" + ); + if ordinal == 5 { + serving = Some(commit); + } else { + superseded.push(( + generation, + manifest.identity(), + CodeGenerationId::new(format!( + "code-generation.{}", + manifest.generation.as_str() + )) + .unwrap(), + SealedGraphStateDigest::try_from(format!( + "sha256:{}", + digit.to_string().repeat(64) + )) + .unwrap(), + )); + drop(commit); + } + } + + assert_eq!(superseded.len(), 4); + assert_eq!( + sealed_receipts(temp.path()).len(), + 5, + "five distinct generations must exist before retirement" + ); + let serving = serving.unwrap(); + assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); + + let database = registered + .registry + .resolve(registration(registered.binding.clone(), temp.path())) + .unwrap(); + let (control, probe) = control_and_probe(); + let context = GraphPublicationOperationContextV1::new(&control, &probe).unwrap(); + for (generation, manifest_identity, code_generation, sealed_digest) in superseded { + assert!(matches!( + registered + .registry + .retire_one_code_generation_replay( + registration(registered.binding.clone(), temp.path()), + &mut authority, + &context, + &code_generation, + &sealed_digest, + ) + .unwrap(), + GraphReplayCollectionOutcome::Retired(_) + )); + assert!( + receipt_for_generation(temp.path(), &generation).is_none(), + "retired generation {generation} must be absent from disk" + ); + assert!( + !database.is_generation_sealed_only(&manifest_identity).unwrap(), + "retired generation {generation} must leave sealed-only state" + ); + assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); + } + assert_eq!( + sealed_receipts(temp.path()).len(), + 1, + "only the live sealed generation should remain" + ); + assert!(receipt_for_generation(temp.path(), "five-retire-g5").is_some()); + assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); +} + /// A live direct-sealed reader (recovered through /// `recover_verified_sealed_snapshot`, which bypasses the staging database's /// verified-generation state) must gate retirement of its generation exactly From 79c53bb732139d98aadd3a1c9188794faac34f04 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:57:44 -0800 Subject: [PATCH 034/111] fix(graph): anchor sealed proof marker writes to held directories Refs #4616. Publish sealed.checked through an exclusively staged sibling on a held no-follow generation capability. Refuse linked generation directories and verify foreign marker bytes survive a directory swap. Partial safety repair; automatic retention acceptance remains outstanding. --- .../tracedecay-graph-db/src/sealed_store.rs | 44 +++++++-- .../src/capability_dir.rs | 96 ++++++++++++++++++- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 6f6ecc4af64..1e50ccf204c 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2789,11 +2789,10 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result } // Never truncate an existing symlink target: the marker is a rebuildable // cache, so stage a regular file and publish it by atomic replacement. - tracedecay_private_fs::framed_log::atomic_write( - &directory.join(SEALED_STORE_CHECKED_FILE), - "sealed-check", + tracedecay_private_fs::capability_dir::atomic_replace_child_file_nofollow( + directory, + std::ffi::OsStr::new(SEALED_STORE_CHECKED_FILE), expected_digest.as_bytes(), - tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, ) .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) } @@ -3518,8 +3517,9 @@ mod build_tests { MAX_SEALED_RECEIPT_BYTES, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, SealedStagingOwnerV1, build_or_open_sealed_store, build_sealed_container, census_sealed_store, - initial_sealed_staging_owner, sealed_artifact_database_options, sealed_copy_proof, - sealed_generation_directory, sealed_store_root, sweep_abandoned_sealed_staging, + initial_sealed_staging_owner, persist_sealed_store_check, sealed_artifact_database_options, + sealed_copy_proof, sealed_generation_directory, sealed_store_root, + sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ @@ -3530,6 +3530,38 @@ mod build_tests { GraphRelationId, GraphRelationKind, GraphWatermark, NeverCancelled, SourceGeneration, }; + #[cfg(unix)] + #[test] + fn sealed_check_persistence_refuses_a_substituted_generation_directory() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned-generation"); + let foreign = temp.path().join("foreign-generation"); + std::fs::create_dir(&owned).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sealed.checked"), b"foreign marker").unwrap(); + std::fs::remove_dir(&owned).unwrap(); + symlink(&foreign, &owned).unwrap(); + + assert!(persist_sealed_store_check(&owned, "accepted-digest").is_err()); + assert_eq!( + std::fs::read(foreign.join("sealed.checked")).unwrap(), + b"foreign marker", + ); + std::fs::remove_file(&owned).unwrap(); + std::fs::create_dir(&owned).unwrap(); + persist_sealed_store_check(&owned, "accepted-digest").unwrap(); + assert_eq!( + std::fs::read(owned.join("sealed.checked")).unwrap(), + b"accepted-digest", + ); + assert_eq!( + std::fs::read(foreign.join("sealed.checked")).unwrap(), + b"foreign marker", + ); + } + fn entity_identity(index: usize) -> GraphEntityId { GraphEntityId::new(format!("symbol:{index:05}")).unwrap() } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 72b9eda88b3..1da1852bec3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -7,8 +7,9 @@ #[cfg(unix)] use std::ffi::CString; -use std::ffi::OsStr; -use std::io; +use std::ffi::{OsStr, OsString}; +use std::io::{self, Write}; +use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(unix)] use std::os::fd::{AsRawFd, FromRawFd}; #[cfg(unix)] @@ -227,6 +228,61 @@ fn single_component(name: &OsStr) -> io::Result<()> { Ok(()) } +/// Publish a rebuildable marker through one held directory. Replacing the +/// pathname after acquisition cannot redirect its bytes into a foreign tree. +pub fn atomic_replace_child_file_nofollow( + directory_path: &Path, + filename: &OsStr, + contents: &[u8], +) -> io::Result<()> { + single_component(filename)?; + let directory = open_root_nofollow(directory_path)?; + atomic_replace_file_in_dir(&directory, filename, contents) +} + +fn atomic_replace_file_in_dir( + directory: &Dir, + filename: &OsStr, + contents: &[u8], +) -> io::Result<()> { + static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); + let (temporary, mut file) = (0..8) + .find_map(|_| { + let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); + let temporary = OsString::from(format!( + ".sealed-proof-{}-{:016x}-{sequence:016x}.tmp", + std::process::id(), + crate::process_instance_nonce(), + )); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + match directory.open_with(&temporary, &options) { + Ok(file) => Some(Ok((temporary, file))), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => None, + Err(error) => Some(Err(error)), + } + }) + .transpose()? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::AlreadyExists, + "could not exclusively stage a sealed proof marker", + ) + })?; + let persisted = file.write_all(contents).and_then(|()| file.sync_all()); + drop(file); + let result = persisted + .and_then(|()| replace_file(directory, &temporary, filename)) + .and_then(|()| match sync_directory(directory) { + Err(error) if error.kind() == io::ErrorKind::InvalidInput => Ok(()), + result => result, + }); + if result.is_err() { + let _ = directory.remove_file(&temporary); + } + result +} + fn open_root_nofollow(root_path: &Path) -> io::Result { let parent_path = root_path .parent() @@ -474,6 +530,42 @@ mod replacement_tests { use super::*; + #[cfg(unix)] + #[test] + fn marker_publication_does_not_follow_a_substituted_directory() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned"); + let moved = temp.path().join("moved"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&owned).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sealed.checked"), b"foreign bytes").unwrap(); + + let held = open_root_nofollow(&owned).unwrap(); + std::fs::rename(&owned, &moved).unwrap(); + symlink(&foreign, &owned).unwrap(); + + assert!(atomic_replace_child_file_nofollow( + &owned, + OsStr::new("sealed.checked"), + b"new proof", + ) + .is_err()); + atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof") + .unwrap(); + assert_eq!( + std::fs::read(foreign.join("sealed.checked")).unwrap(), + b"foreign bytes", + ); + assert_eq!( + std::fs::read(moved.join("sealed.checked")).unwrap(), + b"owned proof", + ); + } + + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From 43b1ac43bc650615634815d819a3710c947ff140 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:59:08 -0800 Subject: [PATCH 035/111] style(fs): normalize sealed proof regression formatting Refs #4616. Keep the exact-head Bazel rustfmt gate unchanged. --- crates/tracedecay-private-fs/src/capability_dir.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 1da1852bec3..b4712db2ed7 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -553,8 +553,7 @@ mod replacement_tests { b"new proof", ) .is_err()); - atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof") - .unwrap(); + atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof").unwrap(); assert_eq!( std::fs::read(foreign.join("sealed.checked")).unwrap(), b"foreign bytes", @@ -565,7 +564,6 @@ mod replacement_tests { ); } - #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From fa9ae40206db0ea3b78a30ffc88c786cdf95d761 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:30:16 -0800 Subject: [PATCH 036/111] fix(graph): preserve foreign sealed proof marker entries Refs #4616 Refuse nonregular proof destinations through the held directory before and after staging. Verify the symlink entry and foreign target survive in native graph and private-filesystem regressions. --- .../tracedecay-graph-db/src/sealed_store.rs | 7 ++ .../src/capability_dir.rs | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 1e50ccf204c..1cfe5642820 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3185,6 +3185,13 @@ mod receipt_budget_tests { "publishing a proof marker must refuse a foreign symlink" ); assert_eq!(std::fs::read(&foreign).unwrap(), b"foreign sentinel"); + assert!( + std::fs::symlink_metadata(&marker) + .unwrap() + .file_type() + .is_symlink(), + "the foreign marker entry itself must survive" + ); } } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index b4712db2ed7..ae599dc4eb3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -245,6 +245,22 @@ fn atomic_replace_file_in_dir( filename: &OsStr, contents: &[u8], ) -> io::Result<()> { + // An accelerator may replace a regular cache file, but cannot take + // ownership of an unrelated symlink, directory, FIFO or device entry. + // Check relative to the held parent, never through an ambient pathname. + fn regular_or_missing(directory: &Dir, filename: &OsStr) -> io::Result<()> { + match directory.symlink_metadata(filename) { + Ok(metadata) if metadata.file_type().is_file() => Ok(()), + Ok(_) => Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sealed proof destination is not a regular file", + )), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } + } + + regular_or_missing(directory, filename)?; static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); let (temporary, mut file) = (0..8) .find_map(|_| { @@ -272,6 +288,8 @@ fn atomic_replace_file_in_dir( let persisted = file.write_all(contents).and_then(|()| file.sync_all()); drop(file); let result = persisted + // Recheck before publishing in case the entry changed during staging. + .and_then(|()| regular_or_missing(directory, filename)) .and_then(|()| replace_file(directory, &temporary, filename)) .and_then(|()| match sync_directory(directory) { Err(error) if error.kind() == io::ErrorKind::InvalidInput => Ok(()), @@ -564,6 +582,54 @@ mod replacement_tests { ); } + #[cfg(unix)] + #[test] + fn proof_publication_preserves_nonregular_marker_entries() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("sealed.checked"); + let foreign = temp.path().join("foreign"); + std::fs::write(&foreign, b"foreign").unwrap(); + symlink(&foreign, &marker).unwrap(); + + assert_eq!( + atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied + ); + assert!( + std::fs::symlink_metadata(&marker) + .unwrap() + .file_type() + .is_symlink(), + "the link itself, not merely its target, must survive" + ); + assert_eq!(std::fs::read(&foreign).unwrap(), b"foreign"); + + std::fs::remove_file(&marker).unwrap(); + std::fs::create_dir(&marker).unwrap(); + assert_eq!( + atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied + ); + assert!(marker.is_dir()); + + std::fs::remove_dir(&marker).unwrap(); + atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") + .unwrap(); + assert_eq!(std::fs::read(&marker).unwrap(), b"proof"); + assert!( + std::fs::read_dir(temp.path()) + .unwrap() + .all(|entry| !entry.unwrap().file_name().to_string_lossy().starts_with(".sealed-proof-")), + "rejected publications must not leave staging files" + ); + } + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From 280928f7b02bd82858bab44741491bdd6284a9e1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:32:20 -0800 Subject: [PATCH 037/111] style(fs): format sealed marker preservation regression Refs #4616 --- .../tracedecay-private-fs/src/capability_dir.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index ae599dc4eb3..76b66c298e0 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -622,12 +622,16 @@ mod replacement_tests { atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") .unwrap(); assert_eq!(std::fs::read(&marker).unwrap(), b"proof"); - assert!( - std::fs::read_dir(temp.path()) - .unwrap() - .all(|entry| !entry.unwrap().file_name().to_string_lossy().starts_with(".sealed-proof-")), - "rejected publications must not leave staging files" - ); + for entry in std::fs::read_dir(temp.path()).unwrap() { + assert!( + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".sealed-proof-"), + "rejected publications must not leave staging files" + ); + } } #[test] From 261682f1643ac634db26276a5fa21e070446edde Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:40:54 -0800 Subject: [PATCH 038/111] style(graph): fix Bazel rustfmt failures in sealed regressions Refs #4616 --- .../sealed_store.rs | 25 ++++++------------- .../src/capability_dir.rs | 12 ++++----- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 94e656c34db..61cbf81297e 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1619,11 +1619,7 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { for (ordinal, digit) in (1..=5).zip(['1', '2', '3', '4', '5']) { let generation = format!("five-retire-g{ordinal}"); - let manifest = rich_manifest( - identity.clone(), - &generation, - &format!("marker-{ordinal}"), - ); + let manifest = rich_manifest(identity.clone(), &generation, &format!("marker-{ordinal}")); let record = stage_sealed_manifest( &mut authority, ®istered.binding, @@ -1632,13 +1628,7 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { prior_head.take(), digit, ); - let commit = publish_sealed( - ®istered, - temp.path(), - &mut authority, - &record, - &manifest, - ); + let commit = publish_sealed(®istered, temp.path(), &mut authority, &record, &manifest); prior_head = Some(commit.head.clone()); assert!( receipt_for_generation(temp.path(), &generation).is_some(), @@ -1650,11 +1640,8 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { superseded.push(( generation, manifest.identity(), - CodeGenerationId::new(format!( - "code-generation.{}", - manifest.generation.as_str() - )) - .unwrap(), + CodeGenerationId::new(format!("code-generation.{}", manifest.generation.as_str())) + .unwrap(), SealedGraphStateDigest::try_from(format!( "sha256:{}", digit.to_string().repeat(64) @@ -1699,7 +1686,9 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { "retired generation {generation} must be absent from disk" ); assert!( - !database.is_generation_sealed_only(&manifest_identity).unwrap(), + !database + .is_generation_sealed_only(&manifest_identity) + .unwrap(), "retired generation {generation} must leave sealed-only state" ); assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 76b66c298e0..1dbb76aaefe 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -9,12 +9,12 @@ use std::ffi::CString; use std::ffi::{OsStr, OsString}; use std::io::{self, Write}; -use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(unix)] use std::os::fd::{AsRawFd, FromRawFd}; #[cfg(unix)] use std::os::unix::ffi::OsStrExt; use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; use cap_fs_ext::DirExt; #[cfg(not(windows))] @@ -565,12 +565,10 @@ mod replacement_tests { std::fs::rename(&owned, &moved).unwrap(); symlink(&foreign, &owned).unwrap(); - assert!(atomic_replace_child_file_nofollow( - &owned, - OsStr::new("sealed.checked"), - b"new proof", - ) - .is_err()); + assert!( + atomic_replace_child_file_nofollow(&owned, OsStr::new("sealed.checked"), b"new proof",) + .is_err() + ); atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof").unwrap(); assert_eq!( std::fs::read(foreign.join("sealed.checked")).unwrap(), From 424ef247942680d46a532f72cc56fe7bd3ac1040 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:45:23 -0800 Subject: [PATCH 039/111] fix(graph): bind sealed census size walks to held directories Refs #4616 Inspect receipts and charge bytes beneath the same held no-follow root and child capabilities, not pathnames that can be swapped. Preserve bounded reads, hardlink accounting and unrecognized entries. Add a native root-substitution regression. --- .../tracedecay-graph-db/src/sealed_store.rs | 178 +++++++----------- .../src/capability_dir.rs | 122 ++++++++++++ 2 files changed, 194 insertions(+), 106 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 1cfe5642820..ba4cf6be8e8 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -571,123 +571,89 @@ pub struct SealedStoreCensusV1 { /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the /// census stays truthful when a receipt is mid-write or corrupt. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum SealedCensusEntry { + Head, + Superseded, + AbandonedStaging, + Unrecognized, +} + pub fn census_sealed_store( database_path: &Path, head_generations: &std::collections::BTreeSet, ) -> std::io::Result { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(SealedStoreCensusV1::default()); - } - Err(error) => return Err(error), - }; - let mut census = SealedStoreCensusV1::default(); - let mut counted = std::collections::HashSet::new(); - for entry in entries { - let entry = entry?; - if !entry.file_type()?.is_dir() { - census.unrecognized_count += 1; - continue; - } - let path = entry.path(); - let name = entry.file_name(); - if name.to_str().is_some_and(sealed_staging_name) { - // A matching filename is not proof that the directory is - // abandoned: remounts can overlap a seal in this process, and - // foreign or interrupted pre-receipt entries may occupy its name. - // Do not walk unverified trees (which may be arbitrarily large). - let verified = tracedecay_private_fs::framed_log::read_bounded( - &path.join(SEALED_STAGING_OWNER_FILE), - MAX_SEALED_RECEIPT_BYTES, - ) - .ok() - .flatten() - .and_then(|bytes| verified_staging_receipt(&name, &bytes)) - .is_some_and(|owner| owner.producer != row_spill_owner().to_string()); - if !verified { - census.unrecognized_count += 1; - continue; + let current_producer = row_spill_owner().to_string(); + let totals = tracedecay_private_fs::capability_dir::census_receipted_directory_bytes( + &root, + SealedCensusEntry::Unrecognized, + |name| { + if name.to_str().is_some_and(sealed_staging_name) { + (SEALED_STAGING_OWNER_FILE, MAX_SEALED_RECEIPT_BYTES) + } else { + (SEALED_STORE_RECEIPT_FILE, MAX_SEALED_RECEIPT_BYTES) } - let bytes = directory_bytes(&path, &mut counted)?; - census.abandoned_staging_count += 1; - census.abandoned_staging_bytes = census.abandoned_staging_bytes.saturating_add(bytes); - continue; - } - // Receipt existence is not sufficient: a copied receipt can name a - // verified head while residing in an unrelated physical directory. - // Classify unproven entries before walking or charging their contents. - let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); - let Some(receipt) = receipt else { - census.unrecognized_count += 1; - continue; - }; - let Some(name) = name.to_str() else { - census.unrecognized_count += 1; - continue; - }; - let form_valid = match receipt.form.as_str() { - SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), - SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), - _ => false, - }; - if !form_valid - || receipt.version != SEALED_STORE_RECEIPT_VERSION - || receipt.graph_format == 0 - || receipt.graph_format > GraphFormatVersion::current().get() - || receipt.recovered_digest.is_empty() - || !sealed_receipt_matches_generation_entry(&receipt, name) - { - census.unrecognized_count += 1; - continue; - } - let bytes = directory_bytes(&path, &mut counted)?; - if head_generations.contains(&receipt.generation) { - census.head_count += 1; - census.head_bytes = census.head_bytes.saturating_add(bytes); - } else { - census.superseded_count += 1; - census.superseded_bytes = census.superseded_bytes.saturating_add(bytes); - } - } - Ok(census) -} - -/// Bytes under `directory` not already counted in `counted`: a layered -/// generation's base container is a hard link to bytes another directory -/// already holds, so each file is charged once per census. -fn directory_bytes( - directory: &Path, - counted: &mut std::collections::HashSet<(u64, u64)>, -) -> std::io::Result { - let mut total = 0u64; - let mut pending = vec![directory.to_path_buf()]; - while let Some(directory) = pending.pop() { - for entry in std::fs::read_dir(&directory)? { - let entry = entry?; - // Count only the link itself: following a foreign directory link - // can mischarge its entire tree or revisit an ancestor forever. - let metadata = std::fs::symlink_metadata(entry.path())?; - if metadata.is_dir() { - pending.push(entry.path()); - continue; + }, + |name, bytes| { + let Some(bytes) = bytes else { + return SealedCensusEntry::Unrecognized; + }; + if name.to_str().is_some_and(sealed_staging_name) { + return if verified_staging_receipt(name, bytes) + .is_some_and(|owner| owner.producer != current_producer) + { + SealedCensusEntry::AbandonedStaging + } else { + SealedCensusEntry::Unrecognized + }; } - #[cfg(unix)] + let Some(name) = name.to_str() else { + return SealedCensusEntry::Unrecognized; + }; + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return SealedCensusEntry::Unrecognized; + }; + let form_valid = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + if !form_valid + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.graph_format == 0 + || receipt.graph_format > GraphFormatVersion::current().get() + || receipt.recovered_digest.is_empty() + || !sealed_receipt_matches_generation_entry(&receipt, name) { - use std::os::unix::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; - } + return SealedCensusEntry::Unrecognized; } - #[cfg(not(unix))] - let _ = &counted; - total = total.saturating_add(metadata.len()); + if head_generations.contains(&receipt.generation) { + SealedCensusEntry::Head + } else { + SealedCensusEntry::Superseded + } + }, + )?; + let mut census = SealedStoreCensusV1::default(); + for (kind, (count, bytes)) in totals { + match kind { + SealedCensusEntry::Head => { + census.head_count = count; + census.head_bytes = bytes; + } + SealedCensusEntry::Superseded => { + census.superseded_count = count; + census.superseded_bytes = bytes; + } + SealedCensusEntry::AbandonedStaging => { + census.abandoned_staging_count = count; + census.abandoned_staging_bytes = bytes; + } + SealedCensusEntry::Unrecognized => census.unrecognized_count = count, } } - Ok(total) + Ok(census) } /// Removes every sealed generation beside `database_path` whose receipt does diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 1dbb76aaefe..1a60b1184fc 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -334,6 +334,84 @@ pub fn install_child_directory_noreplace_nofollow( rename_noreplace(&root, staging_name, &root, generation_name) } +/// Census proof-bound children through held root/child capabilities, not +/// ambient paths. Unknown entries are counted but never traversed; only +/// multiply linked inodes consume deduplication memory. Receipts are bounded +/// and read from the same child whose bytes are subsequently charged. +pub fn census_receipted_directory_bytes( + root_path: &Path, + unrecognized: K, + mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, +) -> io::Result> { + let root = match open_root_nofollow(root_path) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Default::default()), + Err(error) => return Err(error), + }; + let mut totals = std::collections::BTreeMap::::new(); + let mut counted = std::collections::HashSet::<(u64, u64)>::new(); + for entry in root.read_dir(".")? { + let entry = entry?; + let name = entry.file_name(); + let child = if entry.file_type()?.is_dir() { + root.open_dir_nofollow(&name).ok() + } else { + None + }; + let (category, bytes) = if let Some(child) = child { + let (receipt_name, maximum) = receipt_for(&name); + let receipt = + read_bounded_child_receipt(&child, OsStr::new(receipt_name), maximum).ok(); + let category = classify(&name, receipt.as_deref()); + let bytes = if category == unrecognized { + 0 + } else { + directory_bytes_from_held_dir(child, &mut counted)? + }; + (category, bytes) + } else { + (unrecognized, 0) + }; + let totals = totals.entry(category).or_default(); + totals.0 = totals.0.saturating_add(1); + totals.1 = totals.1.saturating_add(bytes); + } + Ok(totals) +} + +/// Walk no-follow entries relative to the same held child, even after its +/// old pathname has been renamed. The stack scales with directory depth. +fn directory_bytes_from_held_dir( + directory: Dir, + counted: &mut std::collections::HashSet<(u64, u64)>, +) -> io::Result { + let mut total = 0_u64; + let mut pending = vec![directory]; + while let Some(directory) = pending.pop() { + for entry in directory.read_dir(".")? { + let entry = entry?; + let name = entry.file_name(); + let metadata = directory.symlink_metadata(&name)?; + if metadata.is_dir() { + pending.push(directory.open_dir_nofollow(&name)?); + continue; + } + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { + continue; + } + } + #[cfg(not(unix))] + let _ = &counted; + total = total.saturating_add(metadata.len()); + } + } + Ok(total) +} + /// Remove exactly one held child directory without following a substituted /// sealed-root or child symlink. Missing entries are idempotent; other failures /// remain errors so callers can retain retry ownership. @@ -820,6 +898,50 @@ mod tests { ); } + /// An ambient-path census would charge the 100,041 foreign bytes after + /// a root swap. This census must charge only its held 41-byte payload. + #[cfg(unix)] + #[test] + fn receipted_census_keeps_opened_child_after_root_substitution() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("displaced"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join("generation")).unwrap(); + std::fs::create_dir_all(foreign.join("generation")).unwrap(); + std::fs::write(root.join("generation/owner"), b"owned").unwrap(); + std::fs::write(root.join("generation/payload"), vec![b'a'; 41]).unwrap(); + std::fs::write(foreign.join("generation/owner"), b"owned").unwrap(); + std::fs::write(foreign.join("generation/payload"), vec![b'f'; 100_041]).unwrap(); + #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] + enum Entry { + Owned, + Unknown, + } + let totals = census_receipted_directory_bytes( + &root, + Entry::Unknown, + |_| ("owner", 16), + |name, receipt| { + assert_eq!(name, OsStr::new("generation")); + assert_eq!(receipt, Some(b"owned".as_slice())); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + Entry::Owned + }, + ) + .unwrap(); + assert_eq!(totals.get(&Entry::Owned), Some(&(1, 46))); + assert_eq!( + std::fs::read(foreign.join("generation/payload")).unwrap(), + vec![b'f'; 100_041], + ); + assert_eq!( + std::fs::read(displaced.join("generation/payload")).unwrap(), + vec![b'a'; 41], + ); + } + /// Candidate filtering must not reopen a potentially substituted root: /// only the receipted owned child goes away; unrelated and foreign trees survive. #[cfg(unix)] From db26ea9948b0829fb15ea92e931f20fb477cc59b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:54:11 -0800 Subject: [PATCH 040/111] fix(graph): bound sealed census directory descriptors Walk verified generation children depth-first through held parent capabilities, not with one open handle per sibling. Add a native resource-limited regression with 128 sibling directories and exact byte accounting. Refs #4616 --- .../src/capability_dir.rs | 101 ++++++++++++++---- 1 file changed, 83 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 1a60b1184fc..17501a7b29b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -387,27 +387,33 @@ fn directory_bytes_from_held_dir( counted: &mut std::collections::HashSet<(u64, u64)>, ) -> io::Result { let mut total = 0_u64; - let mut pending = vec![directory]; - while let Some(directory) = pending.pop() { - for entry in directory.read_dir(".")? { - let entry = entry?; - let name = entry.file_name(); - let metadata = directory.symlink_metadata(&name)?; - if metadata.is_dir() { - pending.push(directory.open_dir_nofollow(&name)?); + // Hold only the current ancestry chain. Opening every sibling before + // visiting any of them exhausted the process descriptor limit on wide + // verified generations and made the whole sealed census fail. + let mut pending = vec![(directory.read_dir(".")?, directory)]; + while let Some((entries, parent)) = pending.last_mut() { + let Some(entry) = entries.next() else { + pending.pop(); + continue; + }; + let entry = entry?; + let name = entry.file_name(); + let metadata = parent.symlink_metadata(&name)?; + if metadata.is_dir() { + let child = parent.open_dir_nofollow(&name)?; + pending.push((child.read_dir(".")?, child)); + continue; + } + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { continue; } - #[cfg(unix)] - { - use cap_std::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; - } - } - #[cfg(not(unix))] - let _ = &counted; - total = total.saturating_add(metadata.len()); } + #[cfg(not(unix))] + let _ = &counted; + total = total.saturating_add(metadata.len()); } Ok(total) } @@ -785,6 +791,65 @@ mod tests { Dir::open_ambient_dir(path, ambient_authority()).expect("open test directory") } + #[cfg(unix)] + #[test] + fn wide_verified_generation_census_stays_within_descriptor_budget() { + const CHILD: &str = "TRACEDECAY_WIDE_CENSUS_RLIMIT_CHILD"; + if std::env::var_os(CHILD).is_none() { + // Resource limits are process-wide; isolate the reduced budget + // from concurrently executing tests in the parent libtest runner. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "capability_dir::tests::wide_verified_generation_census_stays_within_descriptor_budget", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "census under reduced descriptor budget failed: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child must execute the actual behavioral regression: {}", + String::from_utf8_lossy(&output.stdout) + ); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("sealed.json"), b"owned").unwrap(); + for index in 0..128 { + let column = generation.join(format!("column-{index}")); + std::fs::create_dir(&column).unwrap(); + std::fs::write(column.join("value"), b"x").unwrap(); + } + + // The old breadth-first handle stack needed >=128 live handles + // here; the corrected walk needs only the current three levels. + let mut limit: libc::rlimit = unsafe { std::mem::zeroed() }; + assert_eq!(unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, 0); + limit.rlim_cur = limit.rlim_cur.min(64); + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0); + + let totals = census_receipted_directory_bytes( + &root, + false, + |_| ("sealed.json", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + ) + .unwrap(); + assert_eq!(totals.get(&true), Some(&(1, 133))); + assert_eq!(std::fs::read(generation.join("column-127/value")).unwrap(), b"x"); + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 02a570bcfea1a8b8d2db9a856fcaefa5e574f587 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:15:53 -0800 Subject: [PATCH 041/111] style(graph): satisfy Bazel rustfmt for sealed census regression Refs #4616 --- crates/tracedecay-private-fs/src/capability_dir.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 17501a7b29b..5374ca610d8 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -835,7 +835,10 @@ mod tests { // The old breadth-first handle stack needed >=128 live handles // here; the corrected walk needs only the current three levels. let mut limit: libc::rlimit = unsafe { std::mem::zeroed() }; - assert_eq!(unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, 0); + assert_eq!( + unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, + 0 + ); limit.rlim_cur = limit.rlim_cur.min(64); assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0); @@ -847,7 +850,10 @@ mod tests { ) .unwrap(); assert_eq!(totals.get(&true), Some(&(1, 133))); - assert_eq!(std::fs::read(generation.join("column-127/value")).unwrap(), b"x"); + assert_eq!( + std::fs::read(generation.join("column-127/value")).unwrap(), + b"x" + ); } #[test] From 66d70eeefdf2d9766fa24a8189f4055dfe81eb1e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:15:56 -0800 Subject: [PATCH 042/111] fix(graph): remove unowned sealed cleanup after checked retirement Drop obsolete pathname-cleanup helpers superseded by the receipt-verified retirement path. They are unused and fail the Bazel Clippy dead-code gate. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ba4cf6be8e8..257e117bdbc 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -904,18 +904,6 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } } -fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { - let root = directory - .parent() - .ok_or_else(|| GraphDbError::unavailable("sealed directory has no parent"))?; - let name = directory - .file_name() - .ok_or_else(|| GraphDbError::unavailable("sealed directory has no entry name"))?; - tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(root, name) - .map(|_| ()) - .map_err(|error| sealed_store_io_failure("sealed directory removal failed", error)) -} - /// Accepted replay retirement is not permission to recursively delete a /// substituted or unrecognizable artifact. Require the exact durable identity /// under a held child capability; preserve an unverified directory for retry. @@ -985,17 +973,6 @@ fn discard_failed_sealed_adoption( } } -fn remove_sealed_directory(directory: &Path) { - if let Err(error) = remove_sealed_directory_checked(directory) { - tracing::warn!( - event = "sealed_artifact_cleanup_deferred", - path = %directory.display(), - %error, - "sealed artifact cleanup did not complete" - ); - } -} - /// Open options for a sealed artifact: read-only, so a reopen for proof, /// adoption, or serving never re-serializes the immutable container on close /// and never moves the identity its marker binds. A sealed container is only From 807a738e5340bee642b25a5cf82000076ea46628 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:55:05 -0800 Subject: [PATCH 043/111] fix(graph): preserve sealed artifacts after uncertain reader close Refs #4616 Never unlink a verified sealed generation when its native reader failed physical close. Preserve the first non-idempotent close failure across layered readers, while allowing cleanly closed derived readers to retire. Add real sealed-build regressions for a poisoned native close and a previously closed reader. --- .../tracedecay-graph-db/src/sealed_store.rs | 80 +++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 257e117bdbc..2c35bef1130 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -408,13 +408,19 @@ impl SealedGenerationStore { } pub(crate) fn close(&self) -> Result<(), GraphDbError> { - let mut result = Ok(()); + // Closing the same derived reader twice is harmless. A durability + // failure is not: preserve the first real close failure even when a + // later layered-base reader was already closed. + let mut failure = None; for database in self.engines() { - if let Err(error) = database.close() { - result = Err(error); + match database.close() { + Ok(()) | Err(GraphDbError::Closed) => {} + Err(error) => { + failure.get_or_insert(error); + } } } - result + failure.map_or(Ok(()), Err) } pub(crate) fn hibernate_if_lazy(&self) -> Result<(), GraphDbError> { @@ -507,7 +513,9 @@ impl SealedGenerationStore { /// unlink. A failed unlink remains a typed error for the durable cleanup /// owner to retry; a sealed-only live head never enters this path. fn discard(&self) -> Result<(), GraphDbError> { - let _ = self.close(); + // An unconfirmed physical close cannot authorize unlinking the only + // durable sealed copy. Leave its receipt for a later retirement retry. + self.close()?; retire_sealed_directory_checked( &self.directory, &self.locator, @@ -3881,6 +3889,68 @@ mod build_tests { assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); } + #[cfg(unix)] + #[test] + fn retirement_keeps_sealed_bytes_when_reader_close_is_uncertain() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let directory = store.directory.clone(); + let container = directory.join(SEALED_STORE_DATABASE_FILE); + let receipt = directory.join(SEALED_STORE_RECEIPT_FILE); + assert!(container.is_file() && receipt.is_file()); + + // Poison the real native reader's engine lock. Its physical close + // now returns DurabilityUncertain; the old discard ignored that error + // and removed the verified directory anyway on Unix. + let reader = Arc::clone(&store.database); + let poisoned = std::thread::spawn(move || { + let _guard = reader.inner.database.write().unwrap(); + panic!("exercise uncertain sealed reader close"); + }); + assert!(poisoned.join().is_err()); + let refused = store.discard(); + assert!( + matches!(refused, Err(GraphDbError::DurabilityUncertain { .. })), + "retirement must surface the failed native close: {refused:?}" + ); + assert!(container.is_file(), "uncertain close must preserve sealed bytes"); + assert!(receipt.is_file(), "retry must retain its ownership receipt"); + } + + #[test] + fn retirement_accepts_an_already_closed_derived_reader() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let directory = store.directory.clone(); + store.database.close().unwrap(); + store.discard().unwrap(); + assert!(!directory.exists(), "a cleanly closed reader is reclaimable"); + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From 44f9220878afb656995607ace98bc04969582bcc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:57:00 -0800 Subject: [PATCH 044/111] fix(graph): retain poisoned close state across retirement retries Refs #4616 A failed native close consumes the engine handle but leaves durability poisoned; a subsequent close may return Closed. Preserve the poison signal so a later retirement tick cannot delete that unproven artifact. Exercise the exact prior-uncertain-close state with a native sealed store. --- .../tracedecay-graph-db/src/sealed_store.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 2c35bef1130..ea5b933bb02 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -419,6 +419,14 @@ impl SealedGenerationStore { failure.get_or_insert(error); } } + // An earlier failed engine.close() may have consumed its handle + // and set Closed on the next attempt. Closed is idempotent only + // when the underlying reader is not poisoned. + if database.inner.poisoned.load(AtomicOrdering::Acquire) { + failure.get_or_insert_with(|| GraphDbError::DurabilityUncertain { + message: "sealed reader physical close remains uncertain".to_owned(), + }); + } } failure.map_or(Ok(()), Err) } @@ -3951,6 +3959,41 @@ mod build_tests { assert!(!directory.exists(), "a cleanly closed reader is reclaimable"); } + #[test] + fn retirement_does_not_erase_a_prior_uncertain_close() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let directory = store.directory.clone(); + let container = directory.join(SEALED_STORE_DATABASE_FILE); + let receipt = directory.join(SEALED_STORE_RECEIPT_FILE); + + // Reproduce the post-failure state of native engine.close(): its + // handle is gone, Closed will be returned on retry, but durability + // remains uncertain. The first patch must not mistake that for a + // previously successful close and destroy its sealed receipt. + store.database.inner.poisoned.store(true, Ordering::Release); + let first = store.database.close(); + assert!(matches!(first, Err(GraphDbError::DurabilityUncertain { .. }))); + let retry = store.discard(); + assert!( + matches!(retry, Err(GraphDbError::DurabilityUncertain { .. })), + "a prior uncertain close must not become an unlink: {retry:?}" + ); + assert!(container.is_file() && receipt.is_file()); + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From b6f3ed5e6882b6d69e4965787c562135172ac789 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:56:48 -0800 Subject: [PATCH 045/111] fix(graph): retain failed-close readers across retirement retries Refs #4616 Retirement must retain its registered native reader when close or verified unlink fails. Otherwise the next tick takes the disk-only path and erases a sealed artifact whose physical close remains uncertain. Keep the map entry until verified removal, and remove it only if the same reader is still installed. Exercise two real retirement retries with a poisoned reader and successful normal retirement. Correct the three formatter-reported hunks without weakening CI. --- .../tracedecay-graph-db/src/sealed_store.rs | 140 ++++++++++++++++-- 1 file changed, 128 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ea5b933bb02..80a320d16f3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1893,18 +1893,33 @@ impl GraphDb { &self, locator: &GenerationLocator, ) -> Result<(), GraphDbError> { - let removed = self + let registered = self .inner .sealed_generations - .write() + .read() .map_err(|_| GraphDbError::unavailable("sealed generation registry lock poisoned"))? - .remove(locator); - if let Some(store) = removed { - return store.discard(); + .get(locator) + .cloned(); + if let Some(store) = registered { + // Keep the registered reader until its physical close AND sealed + // directory removal both succeed. A failed close may consume the + // native handle; dropping the registry entry would let the next + // tick take the disk-only path and unlink uncertain bytes. + store.discard()?; + let mut sealed = self.inner.sealed_generations.write().map_err(|_| { + GraphDbError::unavailable("sealed generation registry lock poisoned") + })?; + if sealed + .get(locator) + .is_some_and(|current| Arc::ptr_eq(current, &store)) + { + sealed.remove(locator); + } + return Ok(()); } - // A removed in-memory reader is not necessary for the artifact to - // exist on disk. Unlink by exact locator even after a prior failed - // retirement; a durable tombstone can then retry without the reader. + // The artifact may exist on disk without a resident reader, for + // example after reopening a store before generation activation. + // Only receipt-verified, exact-locator disk cleanup is authorized. let Some(reopen) = self.inner.reopen.as_ref() else { return Ok(()); }; @@ -3933,7 +3948,10 @@ mod build_tests { matches!(refused, Err(GraphDbError::DurabilityUncertain { .. })), "retirement must surface the failed native close: {refused:?}" ); - assert!(container.is_file(), "uncertain close must preserve sealed bytes"); + assert!( + container.is_file(), + "uncertain close must preserve sealed bytes" + ); assert!(receipt.is_file(), "retry must retain its ownership receipt"); } @@ -3956,7 +3974,10 @@ mod build_tests { let directory = store.directory.clone(); store.database.close().unwrap(); store.discard().unwrap(); - assert!(!directory.exists(), "a cleanly closed reader is reclaimable"); + assert!( + !directory.exists(), + "a cleanly closed reader is reclaimable" + ); } #[test] @@ -3981,11 +4002,14 @@ mod build_tests { // Reproduce the post-failure state of native engine.close(): its // handle is gone, Closed will be returned on retry, but durability - // remains uncertain. The first patch must not mistake that for a + // remains uncertain. Retirement must never mistake that state for a // previously successful close and destroy its sealed receipt. store.database.inner.poisoned.store(true, Ordering::Release); let first = store.database.close(); - assert!(matches!(first, Err(GraphDbError::DurabilityUncertain { .. }))); + assert!(matches!( + first, + Err(GraphDbError::DurabilityUncertain { .. }) + )); let retry = store.discard(); assert!( matches!(retry, Err(GraphDbError::DurabilityUncertain { .. })), @@ -3994,6 +4018,98 @@ mod build_tests { assert!(container.is_file() && receipt.is_file()); } + #[test] + fn retirement_retries_must_keep_a_reader_with_uncertain_close_registered() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let parent = GraphDb::open(GraphDbOpenOptions { + location: GraphDbLocation::Memory, + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::Memory, + cancellation: Arc::new(NeverCancelled), + }) + .unwrap(); + parent + .inner + .sealed_generations + .write() + .unwrap() + .insert(locator.clone(), Arc::clone(&store)); + + let container = store.directory.join(SEALED_STORE_DATABASE_FILE); + let receipt = store.directory.join(SEALED_STORE_RECEIPT_FILE); + let held_reader = Arc::clone(store.database()); + store.database.inner.poisoned.store(true, Ordering::Release); + for attempt in 1..=2 { + let retirement = parent.retire_sealed_generation_store(&locator); + assert!( + matches!(&retirement, Err(GraphDbError::DurabilityUncertain { .. })), + "retirement attempt {attempt} must refuse uncertain close: {retirement:?}" + ); + let registered = parent + .sealed_generation_reader(&locator) + .expect("failed close must preserve the registered reader"); + assert!(Arc::ptr_eq(®istered, &store)); + assert!(container.is_file() && receipt.is_file()); + } + assert!(Arc::ptr_eq(&held_reader, store.database())); + } + + #[test] + fn successful_retirement_uninstalls_the_reader_after_removing_artifact() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let parent = GraphDb::open(GraphDbOpenOptions { + location: GraphDbLocation::Memory, + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::Memory, + cancellation: Arc::new(NeverCancelled), + }) + .unwrap(); + parent + .inner + .sealed_generations + .write() + .unwrap() + .insert(locator.clone(), Arc::clone(&store)); + + parent.retire_sealed_generation_store(&locator).unwrap(); + assert!(parent.sealed_generation_reader(&locator).is_none()); + assert!(!store.directory.exists()); + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From 0b7c0558f3d1b2e985217eaddb16520638f60cd0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:57:54 -0800 Subject: [PATCH 046/111] test(graph): prove failed retirement cannot delete sealed bytes Refs #4616 Use the real persistent parent store in both retirement regressions. The prior implementation then takes the disk-only path on the second retry and erases the verified sealed artifact despite uncertain close; assert two typed refusals and byte-exact artifact preservation. --- crates/tracedecay-graph-db/src/sealed_store.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 80a320d16f3..db76bd82740 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -4039,9 +4039,9 @@ mod build_tests { identity.generation.clone(), ); let parent = GraphDb::open(GraphDbOpenOptions { - location: GraphDbLocation::Memory, + location: GraphDbLocation::Persistent(database_path.clone()), expected_format: GraphFormatVersion::current(), - durability: GraphDurability::Memory, + durability: GraphDurability::WalSync, cancellation: Arc::new(NeverCancelled), }) .unwrap(); @@ -4092,9 +4092,9 @@ mod build_tests { identity.generation.clone(), ); let parent = GraphDb::open(GraphDbOpenOptions { - location: GraphDbLocation::Memory, + location: GraphDbLocation::Persistent(database_path.clone()), expected_format: GraphFormatVersion::current(), - durability: GraphDurability::Memory, + durability: GraphDurability::WalSync, cancellation: Arc::new(NeverCancelled), }) .unwrap(); From e50180bbcf221a5738d72b0cf886532fdf899498 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:21:56 -0800 Subject: [PATCH 047/111] fix(graph): preserve poisoned close with an absent engine Refs #4616 A failed native close can consume the handle and leave durability poison. Even with no engine resident, a later close must never report success or idempotent Closed. Align the copied-receipt recovery fixture with the producer-bound staging authority, and start the live-reader timing bound after the release worker has actually started. --- crates/tracedecay-graph-db/src/runtime.rs | 14 +++++++++++--- .../verified_generation_contract/sealed_store.rs | 15 ++++++++++----- .../staging_footprint.rs | 6 ++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/runtime.rs b/crates/tracedecay-graph-db/src/runtime.rs index 7c40e5b3573..a75438acdc4 100644 --- a/crates/tracedecay-graph-db/src/runtime.rs +++ b/crates/tracedecay-graph-db/src/runtime.rs @@ -1028,10 +1028,18 @@ impl GraphDb { } }; if !engine_is_open { - if self.inner.closed.swap(true, Ordering::AcqRel) { - return Err(GraphDbError::Closed); + let previously_closed = self.inner.closed.swap(true, Ordering::AcqRel); + // A failed physical close can already have consumed the engine. + // Its absent handle must not turn an uncertain result into a + // successful close (or an idempotent Closed on the next retry). + if self.inner.poisoned.load(Ordering::Acquire) { + return Err(durability_uncertain()); } - return Ok(()); + return if previously_closed { + Err(GraphDbError::Closed) + } else { + Ok(()) + }; } let _snapshot_gate = self.wait_snapshot_gate_write(); let mut guard = diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 61cbf81297e..ec402598a0f 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1229,9 +1229,9 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { assert_snapshot_reads(&snapshot, &identity, "durable"); } -/// A completed interrupted seal carries an exact generation receipt and is -/// reclaimed when reopening the store. Unreceipted, mismatched, oversized and -/// linked lookalikes must survive instead of being deleted by filename. +/// A copied installed receipt is not a staging-owner receipt. Recovery must +/// preserve that lookalike alongside unreceipted, mismatched, oversized and +/// linked entries, rather than deleting them based on their names. #[test] fn reopening_the_store_reclaims_only_receipted_staging() { let temp = TempDir::new().unwrap(); @@ -1331,8 +1331,13 @@ fn reopening_the_store_reclaims_only_receipted_staging() { .unwrap(); assert!( - !verified.exists(), - "a verified abandoned seal must be reclaimed" + verified.is_dir(), + "a copied final receipt must not authorize abandoned-stage cleanup" + ); + assert_eq!( + std::fs::read(verified.join("sealed.json")).unwrap(), + receipt, + "recovery must preserve the copied final receipt byte-for-byte" ); assert_eq!( std::fs::read(unowned.join("generation.grafeo")).unwrap(), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs index 01c55d10f34..fa83a05c605 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs @@ -909,8 +909,10 @@ fn release_sweep_retries_idle_sealed_reader_hibernation() { let root = temp.path().to_path_buf(); let projection = replay.publication.key.projection.clone(); let (result_tx, result_rx) = mpsc::channel(); + let (started_tx, started_rx) = mpsc::channel(); let sweep = thread::spawn(move || { let (control, probe) = control_and_probe(); + started_tx.send(()).unwrap(); let result = registry.release_sealed_generation_staging_rows( registration(binding, &root), &mut authority, @@ -920,6 +922,10 @@ fn release_sweep_retries_idle_sealed_reader_hibernation() { result_tx.send(()).unwrap(); (authority, result) }); + // Start the bound only after the sweep thread has actually been + // scheduled; otherwise the test measures worker startup under CI load, + // not whether a live sealed reader blocks staging cleanup. + started_rx.recv().unwrap(); let busy_completion = result_rx.recv_timeout(Duration::from_millis(100)); let engines_during_read = database.sealed_generation_engine_census(); release.wait(); From 9dfe06dbd6a0e7d385129bc100b932c3215d9270 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:28:10 -0800 Subject: [PATCH 048/111] fix(graph): fence lazy reopen against physical close Refs #4616 A lazy opener checked availability before waiting for the database lock. A concurrent close could mark the handle Closed during that wait and the opener would still publish a native engine into the closed handle. Publish a no-engine close while holding the database read lock and recheck availability under the lazy opener's write lock. A native persistent-store regression races both never-opened and hibernated handles at the existing lifecycle seam, checks no engine resurrects, and verifies a new mount still reads the committed generation. --- crates/tracedecay-graph-db/src/runtime.rs | 16 ++++- .../tracedecay-graph-db/src/runtime/tests.rs | 70 +++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/runtime.rs b/crates/tracedecay-graph-db/src/runtime.rs index a75438acdc4..43ff17f5f56 100644 --- a/crates/tracedecay-graph-db/src/runtime.rs +++ b/crates/tracedecay-graph-db/src/runtime.rs @@ -1015,8 +1015,8 @@ impl GraphDb { /// management, still checkpoints in full before this returns. #[tracing::instrument(name = "graph_db.runtime.close", level = "trace", skip_all)] pub(crate) fn close(&self) -> Result<(), GraphDbError> { - let engine_is_open = match self.inner.database.read() { - Ok(database) => database.is_some(), + let engine = match self.inner.database.read() { + Ok(database) => database, Err(_) => { self.inner.closed.store(true, Ordering::Release); self.inner.poisoned.store(true, Ordering::Release); @@ -1027,7 +1027,9 @@ impl GraphDb { }); } }; - if !engine_is_open { + if engine.is_none() { + // Keep the absence observation locked until close is published. + // A lazy opener must not install an engine in that interval. let previously_closed = self.inner.closed.swap(true, Ordering::AcqRel); // A failed physical close can already have consumed the engine. // Its absent handle must not turn an uncertain result into a @@ -1041,6 +1043,7 @@ impl GraphDb { Ok(()) }; } + drop(engine); let _snapshot_gate = self.wait_snapshot_gate_write(); let mut guard = match crate::observe::wait_lock(crate::observe::LOCK_WAIT_DATABASE_WRITE, || { @@ -1721,11 +1724,16 @@ impl GraphDb { return Ok(()); } self.ensure_available()?; + #[cfg(test)] + test_seams::fire(test_seams::Seam::LazyOpenAdmission); let mut database = crate::observe::wait_lock(crate::observe::LOCK_WAIT_DATABASE_WRITE, || { self.inner.database.write() }) .map_err(|_| GraphDbError::unavailable("graph database write lock is poisoned"))?; + // Close or a failed hibernation may have won while this opener + // waited. Revalidate before publishing another native engine. + self.ensure_available()?; if database.is_some() { return Ok(()); } @@ -1973,6 +1981,8 @@ pub(crate) mod test_seams { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum Seam { + /// After lazy-open admission, before acquiring the database write lock. + LazyOpenAdmission, /// Immediately before grafeo opens the container at `site`. EngineOpen(GraphEngineOpenSite), /// After the engine has closed and synced the container, immediately diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index e682aad20fa..418ab47216a 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -6,6 +6,7 @@ use std::time::Duration; use grafeo_common::types::Value; +use super::test_seams::{self, Seam}; use super::{GraphDb, sync_wal}; use crate::location::PersistentGraphStoreState; use crate::recovery::set_projection_quarantine; @@ -29,6 +30,75 @@ fn memory_db() -> GraphDbLeaseV1 { .unwrap() } +#[test] +fn lazy_open_rechecks_close_before_materializing_an_engine() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("graph.grafeo"); + let options = || GraphDbOpenOptions { + location: GraphDbLocation::Persistent(path.clone()), + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::WalSync, + cancellation: Arc::new(NeverCancelled), + }; + let seed = GraphDb::open(options()).unwrap(); + seed.apply_unverified(scalar_batch("retained")).unwrap(); + seed.close().unwrap(); + drop(seed); + + // Both a never-materialized mount and a hibernated engine can be absent + // when close races a lazy reader's admission. + for hibernated in [false, true] { + let database = GraphDb::open_lazy_with_store_state(options(), PersistentGraphStoreState::Existing) + .unwrap(); + if hibernated { + database.ensure_opened().unwrap(); + database.hibernate_if_lazy().unwrap(); + } + assert!(database.inner.database.read().unwrap().is_none()); + + let closing = Arc::clone(&database); + let admissions = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&admissions); + let hook = test_seams::install(move |seam| { + if seam == Seam::LazyOpenAdmission { + observed.fetch_add(1, Ordering::Relaxed); + // The opener has already observed Ready. Complete the real + // close before it takes its database write lock. + closing.close().unwrap(); + } + }); + let result = database.ensure_opened(); + drop(hook); + + assert_eq!(admissions.load(Ordering::Relaxed), 1); + assert_eq!(result, Err(GraphDbError::Closed)); + assert_eq!(database.runtime_state(), GraphDbRuntimeState::Closed); + assert!( + database.inner.database.read().unwrap().is_none(), + "a completed close must not be followed by a resurrected engine" + ); + } + + // Refusing the stale opener must not corrupt the durable store or block + // a new, independently authorized mount from reading the committed row. + let reopened = GraphDb::open(options()).unwrap(); + let snapshot = reopened.snapshot().unwrap(); + let entity = snapshot + .entity( + &GraphNamespace::new("project").unwrap(), + &GraphEntityId::new("a").unwrap(), + Arc::new(NeverCancelled), + ) + .unwrap() + .unwrap(); + assert_eq!( + entity.properties.get(&GraphPropertyName::new("name").unwrap()), + Some(&GraphProperty::String("retained".to_owned())) + ); + drop(snapshot); + reopened.close().unwrap(); +} + #[test] fn runtime_state_distinguishes_ready_closed_and_durability_uncertain() { let ready = memory_db(); From 4af675591e5bfa25b8ab12230b5e5b34eee90a57 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:29:19 -0800 Subject: [PATCH 049/111] style(graph): format lazy-close regression setup Refs #4616 Use repository-standard multiline call wrapping in the new native test without changing its assertions or the close/reopen behavior. --- crates/tracedecay-graph-db/src/runtime/tests.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index 418ab47216a..8edccc786b9 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -48,8 +48,9 @@ fn lazy_open_rechecks_close_before_materializing_an_engine() { // Both a never-materialized mount and a hibernated engine can be absent // when close races a lazy reader's admission. for hibernated in [false, true] { - let database = GraphDb::open_lazy_with_store_state(options(), PersistentGraphStoreState::Existing) - .unwrap(); + let database = + GraphDb::open_lazy_with_store_state(options(), PersistentGraphStoreState::Existing) + .unwrap(); if hibernated { database.ensure_opened().unwrap(); database.hibernate_if_lazy().unwrap(); From a59921d4bcd4ef40c7a41ea5d3b51e72b20bdbc9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:33:51 -0800 Subject: [PATCH 050/111] fix(graph): verify failed spill publication cleanup Refs #4616 A rejected rename dropped TempDir with pathname-based recursive cleanup still enabled. Another real directory installed at the pending name could be deleted despite having no row-spill ownership receipt. Disable automatic cleanup before publication, and on failure reclaim only a receipt-verified child under its held directory capability. A native regression substitutes a foreign real directory before a refused publication and preserves both it and the occupied destination. --- .../src/generation/spill.rs | 94 ++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 8add23cd36d..88fee1ee212 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -155,13 +155,62 @@ struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { - let mut staging = Self::prepare(&path, row_spill_owner())?; - tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) - .map_err(|error| spill_io("directory install", error))?; - staging.disable_cleanup(true); + let owner = row_spill_owner(); + let mut staging = Self::prepare(&path, owner)?; + Self::install(&mut staging, &path, owner)?; Ok(Self(path)) } + fn install( + staging: &mut tempfile::TempDir, + path: &Path, + owner: u128, + ) -> Result<(), GraphDbError> { + // TempDir's default Drop recursively deletes by pathname. A failed + // no-replace publication must not delete a foreign directory moved + // into that name while we attempted installation. Disable it before + // the rename, including the successful-rename/reused-name window. + staging.disable_cleanup(true); + let name = path + .file_name() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; + let receipt = row_spill_owner_receipt(name, owner) + .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; + let pending = staging_name(owner, &receipt); + if staging.path().file_name() != Some(OsStr::new(&pending)) { + return Err(GraphDbError::invalid( + "graph row spill staging name does not bind its owner receipt", + )); + } + if let Err(error) = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path) + { + let root = staging.path().parent().ok_or_else(|| { + GraphDbError::invalid("graph row spill directory has no parent") + })?; + // On failure, only the exact owner receipt read from the opened + // child authorizes recursive removal. A substituted directory, + // even an ordinary real directory, must survive unchanged. + if let Err(cleanup_error) = + tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + OsStr::new(&pending), + OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |bytes| bytes == receipt, + ) + { + tracing::warn!( + event = "graph_row_spill_prepare_cleanup_refused", + path = %staging.path().display(), + error = %cleanup_error, + "unverified pending row spill preserved for inspection" + ); + } + return Err(spill_io("directory install", error)); + } + Ok(()) + } + fn prepare(path: &Path, owner: u128) -> Result { let name = path .file_name() @@ -1181,6 +1230,43 @@ mod owner_receipt_tests { row_spill_owner, staging_name, }; + #[cfg(unix)] + #[test] + fn failed_pending_publication_preserves_a_substituted_real_directory() { + let root = tempfile::tempdir().unwrap(); + let owner = row_spill_owner(); + let destination = root.path().join(format!("{ROW_SPILL_PREFIX}{owner}-0")); + let mut staged = SpillDirectory::prepare(&destination, owner).unwrap(); + let pending_path = staged.path().to_path_buf(); + let moved = root.path().join("original-pending-spill"); + std::fs::rename(&pending_path, &moved).unwrap(); + std::fs::create_dir(&pending_path).unwrap(); + std::fs::write(pending_path.join("sentinel"), b"foreign pending contents").unwrap(); + std::fs::create_dir(&destination).unwrap(); + std::fs::write(destination.join("sentinel"), b"occupied destination").unwrap(); + + let refused = SpillDirectory::install(&mut staged, &destination, owner); + assert!( + refused.is_err(), + "occupied publication destination must be refused" + ); + drop(staged); + assert_eq!( + std::fs::read(pending_path.join("sentinel")).unwrap(), + b"foreign pending contents", + "failed install must not recursively delete a substituted real directory" + ); + assert_eq!( + std::fs::read(destination.join("sentinel")).unwrap(), + b"occupied destination" + ); + let original_receipt = std::fs::read(moved.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(); + assert!( + original_receipt.starts_with(RECEIPT_PREFIX.as_bytes()), + "original owned staging bytes must remain separate from the foreign entry" + ); + } + #[cfg(unix)] #[test] fn receipt_write_failure_leaves_no_spill_directory() { From 9f04bedee24a8a46ee277a6eacf422264fb3afd3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:34:27 -0800 Subject: [PATCH 051/111] style(graph): keep spill install result explicit Refs #4616 Name the no-replace outcome before matching its error, keeping the receipt-verified cleanup logic and regression unchanged. --- crates/tracedecay-graph-db/src/generation/spill.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 88fee1ee212..7d9d7937eb8 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -182,8 +182,8 @@ impl SpillDirectory { "graph row spill staging name does not bind its owner receipt", )); } - if let Err(error) = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path) - { + let installed = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path); + if let Err(error) = installed { let root = staging.path().parent().ok_or_else(|| { GraphDbError::invalid("graph row spill directory has no parent") })?; From f1f880fb899921d20a251a662ff51b2553a02379 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:44:13 -0800 Subject: [PATCH 052/111] fix(graph): keep pending receipt failure cleanup owner-bound Refs #4616 Disable TempDir pathname cleanup before creating a pending receipt. Use a held, no-follow directory for exclusive marker publication and nonrecursive cleanup after a failed write. Preserve substituted directories and verify the refusal with an on-disk ownership regression. --- .../src/generation/spill.rs | 78 ++++++++++++++++--- .../src/capability_dir.rs | 54 +++++++++++++ 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 7d9d7937eb8..4a533d5e3b3 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -223,24 +223,31 @@ impl SpillDirectory { // Receipt failures must leave the requested name available to retry. // Keep the partial marker private until the complete spill can be installed. let temporary = staging_name(owner, &receipt); - let staging = tempfile::Builder::new() + let mut staging = tempfile::Builder::new() .prefix(&temporary) .rand_bytes(0) .tempdir_in(parent) .map_err(|error| spill_io("directory create", error))?; - let marker = staging.path().join(ROW_SPILL_OWNER_RECEIPT_FILE); - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&marker) - .map_err(|error| spill_io("owner marker create", error))?; - file.write_all(&receipt) - .and_then(|()| file.sync_all()) - .map_err(|error| spill_io("owner marker persist", error))?; - drop(file); + Self::persist_pending_receipt(&mut staging, &receipt)?; Ok(staging) } + fn persist_pending_receipt( + staging: &mut tempfile::TempDir, + receipt: &[u8], + ) -> Result<(), GraphDbError> { + // TempDir's Drop is pathname-based even when receipt creation fails. + // Once our name has been replaced, that Drop could erase foreign + // bytes. Disarm it before the first fallible marker operation. + staging.disable_cleanup(true); + tracedecay_private_fs::capability_dir::write_exclusive_pending_receipt_nofollow( + staging.path(), + OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + receipt, + ) + .map_err(|error| spill_io("owner marker persist", error)) + } + fn path(&self) -> &Path { &self.0 } @@ -1227,7 +1234,7 @@ impl GraphGenerationManifestIdentity { mod owner_receipt_tests { use super::{ RECEIPT_PREFIX, ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, SpillDirectory, - row_spill_owner, staging_name, + row_spill_owner, row_spill_owner_receipt, staging_name, }; #[cfg(unix)] @@ -1267,6 +1274,53 @@ mod owner_receipt_tests { ); } + #[cfg(unix)] + #[test] + fn failed_pending_receipt_open_preserves_substituted_directory() { + let root = tempfile::tempdir().unwrap(); + let owner = row_spill_owner(); + let destination = root.path().join(format!("{ROW_SPILL_PREFIX}{owner}-9")); + let receipt = row_spill_owner_receipt(destination.file_name().unwrap(), owner).unwrap(); + let name = staging_name(owner, &receipt); + let mut staging = tempfile::Builder::new() + .prefix(&name) + .rand_bytes(0) + .tempdir_in(root.path()) + .unwrap(); + let pending_path = staging.path().to_path_buf(); + let moved = root.path().join("original-pending"); + std::fs::rename(&pending_path, &moved).unwrap(); + std::fs::create_dir(&pending_path).unwrap(); + std::fs::write( + pending_path.join(ROW_SPILL_OWNER_RECEIPT_FILE), + b"foreign receipt", + ) + .unwrap(); + std::fs::write(pending_path.join("sentinel"), b"foreign contents").unwrap(); + + let refused = SpillDirectory::persist_pending_receipt(&mut staging, &receipt); + assert!( + refused + .unwrap_err() + .to_string() + .contains("owner marker persist") + ); + drop(staging); + assert_eq!( + std::fs::read(pending_path.join("sentinel")).unwrap(), + b"foreign contents", + "failed receipt creation must not drop a substituted real directory" + ); + assert_eq!( + std::fs::read(pending_path.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(), + b"foreign receipt" + ); + assert!( + moved.is_dir(), + "the original staging directory cannot authorize deleting its replacement" + ); + } + #[cfg(unix)] #[test] fn receipt_write_failure_leaves_no_spill_directory() { diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 5374ca610d8..2ac29928977 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -228,6 +228,60 @@ fn single_component(name: &OsStr) -> io::Result<()> { Ok(()) } +/// Persist a new producer-owned staging receipt through a held directory. +/// +/// An exclusive marker creation is the only authority for removing an +/// incomplete marker. On write/sync failure, remove only that marker and +/// attempt a nonrecursive removal of the same held directory. A substituted +/// directory containing other bytes must survive. The caller must disable +/// pathname-based temporary-directory cleanup before invoking this operation. +pub fn write_exclusive_pending_receipt_nofollow( + directory_path: &Path, + filename: &OsStr, + contents: &[u8], +) -> io::Result<()> { + single_component(filename)?; + let directory = open_root_nofollow(directory_path)?; + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + let mut marker = directory.open_with(filename, &options)?; + #[cfg(unix)] + let marker_identity = { + use cap_std::fs::MetadataExt; + marker.metadata().ok().map(|metadata| (metadata.dev(), metadata.ino())) + }; + let written = marker.write_all(contents).and_then(|()| marker.sync_all()); + drop(marker); + if let Err(error) = written { + // Keep the original write failure. A different marker entry is not + // ours to delete, and a nonempty substituted directory is not ours + // to remove. Neither cleanup operation follows an ambient pathname. + #[cfg(unix)] + let same_marker = { + use cap_std::fs::MetadataExt; + marker_identity.is_some_and(|identity| { + directory + .symlink_metadata(filename) + .ok() + .is_some_and(|metadata| { + metadata.is_file() && (metadata.dev(), metadata.ino()) == identity + }) + }) + }; + #[cfg(not(unix))] + let same_marker = false; + if same_marker { + if let Err(cleanup) = directory.remove_file(filename) { + tracing::warn!(%cleanup, "pending owner marker could not be removed"); + } else if let Err(cleanup) = directory.remove_open_dir() { + tracing::warn!(%cleanup, "pending owner directory was not empty"); + } + } + return Err(error); + } + Ok(()) +} + /// Publish a rebuildable marker through one held directory. Replacing the /// pathname after acquisition cannot redirect its bytes into a foreign tree. pub fn atomic_replace_child_file_nofollow( From 72d885b06661ac558423b357bf69707769a79ae7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:58:29 -0800 Subject: [PATCH 053/111] fix(graph): attribute sealed hardlinks to serving heads Refs #4616 The sealed census charged a shared inode to the first directory visited. A superseded generation could therefore claim live-head physical bytes. Transfer hard-link charges to the stronger verified category regardless of read_dir order, retaining bounded metadata-only accounting. Prove the adverse visitation order using real native hard links and the production census. --- .../src/capability_dir.rs | 98 +++++++++++++++++-- 1 file changed, 92 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 2ac29928977..a8768369cfe 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -404,7 +404,10 @@ pub fn census_receipted_directory_bytes( Err(error) => return Err(error), }; let mut totals = std::collections::BTreeMap::::new(); - let mut counted = std::collections::HashSet::<(u64, u64)>::new(); + // Hard links belong to the highest-priority verified category, not the + // first entry returned by read_dir. Retain only multiply-linked identities + // so the census remains bounded independently of file count and size. + let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); for entry in root.read_dir(".")? { let entry = entry?; let name = entry.file_name(); @@ -421,7 +424,7 @@ pub fn census_receipted_directory_bytes( let bytes = if category == unrecognized { 0 } else { - directory_bytes_from_held_dir(child, &mut counted)? + directory_bytes_from_held_dir(child, category, &mut totals, &mut counted)? }; (category, bytes) } else { @@ -436,9 +439,11 @@ pub fn census_receipted_directory_bytes( /// Walk no-follow entries relative to the same held child, even after its /// old pathname has been renamed. The stack scales with directory depth. -fn directory_bytes_from_held_dir( +fn directory_bytes_from_held_dir( directory: Dir, - counted: &mut std::collections::HashSet<(u64, u64)>, + category: K, + totals: &mut std::collections::BTreeMap, + counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, ) -> io::Result { let mut total = 0_u64; // Hold only the current ancestry chain. Opening every sibling before @@ -461,8 +466,31 @@ fn directory_bytes_from_held_dir( #[cfg(unix)] { use cap_std::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; + if metadata.nlink() > 1 { + let key = (metadata.dev(), metadata.ino()); + if let Some((previous_kind, charged_bytes)) = counted.get_mut(&key) { + // A lower-ordered category has stronger ownership. A + // superseded generation encountered first cannot claim + // hard-linked physical bytes from a later live head. + if category < *previous_kind { + let previous = totals.get_mut(&*previous_kind).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes lack their prior category", + ) + })?; + previous.1 = previous.1.checked_sub(*charged_bytes).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes exceed their prior category", + ) + })?; + total = total.saturating_add(*charged_bytes); + *previous_kind = category; + } + continue; + } + counted.insert(key, (category, metadata.len())); } } #[cfg(not(unix))] @@ -910,6 +938,64 @@ mod tests { ); } + /// Hard-linked base bytes count once and belong to the live head even + /// when a superseded generation was physically visited first. + #[cfg(unix)] + #[test] + fn shared_sealed_bytes_follow_head_priority_not_directory_visitation() { + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] + enum Kind { + Head, + Superseded, + Unrecognized, + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let superseded = root.join("old"); + let head = root.join("head"); + std::fs::create_dir_all(&superseded).unwrap(); + std::fs::create_dir(&head).unwrap(); + std::fs::write(superseded.join("owner"), b"old").unwrap(); + std::fs::write(head.join("owner"), b"head").unwrap(); + std::fs::write(superseded.join("shared"), b"shared-bytes").unwrap(); + std::fs::hard_link(superseded.join("shared"), head.join("shared")).unwrap(); + + // Exercise the actual walker in the adverse order without relying on + // the operating system's unspecified read_dir ordering. + let mut counted = std::collections::HashMap::new(); + let mut charged = std::collections::BTreeMap::::new(); + for (kind, path) in [(Kind::Superseded, &superseded), (Kind::Head, &head)] { + let bytes = + directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) + .unwrap(); + let entry = charged.entry(kind).or_default(); + entry.0 += 1; + entry.1 += bytes; + } + assert_eq!(charged.get(&Kind::Superseded), Some(&(1, 3))); + assert_eq!(charged.get(&Kind::Head), Some(&(1, 16))); + assert_eq!(counted.len(), 1, "one shared inode was visited twice"); + + // The production census also classifies receipt-bound children and + // must report one physical copy without charging retired bytes twice. + let reported = census_receipted_directory_bytes( + &root, + Kind::Unrecognized, + |_| ("owner", 64), + |_, receipt| match receipt { + Some(bytes) if bytes == b"head" => Kind::Head, + Some(bytes) if bytes == b"old" => Kind::Superseded, + _ => Kind::Unrecognized, + }, + ) + .unwrap(); + assert_eq!(reported.get(&Kind::Head), Some(&(1, 16))); + assert_eq!(reported.get(&Kind::Superseded), Some(&(1, 3))); + assert_eq!(reported.get(&Kind::Unrecognized), None); + assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 8b1b06206ae8ee9542bad85c860b055e171a9463 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:00:12 -0800 Subject: [PATCH 054/111] style(graph): normalize sealed hardlink regression wrapping Refs #4616 Keep the native test's method chain inside the repository rustfmt width. No production behavior or test assertion changes. --- crates/tracedecay-private-fs/src/capability_dir.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index a8768369cfe..0d1c8e55d15 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -966,9 +966,8 @@ mod tests { let mut counted = std::collections::HashMap::new(); let mut charged = std::collections::BTreeMap::::new(); for (kind, path) in [(Kind::Superseded, &superseded), (Kind::Head, &head)] { - let bytes = - directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) - .unwrap(); + let bytes = directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) + .unwrap(); let entry = charged.entry(kind).or_default(); entry.0 += 1; entry.1 += bytes; From 94b5d9fcc68f0242617e5e3a92acc92e1f840405 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:01:03 -0800 Subject: [PATCH 055/111] docs(graph): clarify shared-inode census accounting bounds Refs #4616 State that only multi-linked inode identities consume deduplication metadata, and explain the priority ordering used by the census. No behavior or test assertion changes. --- crates/tracedecay-private-fs/src/capability_dir.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 0d1c8e55d15..705c53ebbf3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -392,6 +392,8 @@ pub fn install_child_directory_noreplace_nofollow( /// ambient paths. Unknown entries are counted but never traversed; only /// multiply linked inodes consume deduplication memory. Receipts are bounded /// and read from the same child whose bytes are subsequently charged. +/// Category ordering is authority ordering: lower keys receive shared +/// physical bytes before higher keys, independent of directory iteration. pub fn census_receipted_directory_bytes( root_path: &Path, unrecognized: K, @@ -405,8 +407,8 @@ pub fn census_receipted_directory_bytes( }; let mut totals = std::collections::BTreeMap::::new(); // Hard links belong to the highest-priority verified category, not the - // first entry returned by read_dir. Retain only multiply-linked identities - // so the census remains bounded independently of file count and size. + // first entry returned by read_dir. Only multiply-linked identities need + // tracking; memory depends on shared inodes, not file byte size. let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); for entry in root.read_dir(".")? { let entry = entry?; From b23f36fb15ef33d9c3e4f355dd5d808d0427e6d8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:43:28 -0800 Subject: [PATCH 056/111] style(graph): align native tests with pinned rustfmt Refs #4616 Apply the three exact formatting hunks reported by Bazel repository gates at PR head 94b5d9fc; no behavior, tests, or CI configuration changed. --- crates/tracedecay-graph-db/src/generation/spill.rs | 7 ++++--- crates/tracedecay-graph-db/src/runtime/tests.rs | 4 +++- crates/tracedecay-private-fs/src/capability_dir.rs | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 4a533d5e3b3..2efee498bcd 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -184,9 +184,10 @@ impl SpillDirectory { } let installed = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path); if let Err(error) = installed { - let root = staging.path().parent().ok_or_else(|| { - GraphDbError::invalid("graph row spill directory has no parent") - })?; + let root = staging + .path() + .parent() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; // On failure, only the exact owner receipt read from the opened // child authorizes recursive removal. A substituted directory, // even an ordinary real directory, must survive unchanged. diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index 8edccc786b9..dd15c12f819 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -93,7 +93,9 @@ fn lazy_open_rechecks_close_before_materializing_an_engine() { .unwrap() .unwrap(); assert_eq!( - entity.properties.get(&GraphPropertyName::new("name").unwrap()), + entity + .properties + .get(&GraphPropertyName::new("name").unwrap()), Some(&GraphProperty::String("retained".to_owned())) ); drop(snapshot); diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 705c53ebbf3..3f5dd0a7c3c 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -248,7 +248,10 @@ pub fn write_exclusive_pending_receipt_nofollow( #[cfg(unix)] let marker_identity = { use cap_std::fs::MetadataExt; - marker.metadata().ok().map(|metadata| (metadata.dev(), metadata.ino())) + marker + .metadata() + .ok() + .map(|metadata| (metadata.dev(), metadata.ino())) }; let written = marker.write_all(contents).and_then(|()| marker.sync_all()); drop(marker); From bca1e23adbef3b0e32f86dc345454bbdfc017c48 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:45:57 -0800 Subject: [PATCH 057/111] fix(graph): bound sealed census hard-link bookkeeping Refs #4616 Bound shared-inode metadata and reject a census as partial before its allocation scales indefinitely with millions of hard-linked files. Use fallible reservation and a filesystem regression for the refusal and the normal complete census. No CI or release settings changed. --- .../src/capability_dir.rs | 101 +++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 3f5dd0a7c3c..b77cd86f1ce 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -397,11 +397,32 @@ pub fn install_child_directory_noreplace_nofollow( /// and read from the same child whose bytes are subsequently charged. /// Category ordering is authority ordering: lower keys receive shared /// physical bytes before higher keys, independent of directory iteration. +/// Bound deduplication metadata independently of the size of the store: the +/// Doctor caller reports partial coverage rather than trusting a census +/// whose shared-inode inventory would grow without limit. +const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; + pub fn census_receipted_directory_bytes( + root_path: &Path, + unrecognized: K, + receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, +) -> io::Result> { + census_receipted_directory_bytes_bounded( + root_path, + unrecognized, + receipt_for, + classify, + MAX_SEALED_CENSUS_SHARED_INODES, + ) +} + +fn census_receipted_directory_bytes_bounded( root_path: &Path, unrecognized: K, mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + max_shared_inodes: usize, ) -> io::Result> { let root = match open_root_nofollow(root_path) { Ok(root) => root, @@ -429,7 +450,13 @@ pub fn census_receipted_directory_bytes( let bytes = if category == unrecognized { 0 } else { - directory_bytes_from_held_dir(child, category, &mut totals, &mut counted)? + directory_bytes_from_held_dir( + child, + category, + &mut totals, + &mut counted, + max_shared_inodes, + )? }; (category, bytes) } else { @@ -449,6 +476,7 @@ fn directory_bytes_from_held_dir( category: K, totals: &mut std::collections::BTreeMap, counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, + max_shared_inodes: usize, ) -> io::Result { let mut total = 0_u64; // Hold only the current ancestry chain. Opening every sibling before @@ -495,6 +523,18 @@ fn directory_bytes_from_held_dir( } continue; } + if counted.len() >= max_shared_inodes { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census shared-inode budget exhausted; coverage is partial", + )); + } + counted.try_reserve(1).map_err(|_| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census shared-inode allocation failed; coverage is partial", + ) + })?; counted.insert(key, (category, metadata.len())); } } @@ -971,8 +1011,14 @@ mod tests { let mut counted = std::collections::HashMap::new(); let mut charged = std::collections::BTreeMap::::new(); for (kind, path) in [(Kind::Superseded, &superseded), (Kind::Head, &head)] { - let bytes = directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) - .unwrap(); + let bytes = directory_bytes_from_held_dir( + open(path), + kind, + &mut charged, + &mut counted, + MAX_SEALED_CENSUS_SHARED_INODES, + ) + .unwrap(); let entry = charged.entry(kind).or_default(); entry.0 += 1; entry.1 += bytes; @@ -1000,6 +1046,55 @@ mod tests { assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); } + /// An adversarial store with many distinct hard-linked inodes must + /// decline its complete census before unbounded heap growth. A caller + /// can then truthfully mark coverage partial rather than report zeros as + /// a complete inventory. The ordinary production entrypoint still counts + /// all the same real files when within its budget. + #[cfg(unix)] + #[test] + fn sealed_census_refuses_excess_shared_inodes_without_losing_files() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("owner"), b"owned").unwrap(); + for index in 0..3 { + let source = generation.join(format!("payload-{index}")); + std::fs::write(&source, b"x").unwrap(); + std::fs::hard_link(&source, temp.path().join(format!("alias-{index}"))).unwrap(); + } + + let refused = census_receipted_directory_bytes_bounded( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + 2, + ) + .unwrap_err(); + assert_eq!(refused.kind(), io::ErrorKind::OutOfMemory, "{refused}"); + + let complete = census_receipted_directory_bytes( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + ) + .unwrap(); + assert_eq!(complete.get(&true), Some(&(1, 8))); + for index in 0..3 { + assert_eq!( + std::fs::read(generation.join(format!("payload-{index}"))).unwrap(), + b"x" + ); + assert_eq!( + std::fs::read(temp.path().join(format!("alias-{index}"))).unwrap(), + b"x" + ); + } + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 3c4fab3b7bbfcf9773ae94670a1aaa3d3fad81e4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:46:33 -0800 Subject: [PATCH 058/111] fix(graph): keep shared census budget portable Refs #4616 The bounded Unix inode census leaves its budget unused on other targets. Retain the existing non-Unix no-op binding to keep strict lint clean. --- crates/tracedecay-private-fs/src/capability_dir.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index b77cd86f1ce..426fba5da39 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -539,7 +539,7 @@ fn directory_bytes_from_held_dir( } } #[cfg(not(unix))] - let _ = &counted; + let _ = (&counted, max_shared_inodes); total = total.saturating_add(metadata.len()); } Ok(total) From d7bbee1d551773e1b1bf77ed56ee273ada627637 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:47:53 -0800 Subject: [PATCH 059/111] docs(graph): retain public sealed census API documentation Refs #4616 Attach the invariant documentation to the exported census API, not to the shared-inode memory ceiling. --- crates/tracedecay-private-fs/src/capability_dir.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 426fba5da39..5dc80d2e995 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -391,17 +391,17 @@ pub fn install_child_directory_noreplace_nofollow( rename_noreplace(&root, staging_name, &root, generation_name) } +/// Bound deduplication metadata independently of the size of the store. +/// The Doctor caller reports partial coverage instead of trusting a census +/// whose shared-inode inventory would otherwise grow without limit. +const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; + /// Census proof-bound children through held root/child capabilities, not /// ambient paths. Unknown entries are counted but never traversed; only /// multiply linked inodes consume deduplication memory. Receipts are bounded /// and read from the same child whose bytes are subsequently charged. /// Category ordering is authority ordering: lower keys receive shared /// physical bytes before higher keys, independent of directory iteration. -/// Bound deduplication metadata independently of the size of the store: the -/// Doctor caller reports partial coverage rather than trusting a census -/// whose shared-inode inventory would grow without limit. -const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; - pub fn census_receipted_directory_bytes( root_path: &Path, unrecognized: K, From 6027cbacfd7e53332053beeef4e6b172fc035b62 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:49:05 -0800 Subject: [PATCH 060/111] fix(graph): refuse missing registered sealed retirement Refs #4616 When a registered sealed store's root or generation directory was moved before receipt-verified deletion, no-follow cleanup returned Ok(false). That was mistaken for successful retirement, uninstalling its reader and marking generation state collected while durable bytes survived elsewhere. Only unregistered disk-only retries may treat an absent path as idempotent. A real sealed store regression moves the root and child independently, requires typed refusal and retained reader, then restores and reclaims both. --- .../tracedecay-graph-db/src/sealed_store.rs | 97 ++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index db76bd82740..831c31b8c56 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -965,9 +965,19 @@ fn retire_sealed_directory_checked( && expected_digest.is_none_or(|digest| receipt.recovered_digest == digest) }, ); - result - .map(|_| ()) - .map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error)) + let removed = + result.map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error))?; + // A resident reader carries exact proof that this artifact was installed. + // If its root or child disappeared before we acquired a deletion capability, + // physical retirement was not confirmed: the directory may have been moved + // aside and still contain the sole durable generation. Keep the reader + // registered so maintenance can retry rather than mark it collected. + if !removed && expected_digest.is_some() { + return Err(GraphDbError::unavailable( + "registered sealed generation retirement found no owned artifact", + )); + } + Ok(()) } /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted @@ -4110,6 +4120,87 @@ mod build_tests { assert!(!store.directory.exists()); } + /// A missing pathname is not evidence that an installed generation's + /// durable bytes were removed. It may have been relocated between + /// registration and the retirement attempt. + #[test] + fn registered_retirement_refuses_disappeared_root_or_child_until_restored() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + for move_root in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let parent = GraphDb::open(GraphDbOpenOptions { + location: GraphDbLocation::Persistent(database_path.clone()), + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::WalSync, + cancellation: Arc::new(NeverCancelled), + }) + .unwrap(); + parent + .inner + .sealed_generations + .write() + .unwrap() + .insert(locator.clone(), Arc::clone(&store)); + + let root = sealed_store_root(&database_path); + let (original, relocated) = if move_root { + (root, temp.path().join("relocated-sealed-root")) + } else { + (store.directory.clone(), root.join("relocated-sealed-child")) + }; + std::fs::rename(&original, &relocated).unwrap(); + let surviving_receipt = if move_root { + relocated + .join(store.directory.file_name().unwrap()) + .join(SEALED_STORE_RECEIPT_FILE) + } else { + relocated.join(SEALED_STORE_RECEIPT_FILE) + }; + let receipt = std::fs::read(&surviving_receipt).unwrap(); + + let refused = parent.retire_sealed_generation_store(&locator); + assert!( + matches!(&refused, Err(GraphDbError::Unavailable { .. })), + "missing owned artifact must remain pending (root={move_root}): {refused:?}" + ); + assert_eq!( + std::fs::read(&surviving_receipt).unwrap(), + receipt, + "retirement must preserve the relocated sealed receipt" + ); + let retained = parent + .sealed_generation_reader(&locator) + .expect("failed retirement must retain its registered reader"); + assert!(Arc::ptr_eq(&retained, &store)); + drop(retained); + + // Retry against the restored owner must really delete the bytes + // and remove the reader. A permanent refusal cannot pass. + std::fs::rename(&relocated, &original).unwrap(); + parent.retire_sealed_generation_store(&locator).unwrap(); + assert!(parent.sealed_generation_reader(&locator).is_none()); + assert!(!store.directory.exists()); + // Disk-only duplicate notifications remain idempotent. + retire_sealed_directory_checked(&store.directory, &locator, None).unwrap(); + } + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From 9c3dfc482ae3143d6d29739632684706477f8f85 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:50:14 -0800 Subject: [PATCH 061/111] style(graph): align retirement error mapping with rustfmt Refs #4616 Keep the new typed missing-artifact retirement branch unchanged. --- crates/tracedecay-graph-db/src/sealed_store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 831c31b8c56..a36636f4119 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -965,8 +965,8 @@ fn retire_sealed_directory_checked( && expected_digest.is_none_or(|digest| receipt.recovered_digest == digest) }, ); - let removed = - result.map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error))?; + let removed = result + .map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error))?; // A resident reader carries exact proof that this artifact was installed. // If its root or child disappeared before we acquired a deletion capability, // physical retirement was not confirmed: the directory may have been moved From b9798d7169bcc15f9ad19731d483be863c760621 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:50:59 -0800 Subject: [PATCH 062/111] test(graph): exercise relocated sealed paths on Unix Refs #4616 The native directory-rename regression uses Unix unlink/rename semantics. Keep cross-platform product behavior unchanged. --- crates/tracedecay-graph-db/src/sealed_store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a36636f4119..7e2558c9e5e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -4123,6 +4123,7 @@ mod build_tests { /// A missing pathname is not evidence that an installed generation's /// durable bytes were removed. It may have been relocated between /// registration and the retirement attempt. + #[cfg(unix)] #[test] fn registered_retirement_refuses_disappeared_root_or_child_until_restored() { let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); From 1f3e3364cb0885c055cf26e9dccd0cc4b0029e88 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:47:19 -0800 Subject: [PATCH 063/111] fix(graph): import sealed retirement verifier in native test Refs #4616 The moved-sealed-root regression calls the production receipt-verified retirement helper from its nested test module without importing it. Both Linux Bazel graph-db unit compilation and strict Clippy fail with E0425 before the regression executes. Import the actual helper and keep the native relocation, refusal, restoration, and idempotency assertions. --- crates/tracedecay-graph-db/src/sealed_store.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7e2558c9e5e..c54bb1b1a83 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3508,9 +3508,9 @@ mod build_tests { MAX_SEALED_RECEIPT_BYTES, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, SealedStagingOwnerV1, build_or_open_sealed_store, build_sealed_container, census_sealed_store, - initial_sealed_staging_owner, persist_sealed_store_check, sealed_artifact_database_options, - sealed_copy_proof, sealed_generation_directory, sealed_store_root, - sweep_abandoned_sealed_staging, + initial_sealed_staging_owner, persist_sealed_store_check, retire_sealed_directory_checked, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ From 736779337edbefd5a640c37c1dff432e5079c2ec Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:05:39 -0800 Subject: [PATCH 064/111] test(graph): align graph-db unit tests with sealed retirement contracts Refs #4616 Unit tests never executed while compilation failed, so three assertions still described superseded contracts. A registered sealed generation whose artifact is absent is now an unconfirmed retirement (typed refusal), with only a disk-only retry idempotent. A close retry after a poisoned close stays DurabilityUncertain, not Closed, and the engine is still verified released. A prepared pending row spill survives drop by design, so the live-producer test installs it before checking name reuse. Co-Authored-By: Claude Sonnet 5.5 --- crates/tracedecay-graph-db/src/generation/spill.rs | 12 +++++++++--- crates/tracedecay-graph-db/src/runtime/tests.rs | 11 +++++++++-- crates/tracedecay-graph-db/src/sealed_store.rs | 9 ++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 2efee498bcd..15f501d30c7 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -1474,7 +1474,7 @@ mod owner_receipt_tests { let stale = SpillDirectory::prepare(&previous_path, previous) .unwrap() .keep(); - let live = SpillDirectory::prepare(&final_path, current).unwrap(); + let mut live = SpillDirectory::prepare(&final_path, current).unwrap(); let peer_path = root.join(format!(".rows-{current}-1")); let _peer = SpillDirectory::create(peer_path.clone()).unwrap(); @@ -1482,8 +1482,14 @@ mod owner_receipt_tests { assert!(!stale.exists()); assert!(live.path().is_dir()); assert!(peer_path.is_dir()); - drop(live); - let _published = SpillDirectory::create(final_path.clone()).unwrap(); + // A prepared pending spill deliberately survives a drop; the live + // producer completes it by installing under its canonical name. + SpillDirectory::install(&mut live, &final_path, current).unwrap(); + let published = SpillDirectory(final_path.clone()); + assert!(final_path.is_dir()); + drop(published); + assert!(!final_path.exists(), "the owner releases its own spill"); + let _reused = SpillDirectory::create(final_path.clone()).unwrap(); assert!(final_path.is_dir()); } diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index dd15c12f819..879e362892b 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -136,10 +136,17 @@ fn owner_close_releases_the_physical_database_after_durability_uncertainty() { .to_owned(), }) ); + assert!( + !owner.engine_is_open().unwrap(), + "the poisoned close must release the physical database" + ); assert_eq!( owner.close(), - Err(GraphDbError::Closed), - "the poisoned close must release the physical database before a second close" + Err(GraphDbError::DurabilityUncertain { + message: "the handle was poisoned after an observed post-commit persistence failure" + .to_owned(), + }), + "a retry after a poisoned close must stay uncertain, never idempotent Closed" ); } diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index c54bb1b1a83..7b2cf45ee89 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3386,7 +3386,14 @@ mod receipt_budget_tests { !directory.exists(), "verified retired generation is physically removed" ); - retire_sealed_directory_checked(&directory, &locator, Some("digest")).unwrap(); + // A registered reader proves the artifact was installed, so its + // absence is unconfirmed retirement. Only a disk-only retry with no + // resident reader may treat an already-absent directory as done. + assert!( + retire_sealed_directory_checked(&directory, &locator, Some("digest")).is_err(), + "a registered generation must not report retirement of an absent artifact" + ); + retire_sealed_directory_checked(&directory, &locator, None).unwrap(); } #[cfg(unix)] From 6939336af6c195e05642b6d94e20cc15f57abec7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:18:13 -0800 Subject: [PATCH 065/111] fix(graph): release fully observed hard-link census identities Refs #4616 A sealed store with many disjoint hard-link pairs exhausted the fixed shared-inode budget although only one dedup identity was unresolved at a time. Track the remaining filesystem-reported links and discard an identity after the final alias. Preserve ownership-priority byte transfer and the fail-closed bound for links outside the scanned store. Native filesystem regression walks four disjoint hard-link pairs with a one-entry unresolved budget; the old implementation fails on pair two. --- .../src/capability_dir.rs | 65 +++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 5dc80d2e995..db2191690c6 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -432,8 +432,9 @@ fn census_receipted_directory_bytes_bounded( let mut totals = std::collections::BTreeMap::::new(); // Hard links belong to the highest-priority verified category, not the // first entry returned by read_dir. Only multiply-linked identities need - // tracking; memory depends on shared inodes, not file byte size. - let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); + // tracking; memory depends on unresolved shared inodes, not the total + // number of linked files visited or their byte size. + let mut counted = std::collections::HashMap::<(u64, u64), (K, u64, u64)>::new(); for entry in root.read_dir(".")? { let entry = entry?; let name = entry.file_name(); @@ -475,7 +476,7 @@ fn directory_bytes_from_held_dir( directory: Dir, category: K, totals: &mut std::collections::BTreeMap, - counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, + counted: &mut std::collections::HashMap<(u64, u64), (K, u64, u64)>, max_shared_inodes: usize, ) -> io::Result { let mut total = 0_u64; @@ -501,7 +502,9 @@ fn directory_bytes_from_held_dir( use cap_std::fs::MetadataExt; if metadata.nlink() > 1 { let key = (metadata.dev(), metadata.ino()); - if let Some((previous_kind, charged_bytes)) = counted.get_mut(&key) { + if let Some((previous_kind, charged_bytes, remaining_links)) = + counted.get_mut(&key) + { // A lower-ordered category has stronger ownership. A // superseded generation encountered first cannot claim // hard-linked physical bytes from a later live head. @@ -521,6 +524,15 @@ fn directory_bytes_from_held_dir( total = total.saturating_add(*charged_bytes); *previous_kind = category; } + // An immutable hard-link group no longer needs its key + // after every filesystem-reported link has been seen. + // Retaining completed groups makes a store with many + // disjoint pairs exhaust the budget despite bounded + // concurrent deduplication demand. + *remaining_links -= 1; + if *remaining_links == 0 { + counted.remove(&key); + } continue; } if counted.len() >= max_shared_inodes { @@ -535,7 +547,7 @@ fn directory_bytes_from_held_dir( "sealed census shared-inode allocation failed; coverage is partial", ) })?; - counted.insert(key, (category, metadata.len())); + counted.insert(key, (category, metadata.len(), metadata.nlink() - 1)); } } #[cfg(not(unix))] @@ -1025,7 +1037,10 @@ mod tests { } assert_eq!(charged.get(&Kind::Superseded), Some(&(1, 3))); assert_eq!(charged.get(&Kind::Head), Some(&(1, 16))); - assert_eq!(counted.len(), 1, "one shared inode was visited twice"); + assert!( + counted.is_empty(), + "a fully observed hard-link group must free its deduplication slot" + ); // The production census also classifies receipt-bound children and // must report one physical copy without charging retired bytes twice. @@ -1046,6 +1061,44 @@ mod tests { assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); } + /// Completed hard-link groups need no further identity retention. A + /// bounded census must still cover many disjoint alias pairs whose total + /// exceeds its simultaneous unresolved-link budget. + #[cfg(unix)] + #[test] + fn complete_link_pairs_reuse_census_deduplication_budget() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("owner"), b"owned").unwrap(); + for index in 0..4 { + let column = generation.join(format!("column-{index}")); + std::fs::create_dir(&column).unwrap(); + let original = column.join("original"); + std::fs::write(&original, b"rows").unwrap(); + std::fs::hard_link(&original, column.join("alias")).unwrap(); + } + + // Each column is fully traversed before the next one is opened. + // Previously the four completed identities accumulated and this + // real filesystem walk failed at the second column's first link. + let census = census_receipted_directory_bytes_bounded( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + 1, + ) + .unwrap(); + assert_eq!(census.get(&true), Some(&(1, 21))); + for index in 0..4 { + let column = generation.join(format!("column-{index}")); + assert_eq!(std::fs::read(column.join("original")).unwrap(), b"rows"); + assert_eq!(std::fs::read(column.join("alias")).unwrap(), b"rows"); + } + } + /// An adversarial store with many distinct hard-linked inodes must /// decline its complete census before unbounded heap growth. A caller /// can then truthfully mark coverage partial rather than report zeros as From b1a9acd1067832281e32162418ee6c82c6ae1552 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:07:01 -0800 Subject: [PATCH 066/111] test(graph): fix canonical corruption paths and formatting Compare corruption census paths with the physical canonical path selected by the registry, including macOS temporary roots reached through /var aliases. Apply the pinned formatter to the new hard-link census branch. Bazel: graph-db 198 passed with one pre-existing ignored; private-fs 49 passed; repository rustfmt and strict Clippy on both unit targets passed. Concurrent-publication census correctness remains a separate unresolved finding. --- crates/tracedecay-graph-db/src/registry.rs | 4 +++- crates/tracedecay-private-fs/src/capability_dir.rs | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/registry.rs b/crates/tracedecay-graph-db/src/registry.rs index 2cb9827ec44..808696092f3 100644 --- a/crates/tracedecay-graph-db/src/registry.rs +++ b/crates/tracedecay-graph-db/src/registry.rs @@ -2806,7 +2806,9 @@ mod tests { registry .retain_verification_fault_for_lease(&lease, &fault) .unwrap(); - let path = corrupt_registration.canonical_path().to_path_buf(); + // TempDir may live under a symlinked temporary root. The registry + // reports the physical canonical path selected at admission. + let path = std::fs::canonicalize(corrupt_registration.canonical_path()).unwrap(); let bytes = std::fs::read(&path).unwrap(); for _ in 0..2 { diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index db2191690c6..255132c2644 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -502,8 +502,7 @@ fn directory_bytes_from_held_dir( use cap_std::fs::MetadataExt; if metadata.nlink() > 1 { let key = (metadata.dev(), metadata.ino()); - if let Some((previous_kind, charged_bytes, remaining_links)) = - counted.get_mut(&key) + if let Some((previous_kind, charged_bytes, remaining_links)) = counted.get_mut(&key) { // A lower-ordered category has stronger ownership. A // superseded generation encountered first cannot claim From 580c546ff371374ab6812d8ae6aed23d735ce9ae Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:24:30 -0800 Subject: [PATCH 067/111] fix(fs): retain census identities across sealed publication A filesystem link count does not close an inode group. Layer publication can add aliases after an observed group completes, including files originally visited with one link. Retain bounded inode evidence for the whole census and preserve priority transfer; refuse incomplete coverage when that evidence fills. Two native regressions publish real layered artifacts between visits of one census. Both reproduced a 42,246-byte excess charge before the fix and now match independent unique-byte and category accounting. Keep the disjoint-pair fixture with exact accounting at sufficient capacity and typed refusal below it. Validation: graph-db unit suite 200 passed, one pre-existing ignored; private-fs unit suite 49 passed; strict Bazel Clippy and pinned rustfmt checks pass for both. The 65,536-entry bound remains unmeasured; this does not establish retirement or CPU incident acceptance. --- .../tracedecay-graph-db/src/sealed_store.rs | 173 +++++++++++++++ .../src/capability_dir.rs | 198 +++++++++++------- 2 files changed, 290 insertions(+), 81 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7b2cf45ee89..914d1dd955e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3617,6 +3617,179 @@ mod build_tests { .unwrap() } + /// Publish a real layered artifact between visits of the same census. + /// Both a formerly single link and an exhausted two-link group must + /// remain one physical charge owned by the newly visited head. + #[cfg(unix)] + #[test] + fn sealed_census_counts_concurrently_published_single_base_link_once() { + census_with_concurrent_layer_publication(0); + } + + #[cfg(unix)] + #[test] + fn sealed_census_counts_concurrently_published_completed_link_group_once() { + census_with_concurrent_layer_publication(1); + } + + #[cfg(unix)] + fn census_with_concurrent_layer_publication(initial_layers: u64) { + use std::os::unix::fs::MetadataExt; + + use tracedecay_private_fs::capability_dir::test_support::census_in_entry_order; + + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let database = open_source(&database_path); + let manifest = manifest(3, 2); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let mut spill = database + .generation_row_spill(manifest.projection.clone()) + .unwrap(); + spill + .push_batch( + manifest.entities.clone(), + manifest.relations.clone(), + &|| Ok(()), + ) + .unwrap(); + std::fs::write(spill.attachment_path(), b"producer inputs").unwrap(); + let spilled = spill.finish(identity.clone(), &|| Ok(())).unwrap(); + let (cold, _) = build_or_open_sealed_store( + SealedRowSource::Spilled(&spilled), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + drop(spilled); + let base = cold.sealed_base(&|| Ok(())).unwrap().unwrap(); + let publish = |generation: &str| { + let mut identity = identity.clone(); + identity.generation = GraphGenerationId::new(generation).unwrap(); + let layered = database + .layered_row_spill(identity.projection.clone(), base.clone()) + .unwrap() + .finish(identity.clone(), &|| Ok(())) + .unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Layered(&layered), + &identity, + layered.expected_recovered_digest(), + &database_path, + &|| Ok(()), + ) + .unwrap(); + drop(layered); + store + }; + let mut stores = vec![cold]; + if initial_layers == 1 { + stores.push(publish("generation:child1")); + } + let root = sealed_store_root(&database_path); + let original = stores[0].directory.join(SEALED_STORE_DATABASE_FILE); + let original_bytes = std::fs::read(&original).unwrap(); + assert_eq!( + std::fs::metadata(&original).unwrap().nlink(), + 1 + initial_layers + ); + let initial_names: Vec<_> = stores + .iter() + .map(|store| store.directory.file_name().unwrap().to_owned()) + .collect(); + let mut initial_names = initial_names.into_iter(); + let receipts: Vec<_> = stores + .iter() + .map(|store| { + let path = store.directory.join(SEALED_STORE_RECEIPT_FILE); + let bytes = std::fs::read(&path).unwrap(); + (path, bytes) + }) + .collect(); + let mut published = false; + // The entry source determines only scheduling: all receipt reads, + // capability opens and accounting use the production census core. + let entries = std::iter::from_fn(|| { + if let Some(name) = initial_names.next() { + return Some(Ok(name)); + } + if published { + return None; + } + let child = publish("generation:child2"); + let name = child.directory.file_name().unwrap().to_owned(); + stores.push(child); + published = true; + Some(Ok(name)) + }); + let totals = census_in_entry_order( + &root, + entries, + 2_u8, + |_| (SEALED_STORE_RECEIPT_FILE, MAX_SEALED_RECEIPT_BYTES), + |_, receipt| { + let receipt: super::SealedStoreReceiptV1 = + serde_json::from_slice(receipt.unwrap()).unwrap(); + if receipt.generation == "generation:child2" { + 0 + } else { + 1 + } + }, + ) + .unwrap(); + assert!(published); + assert_eq!( + std::fs::metadata(&original).unwrap().nlink(), + 2 + initial_layers + ); + assert_eq!(std::fs::read(&original).unwrap(), original_bytes); + for (path, bytes) in receipts { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + // Independent native identity census of the completed real artifacts. + // Compare each category as well as the unique physical total. + let mut physical = BTreeMap::<_, (u8, u64)>::new(); + for (index, store) in stores.iter().enumerate() { + let kind = if index + 1 == stores.len() { 0 } else { 1 }; + for entry in std::fs::read_dir(&store.directory).unwrap() { + let metadata = entry.unwrap().metadata().unwrap(); + assert!(metadata.is_file()); + physical + .entry((metadata.dev(), metadata.ino())) + .and_modify(|(owner, _)| *owner = (*owner).min(kind)) + .or_insert((kind, metadata.len())); + } + } + for kind in [0, 1] { + let bytes: u64 = physical + .values() + .filter(|(owner, _)| *owner == kind) + .map(|(_, bytes)| bytes) + .sum(); + assert_eq!( + totals[&kind].1, bytes, + "initial layers {initial_layers}, category {kind}" + ); + } + assert_eq!(totals[&0].0, 1); + assert_eq!(totals[&1].0, 1 + initial_layers); + let census = census_sealed_store( + &database_path, + &BTreeSet::from(["generation:child2".to_owned()]), + ) + .unwrap(); + assert_eq!(census.head_bytes, totals[&0].1); + assert_eq!(census.superseded_bytes, totals[&1].1); + for store in stores { + store.close().unwrap(); + } + database.close().unwrap(); + } + #[cfg(unix)] #[test] fn sealed_census_only_attributes_verified_abandoned_staging() { diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 255132c2644..f4c179e072b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -393,13 +393,15 @@ pub fn install_child_directory_noreplace_nofollow( /// Bound deduplication metadata independently of the size of the store. /// The Doctor caller reports partial coverage instead of trusting a census -/// whose shared-inode inventory would otherwise grow without limit. -const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; +/// whose inode inventory would otherwise grow without limit. +const MAX_SEALED_CENSUS_INODES: usize = 65_536; /// Census proof-bound children through held root/child capabilities, not -/// ambient paths. Unknown entries are counted but never traversed; only -/// multiply linked inodes consume deduplication memory. Receipts are bounded -/// and read from the same child whose bytes are subsequently charged. +/// ambient paths. Unknown entries are counted but never traversed. On Unix, +/// every visited inode consumes bounded deduplication memory until the walk +/// ends: publication can add links even to a formerly single-linked file. +/// Exhausting that evidence budget returns `OutOfMemory`, never complete +/// totals. Receipts are bounded and read from the child being charged. /// Category ordering is authority ordering: lower keys receive shared /// physical bytes before higher keys, independent of directory iteration. pub fn census_receipted_directory_bytes( @@ -413,32 +415,50 @@ pub fn census_receipted_directory_bytes( unrecognized, receipt_for, classify, - MAX_SEALED_CENSUS_SHARED_INODES, + MAX_SEALED_CENSUS_INODES, ) } fn census_receipted_directory_bytes_bounded( root_path: &Path, unrecognized: K, - mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), - mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, - max_shared_inodes: usize, + receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + max_inodes: usize, ) -> io::Result> { let root = match open_root_nofollow(root_path) { Ok(root) => root, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Default::default()), Err(error) => return Err(error), }; + census_receipted_children( + &root, + root.read_dir(".")? + .map(|entry| entry.map(|entry| entry.file_name())), + unrecognized, + receipt_for, + classify, + max_inodes, + ) +} + +fn census_receipted_children( + root: &Dir, + entries: impl Iterator>, + unrecognized: K, + mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + max_inodes: usize, +) -> io::Result> { let mut totals = std::collections::BTreeMap::::new(); // Hard links belong to the highest-priority verified category, not the - // first entry returned by read_dir. Only multiply-linked identities need - // tracking; memory depends on unresolved shared inodes, not the total - // number of linked files visited or their byte size. - let mut counted = std::collections::HashMap::<(u64, u64), (K, u64, u64)>::new(); - for entry in root.read_dir(".")? { - let entry = entry?; - let name = entry.file_name(); - let child = if entry.file_type()?.is_dir() { + // first entry returned by read_dir. A link count is not a closed group: + // concurrent publication can add an alias after its last observed link. + let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); + for name in entries { + let name = name?; + single_component(&name)?; + let child = if root.symlink_metadata(&name)?.is_dir() { root.open_dir_nofollow(&name).ok() } else { None @@ -456,7 +476,7 @@ fn census_receipted_directory_bytes_bounded( category, &mut totals, &mut counted, - max_shared_inodes, + max_inodes, )? }; (category, bytes) @@ -470,14 +490,40 @@ fn census_receipted_directory_bytes_bounded( Ok(totals) } +/// Deterministic native scheduling for consumer publication regressions. +/// Receipt validation, held capabilities and inode accounting are shared with +/// the public census; only the filesystem's unspecified entry order is supplied. +#[cfg(feature = "test-helpers")] +pub mod test_support { + use super::{OsStr, OsString, Path, census_receipted_children, io, open_root_nofollow}; + + pub fn census_in_entry_order( + root_path: &Path, + entries: impl Iterator>, + unrecognized: K, + receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + ) -> io::Result> { + let root = open_root_nofollow(root_path)?; + census_receipted_children( + &root, + entries, + unrecognized, + receipt_for, + classify, + super::MAX_SEALED_CENSUS_INODES, + ) + } +} + /// Walk no-follow entries relative to the same held child, even after its /// old pathname has been renamed. The stack scales with directory depth. fn directory_bytes_from_held_dir( directory: Dir, category: K, totals: &mut std::collections::BTreeMap, - counted: &mut std::collections::HashMap<(u64, u64), (K, u64, u64)>, - max_shared_inodes: usize, + counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, + max_inodes: usize, ) -> io::Result { let mut total = 0_u64; // Hold only the current ancestry chain. Opening every sibling before @@ -500,57 +546,45 @@ fn directory_bytes_from_held_dir( #[cfg(unix)] { use cap_std::fs::MetadataExt; - if metadata.nlink() > 1 { - let key = (metadata.dev(), metadata.ino()); - if let Some((previous_kind, charged_bytes, remaining_links)) = counted.get_mut(&key) - { - // A lower-ordered category has stronger ownership. A - // superseded generation encountered first cannot claim - // hard-linked physical bytes from a later live head. - if category < *previous_kind { - let previous = totals.get_mut(&*previous_kind).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "hard-linked sealed bytes lack their prior category", - ) - })?; - previous.1 = previous.1.checked_sub(*charged_bytes).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "hard-linked sealed bytes exceed their prior category", - ) - })?; - total = total.saturating_add(*charged_bytes); - *previous_kind = category; - } - // An immutable hard-link group no longer needs its key - // after every filesystem-reported link has been seen. - // Retaining completed groups makes a store with many - // disjoint pairs exhaust the budget despite bounded - // concurrent deduplication demand. - *remaining_links -= 1; - if *remaining_links == 0 { - counted.remove(&key); - } - continue; + let key = (metadata.dev(), metadata.ino()); + if let Some((previous_kind, charged_bytes)) = counted.get_mut(&key) { + // A lower-ordered category has stronger ownership. A + // superseded generation encountered first cannot claim + // hard-linked physical bytes from a later live head. + if category < *previous_kind { + let previous = totals.get_mut(&*previous_kind).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes lack their prior category", + ) + })?; + previous.1 = previous.1.checked_sub(*charged_bytes).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes exceed their prior category", + ) + })?; + total = total.saturating_add(*charged_bytes); + *previous_kind = category; } - if counted.len() >= max_shared_inodes { - return Err(io::Error::new( - io::ErrorKind::OutOfMemory, - "sealed census shared-inode budget exhausted; coverage is partial", - )); - } - counted.try_reserve(1).map_err(|_| { - io::Error::new( - io::ErrorKind::OutOfMemory, - "sealed census shared-inode allocation failed; coverage is partial", - ) - })?; - counted.insert(key, (category, metadata.len(), metadata.nlink() - 1)); + continue; + } + if counted.len() >= max_inodes { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census inode budget exhausted; coverage is partial", + )); } + counted.try_reserve(1).map_err(|_| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census inode allocation failed; coverage is partial", + ) + })?; + counted.insert(key, (category, metadata.len())); } #[cfg(not(unix))] - let _ = (&counted, max_shared_inodes); + let _ = (&counted, max_inodes); total = total.saturating_add(metadata.len()); } Ok(total) @@ -1027,7 +1061,7 @@ mod tests { kind, &mut charged, &mut counted, - MAX_SEALED_CENSUS_SHARED_INODES, + MAX_SEALED_CENSUS_INODES, ) .unwrap(); let entry = charged.entry(kind).or_default(); @@ -1036,10 +1070,6 @@ mod tests { } assert_eq!(charged.get(&Kind::Superseded), Some(&(1, 3))); assert_eq!(charged.get(&Kind::Head), Some(&(1, 16))); - assert!( - counted.is_empty(), - "a fully observed hard-link group must free its deduplication slot" - ); // The production census also classifies receipt-bound children and // must report one physical copy without charging retired bytes twice. @@ -1060,12 +1090,12 @@ mod tests { assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); } - /// Completed hard-link groups need no further identity retention. A - /// bounded census must still cover many disjoint alias pairs whose total - /// exceeds its simultaneous unresolved-link budget. + /// Disjoint pairs still require evidence for the full walk: a publisher + /// can add another alias after any pair completes. A small budget must + /// refuse; a sufficient budget must count every pair exactly once. #[cfg(unix)] #[test] - fn complete_link_pairs_reuse_census_deduplication_budget() { + fn complete_link_pairs_require_retained_census_evidence() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("sealed"); let generation = root.join("generation"); @@ -1079,15 +1109,21 @@ mod tests { std::fs::hard_link(&original, column.join("alias")).unwrap(); } - // Each column is fully traversed before the next one is opened. - // Previously the four completed identities accumulated and this - // real filesystem walk failed at the second column's first link. + let refused = census_receipted_directory_bytes_bounded( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + 2, + ) + .unwrap_err(); + assert_eq!(refused.kind(), io::ErrorKind::OutOfMemory, "{refused}"); let census = census_receipted_directory_bytes_bounded( &root, false, |_| ("owner", 64), |_, receipt| receipt == Some(b"owned".as_slice()), - 1, + 5, ) .unwrap(); assert_eq!(census.get(&true), Some(&(1, 21))); @@ -1122,7 +1158,7 @@ mod tests { false, |_| ("owner", 64), |_, receipt| receipt == Some(b"owned".as_slice()), - 2, + 3, ) .unwrap_err(); assert_eq!(refused.kind(), io::ErrorKind::OutOfMemory, "{refused}"); From cc772b303d9616031a6bba2392fd2d3cf7225dde Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:41:43 -0800 Subject: [PATCH 068/111] test(graph): keep census scheduling private and preserve layered proof Refs #4616 Keep deterministic concurrent hard-link publication coverage in the filesystem census's private unit tests instead of exporting a feature-gated production API. Retain real layered-generation coverage through the public census, and move Unix-only metadata imports to module scope. --- .../tracedecay-graph-db/src/sealed_store.rs | 83 ++++---------- .../src/capability_dir.rs | 105 +++++++++++++----- 2 files changed, 101 insertions(+), 87 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 914d1dd955e..69f773ae1a3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3506,6 +3506,8 @@ mod receipt_budget_tests { #[cfg(test)] mod build_tests { use std::collections::{BTreeMap, BTreeSet}; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3617,27 +3619,23 @@ mod build_tests { .unwrap() } - /// Publish a real layered artifact between visits of the same census. - /// Both a formerly single link and an exhausted two-link group must - /// remain one physical charge owned by the newly visited head. + /// Real layered generations share one physical base. The production + /// census must attribute those hard links to the live head, regardless + /// of how many older layered generations already reference the base. #[cfg(unix)] #[test] - fn sealed_census_counts_concurrently_published_single_base_link_once() { - census_with_concurrent_layer_publication(0); + fn sealed_census_counts_published_single_base_link_once() { + census_with_layered_publication(0); } #[cfg(unix)] #[test] - fn sealed_census_counts_concurrently_published_completed_link_group_once() { - census_with_concurrent_layer_publication(1); + fn sealed_census_counts_published_multiple_base_links_once() { + census_with_layered_publication(1); } #[cfg(unix)] - fn census_with_concurrent_layer_publication(initial_layers: u64) { - use std::os::unix::fs::MetadataExt; - - use tracedecay_private_fs::capability_dir::test_support::census_in_entry_order; - + fn census_with_layered_publication(initial_layers: u64) { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("source.grafeo"); let database = open_source(&database_path); @@ -3689,18 +3687,12 @@ mod build_tests { if initial_layers == 1 { stores.push(publish("generation:child1")); } - let root = sealed_store_root(&database_path); let original = stores[0].directory.join(SEALED_STORE_DATABASE_FILE); let original_bytes = std::fs::read(&original).unwrap(); assert_eq!( std::fs::metadata(&original).unwrap().nlink(), 1 + initial_layers ); - let initial_names: Vec<_> = stores - .iter() - .map(|store| store.directory.file_name().unwrap().to_owned()) - .collect(); - let mut initial_names = initial_names.into_iter(); let receipts: Vec<_> = stores .iter() .map(|store| { @@ -3709,39 +3701,7 @@ mod build_tests { (path, bytes) }) .collect(); - let mut published = false; - // The entry source determines only scheduling: all receipt reads, - // capability opens and accounting use the production census core. - let entries = std::iter::from_fn(|| { - if let Some(name) = initial_names.next() { - return Some(Ok(name)); - } - if published { - return None; - } - let child = publish("generation:child2"); - let name = child.directory.file_name().unwrap().to_owned(); - stores.push(child); - published = true; - Some(Ok(name)) - }); - let totals = census_in_entry_order( - &root, - entries, - 2_u8, - |_| (SEALED_STORE_RECEIPT_FILE, MAX_SEALED_RECEIPT_BYTES), - |_, receipt| { - let receipt: super::SealedStoreReceiptV1 = - serde_json::from_slice(receipt.unwrap()).unwrap(); - if receipt.generation == "generation:child2" { - 0 - } else { - 1 - } - }, - ) - .unwrap(); - assert!(published); + stores.push(publish("generation:child2")); assert_eq!( std::fs::metadata(&original).unwrap().nlink(), 2 + initial_layers @@ -3764,26 +3724,27 @@ mod build_tests { .or_insert((kind, metadata.len())); } } + let census = census_sealed_store( + &database_path, + &BTreeSet::from(["generation:child2".to_owned()]), + ) + .unwrap(); for kind in [0, 1] { let bytes: u64 = physical .values() .filter(|(owner, _)| *owner == kind) .map(|(_, bytes)| bytes) .sum(); + let observed = if kind == 0 { + census.head_bytes + } else { + census.superseded_bytes + }; assert_eq!( - totals[&kind].1, bytes, + observed, bytes, "initial layers {initial_layers}, category {kind}" ); } - assert_eq!(totals[&0].0, 1); - assert_eq!(totals[&1].0, 1 + initial_layers); - let census = census_sealed_store( - &database_path, - &BTreeSet::from(["generation:child2".to_owned()]), - ) - .unwrap(); - assert_eq!(census.head_bytes, totals[&0].1); - assert_eq!(census.superseded_bytes, totals[&1].1); for store in stores { store.close().unwrap(); } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index f4c179e072b..c81e789311a 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -490,32 +490,6 @@ fn census_receipted_children( Ok(totals) } -/// Deterministic native scheduling for consumer publication regressions. -/// Receipt validation, held capabilities and inode accounting are shared with -/// the public census; only the filesystem's unspecified entry order is supplied. -#[cfg(feature = "test-helpers")] -pub mod test_support { - use super::{OsStr, OsString, Path, census_receipted_children, io, open_root_nofollow}; - - pub fn census_in_entry_order( - root_path: &Path, - entries: impl Iterator>, - unrecognized: K, - receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), - classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, - ) -> io::Result> { - let root = open_root_nofollow(root_path)?; - census_receipted_children( - &root, - entries, - unrecognized, - receipt_for, - classify, - super::MAX_SEALED_CENSUS_INODES, - ) - } -} - /// Walk no-follow entries relative to the same held child, even after its /// old pathname has been renamed. The stack scales with directory depth. fn directory_bytes_from_held_dir( @@ -801,6 +775,8 @@ pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(test)] mod replacement_tests { use std::io::Read; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; use super::*; @@ -1183,6 +1159,83 @@ mod tests { } } + /// A link group is not closed by its observed nlink count: a layered + /// publisher can add another alias after the census has visited every + /// old link. The head must inherit the one physical charge in both cases. + #[cfg(unix)] + #[test] + fn census_retains_inode_identity_across_later_link_publication() { + for initial_peer in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let old = root.join("old"); + std::fs::create_dir_all(&old).unwrap(); + std::fs::write(old.join("owner"), b"old").unwrap(); + let base = old.join("shared"); + std::fs::write(&base, b"rows").unwrap(); + let mut initial = vec![OsString::from("old")]; + if initial_peer { + let peer = root.join("old-peer"); + std::fs::create_dir(&peer).unwrap(); + std::fs::write(peer.join("owner"), b"old").unwrap(); + std::fs::hard_link(&base, peer.join("shared")).unwrap(); + initial.push(OsString::from("old-peer")); + } + assert_eq!( + std::fs::metadata(&base).unwrap().nlink(), + if initial_peer { 2 } else { 1 }, + ); + let root_dir = open_root_nofollow(&root).unwrap(); + let mut names = initial.into_iter(); + let mut published = false; + let entries = std::iter::from_fn(|| { + if let Some(name) = names.next() { + return Some(Ok(name)); + } + if published { + return None; + } + let head = root.join("head"); + std::fs::create_dir(&head).unwrap(); + std::fs::write(head.join("owner"), b"head").unwrap(); + std::fs::hard_link(&base, head.join("shared")).unwrap(); + published = true; + Some(Ok(OsString::from("head"))) + }); + let census = census_receipted_children( + &root_dir, + entries, + 2_u8, + |_| ("owner", 16), + |_, receipt| { + if receipt == Some(b"head".as_slice()) { + 0 + } else if receipt == Some(b"old".as_slice()) { + 1 + } else { + 2 + } + }, + MAX_SEALED_CENSUS_INODES, + ) + .unwrap(); + assert!(published, "the head must have been published mid-census"); + assert_eq!( + std::fs::metadata(&base).unwrap().nlink(), + if initial_peer { 3 } else { 2 }, + ); + assert_eq!(census.get(&0), Some(&(1, 8)), "live head owns shared bytes"); + assert_eq!( + census.get(&1), + Some(&(if initial_peer { 2 } else { 1 }, if initial_peer { 6 } else { 3 })), + "old generations retain only their own receipts" + ); + assert!(!census.contains_key(&2)); + assert_eq!(std::fs::read(root.join("head/shared")).unwrap(), b"rows"); + assert_eq!(std::fs::read(&base).unwrap(), b"rows"); + } + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 80977b3e6d6264e5a2dd6a48bebde2762176dcc2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:43:16 -0800 Subject: [PATCH 069/111] test(fs): import Unix inode metadata in census test module Refs #4616 Keep the private concurrent-publication regression compilable by placing the filesystem metadata trait in the module that owns that test, rather than the unrelated marker-replacement tests. --- crates/tracedecay-private-fs/src/capability_dir.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c81e789311a..734a209b8ff 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -775,8 +775,6 @@ pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(test)] mod replacement_tests { use std::io::Read; - #[cfg(unix)] - use std::os::unix::fs::MetadataExt; use super::*; @@ -931,6 +929,9 @@ mod replacement_tests { any(target_os = "macos", all(target_os = "linux", target_env = "gnu")) ))] mod tests { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + use cap_std::ambient_authority; use super::*; From 6ac18619cace62957cabbb11cee95b9cdf1e41b9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:45:07 -0800 Subject: [PATCH 070/111] test(graph): assert layered census generation counts Refs #4616 The real layered-generation regression must require a single live head, all retired siblings and no unrecognized artifacts, not only byte totals. --- crates/tracedecay-graph-db/src/sealed_store.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 69f773ae1a3..683a3e14f59 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3729,6 +3729,9 @@ mod build_tests { &BTreeSet::from(["generation:child2".to_owned()]), ) .unwrap(); + assert_eq!(census.head_count, 1); + assert_eq!(census.superseded_count, 1 + initial_layers); + assert_eq!(census.unrecognized_count, 0); for kind in [0, 1] { let bytes: u64 = physical .values() From 9e581d7483660080278bc21cae8edfedf1a8b4ac Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:42:02 -0800 Subject: [PATCH 071/111] style(graph): match Bazel rustfmt for sealed census test Refs #4616 Apply the exact hosted rustfmt 2024 hunk for the head/retired inode census regression, preserving behavior and assertions. --- crates/tracedecay-private-fs/src/capability_dir.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 734a209b8ff..2c5db0377b5 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -1228,7 +1228,10 @@ mod tests { assert_eq!(census.get(&0), Some(&(1, 8)), "live head owns shared bytes"); assert_eq!( census.get(&1), - Some(&(if initial_peer { 2 } else { 1 }, if initial_peer { 6 } else { 3 })), + Some(&( + if initial_peer { 2 } else { 1 }, + if initial_peer { 6 } else { 3 } + )), "old generations retain only their own receipts" ); assert!(!census.contains_key(&2)); From 527b1df25228abf27b217e3aca0b243591edc684 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:41:05 -0800 Subject: [PATCH 072/111] fix(retention): refuse unreadable sealed census children Refs #4616 An inaccessible real generation must not be silently reclassified as an unknown zero-byte directory during Doctor's sealed-store census. Propagate the held-child open refusal and exercise restoration plus complete recovery against the same durable payload. --- .../src/capability_dir.rs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 2c5db0377b5..b5ff66043a0 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -458,8 +458,11 @@ fn census_receipted_children( for name in entries { let name = name?; single_component(&name)?; + // A real generation that cannot be opened may still own gigabytes. + // Do not report complete zero-byte coverage by treating I/O failure + // as an unknown entry; preserve the failure for partial reporting. let child = if root.symlink_metadata(&name)?.is_dir() { - root.open_dir_nofollow(&name).ok() + Some(root.open_dir_nofollow(&name)?) } else { None }; @@ -1240,6 +1243,46 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn inaccessible_generation_refuses_complete_census_and_recovers() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("owner"), b"owned").unwrap(); + std::fs::write(generation.join("payload"), b"durable").unwrap(); + + let original = std::fs::metadata(&generation).unwrap().permissions(); + std::fs::set_permissions(&generation, std::fs::Permissions::from_mode(0o000)).unwrap(); + let refused = census_receipted_directory_bytes( + &root, + false, + |_| ("owner", 16), + |_, receipt| receipt == Some(b"owned".as_slice()), + ); + // Restore access before assertions so even a failed test can reclaim + // its fixture, and so recovery exercises the exact same stored bytes. + std::fs::set_permissions(&generation, original).unwrap(); + assert_eq!( + refused + .expect_err("an unreadable generation cannot be reported as zero") + .kind(), + io::ErrorKind::PermissionDenied, + ); + let complete = census_receipted_directory_bytes( + &root, + false, + |_| ("owner", 16), + |_, receipt| receipt == Some(b"owned".as_slice()), + ) + .unwrap(); + assert_eq!(complete.get(&true), Some(&(1, 12))); + assert_eq!(std::fs::read(generation.join("payload")).unwrap(), b"durable"); + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 942a95b305e89e20f6436966e030d46e3090a2a8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:43:25 -0800 Subject: [PATCH 073/111] fix(retention): preserve receipt I/O refusals in sealed census Refs #4616 Only absent, malformed and nonregular receipts become unrecognized entries. A permission or resource failure reading a real generation's receipt instead refuses complete coverage. Extend the native regression to restore permissions and prove the original bytes remain measurable. --- .../src/capability_dir.rs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index b5ff66043a0..ea40b265065 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -468,8 +468,26 @@ fn census_receipted_children( }; let (category, bytes) = if let Some(child) = child { let (receipt_name, maximum) = receipt_for(&name); + // Malformed, absent, or nonregular receipts identify unrecognized + // entries. An I/O refusal such as EACCES or EMFILE does not prove + // absence and must leave coverage explicitly unavailable. let receipt = - read_bounded_child_receipt(&child, OsStr::new(receipt_name), maximum).ok(); + match read_bounded_child_receipt(&child, OsStr::new(receipt_name), maximum) { + Ok(bytes) => Some(bytes), + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound + | io::ErrorKind::InvalidData + | io::ErrorKind::InvalidInput + | io::ErrorKind::IsADirectory + | io::ErrorKind::FilesystemLoop + ) => + { + None + } + Err(error) => return Err(error), + }; let category = classify(&name, receipt.as_deref()); let bytes = if category == unrecognized { 0 @@ -1272,6 +1290,27 @@ mod tests { .kind(), io::ErrorKind::PermissionDenied, ); + let owner_marker = generation.join("owner"); + let marker_permissions = std::fs::metadata(&owner_marker).unwrap().permissions(); + std::fs::set_permissions( + &owner_marker, + std::fs::Permissions::from_mode(0o000), + ) + .unwrap(); + let unreadable_marker = census_receipted_directory_bytes( + &root, + false, + |_| ("owner", 16), + |_, receipt| receipt == Some(b"owned".as_slice()), + ); + std::fs::set_permissions(&owner_marker, marker_permissions).unwrap(); + assert_eq!( + unreadable_marker + .expect_err("unreadable receipt cannot silently become unknown") + .kind(), + io::ErrorKind::PermissionDenied, + ); + let complete = census_receipted_directory_bytes( &root, false, From eea5e3f83db8dae859e83d74e3c347bead5fa1ba Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:44:10 -0800 Subject: [PATCH 074/111] style(retention): normalize denied-receipt fixture formatting Refs #4616 Keep the native permission-refusal assertion and fixture behavior unchanged while matching the repository's short-call layout. --- crates/tracedecay-private-fs/src/capability_dir.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index ea40b265065..6bf51f9a774 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -1292,11 +1292,7 @@ mod tests { ); let owner_marker = generation.join("owner"); let marker_permissions = std::fs::metadata(&owner_marker).unwrap().permissions(); - std::fs::set_permissions( - &owner_marker, - std::fs::Permissions::from_mode(0o000), - ) - .unwrap(); + std::fs::set_permissions(&owner_marker, std::fs::Permissions::from_mode(0o000)).unwrap(); let unreadable_marker = census_receipted_directory_bytes( &root, false, From 3bc8783eea1ea843b73dd57bdf4e5359ee1a9571 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:02:21 -0800 Subject: [PATCH 075/111] style(private-fs): format retained payload assertion Apply the pinned formatter's wrapping to the retained payload check in the receipt census regression. --- crates/tracedecay-private-fs/src/capability_dir.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 6bf51f9a774..81f7d8d408b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -1315,7 +1315,10 @@ mod tests { ) .unwrap(); assert_eq!(complete.get(&true), Some(&(1, 12))); - assert_eq!(std::fs::read(generation.join("payload")).unwrap(), b"durable"); + assert_eq!( + std::fs::read(generation.join("payload")).unwrap(), + b"durable" + ); } #[test] From 0a098e15c5d201a48a2cd9f76195fc7835d60ccf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:40:37 -0800 Subject: [PATCH 076/111] fix(retention): classify socket receipts through held metadata Refs #4616 Preserve typed I/O refusals for regular receipts, but classify a Unix socket's ENXIO as nonregular only after a held-directory no-follow stat. Exercise a real socket beside an intact sealed head without hiding its bytes or deleting foreign data. --- .../src/capability_dir.rs | 66 ++++++++++++++++++- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 81f7d8d408b..ee0a49938d2 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -738,7 +738,7 @@ fn read_bounded_child_receipt( name: &OsStr, maximum: usize, ) -> io::Result> { - let name = CString::new(name.as_bytes()) + let c_name = CString::new(name.as_bytes()) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL"))?; // SAFETY: the held directory fd is valid; name is a NUL-terminated // sibling component. O_NOFOLLOW prevents link traversal and O_NONBLOCK @@ -746,12 +746,27 @@ fn read_bounded_child_receipt( let descriptor = unsafe { libc::openat( directory.as_raw_fd(), - name.as_ptr(), + c_name.as_ptr(), libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, ) }; if descriptor < 0 { - return Err(io::Error::last_os_error()); + let error = io::Error::last_os_error(); + // Unix sockets may reject O_RDONLY with ENXIO before fstat can + // establish their type. Only downgrade that error when the SAME + // held directory still names a nonregular receipt. A regular file's + // I/O failure must remain a refusal of complete census coverage. + if error.raw_os_error() == Some(libc::ENXIO) + && directory + .symlink_metadata(name) + .is_ok_and(|metadata| !metadata.file_type().is_file()) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "sealed receipt is not a regular file", + )); + } + return Err(error); } // SAFETY: openat returned a new owned fd and this File is its sole owner. let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; @@ -1261,6 +1276,51 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn socket_receipt_is_unrecognized_without_hiding_the_verified_head() { + use std::os::unix::net::UnixListener; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("s"); + let head = root.join("head"); + let unknown = root.join("unknown"); + std::fs::create_dir_all(&head).unwrap(); + std::fs::create_dir(&unknown).unwrap(); + std::fs::write(head.join("owner"), b"owned").unwrap(); + std::fs::write(head.join("payload"), b"durable").unwrap(); + std::fs::write(unknown.join("sentinel"), b"preserve").unwrap(); + let _socket = UnixListener::bind(unknown.join("owner")).unwrap(); + + // On Unix, openat(O_RDONLY | O_NONBLOCK) on a socket may fail with + // ENXIO before a regular-file check. The held directory's metadata + // proves that entry is nonregular, not an unreadable owned receipt. + let receipt = read_bounded_child_receipt(&open(&unknown), OsStr::new("owner"), 64); + assert_eq!(receipt.unwrap_err().kind(), io::ErrorKind::InvalidData); + + let totals = census_receipted_directory_bytes( + &root, + 2_u8, + |_| ("owner", 64), + |_, receipt| { + if receipt == Some(b"owned".as_slice()) { + 0 + } else { + 2 + } + }, + ) + .unwrap(); + assert_eq!(totals.get(&0), Some(&(1, 12))); + assert_eq!(totals.get(&2), Some(&(1, 0))); + assert_eq!(std::fs::read(head.join("payload")).unwrap(), b"durable"); + assert_eq!( + std::fs::read(unknown.join("sentinel")).unwrap(), + b"preserve" + ); + assert!(unknown.join("owner").exists()); + } + #[cfg(unix)] #[test] fn inaccessible_generation_refuses_complete_census_and_recovers() { From 78e8d3edd860653880e8bcab57e663e1d7298c44 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:41:14 -0800 Subject: [PATCH 077/111] style(retention): scope socket test import to module Refs #4616 Keep native regression imports at module scope per repository conventions. --- crates/tracedecay-private-fs/src/capability_dir.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index ee0a49938d2..03829960590 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -967,6 +967,8 @@ mod replacement_tests { mod tests { #[cfg(unix)] use std::os::unix::fs::MetadataExt; + #[cfg(unix)] + use std::os::unix::net::UnixListener; use cap_std::ambient_authority; @@ -1279,8 +1281,6 @@ mod tests { #[cfg(unix)] #[test] fn socket_receipt_is_unrecognized_without_hiding_the_verified_head() { - use std::os::unix::net::UnixListener; - let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("s"); let head = root.join("head"); From c59bcfe5eed0709facde52ad567ab70ed983d41c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:21:15 -0800 Subject: [PATCH 078/111] fix(private-fs): normalize no-follow receipt refusals Use the existing stable ELOOP-to-InvalidInput normalization for held-directory receipt reads instead of naming the unstable FilesystemLoop error kind. Keep permission and resource errors visible as incomplete census coverage. Add a real symlink receipt regression proving a link to otherwise valid owner bytes stays unrecognized while the verified head and all stored bytes remain. Refs #4616. --- .../src/capability_dir.rs | 48 +++++++++++++++++-- .../tracedecay-private-fs/src/framed_log.rs | 2 +- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 03829960590..cd82d0cf55b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -23,6 +23,9 @@ use cap_fs_ext::OpenOptionsMaybeDirExt; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, File, OpenOptions}; +#[cfg(unix)] +use crate::framed_log::normalize_no_follow_error; + /// How many times a create-or-open that reports the entry missing is asked /// again before the answer is believed. xnu bounds its own retry of the same /// condition at ten; downstream measurements found one extra look always @@ -481,7 +484,6 @@ fn census_receipted_children( | io::ErrorKind::InvalidData | io::ErrorKind::InvalidInput | io::ErrorKind::IsADirectory - | io::ErrorKind::FilesystemLoop ) => { None @@ -766,7 +768,7 @@ fn read_bounded_child_receipt( "sealed receipt is not a regular file", )); } - return Err(error); + return Err(normalize_no_follow_error(error)); } // SAFETY: openat returned a new owned fd and this File is its sole owner. let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; @@ -966,7 +968,7 @@ mod replacement_tests { ))] mod tests { #[cfg(unix)] - use std::os::unix::fs::MetadataExt; + use std::os::unix::fs::{MetadataExt, symlink}; #[cfg(unix)] use std::os::unix::net::UnixListener; @@ -1278,6 +1280,46 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn symlink_receipt_is_unrecognized_without_hiding_the_verified_head() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let head = root.join("head"); + let unknown = root.join("unknown"); + std::fs::create_dir_all(&head).unwrap(); + std::fs::create_dir(&unknown).unwrap(); + std::fs::write(head.join("owner"), b"owned").unwrap(); + std::fs::write(head.join("payload"), b"durable").unwrap(); + std::fs::write(unknown.join("sentinel"), b"preserve").unwrap(); + symlink(head.join("owner"), unknown.join("owner")).unwrap(); + + let receipt = read_bounded_child_receipt(&open(&unknown), OsStr::new("owner"), 64); + assert_eq!(receipt.unwrap_err().kind(), io::ErrorKind::InvalidInput); + + let totals = census_receipted_directory_bytes( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + ) + .unwrap(); + assert_eq!(totals.get(&true), Some(&(1, 12))); + assert_eq!(totals.get(&false), Some(&(1, 0))); + assert_eq!(std::fs::read(head.join("payload")).unwrap(), b"durable"); + assert_eq!(std::fs::read(head.join("owner")).unwrap(), b"owned"); + assert_eq!( + std::fs::read(unknown.join("sentinel")).unwrap(), + b"preserve" + ); + assert!( + std::fs::symlink_metadata(unknown.join("owner")) + .unwrap() + .file_type() + .is_symlink() + ); + } + #[cfg(unix)] #[test] fn socket_receipt_is_unrecognized_without_hiding_the_verified_head() { diff --git a/crates/tracedecay-private-fs/src/framed_log.rs b/crates/tracedecay-private-fs/src/framed_log.rs index 408bc6780e9..19c621b4100 100644 --- a/crates/tracedecay-private-fs/src/framed_log.rs +++ b/crates/tracedecay-private-fs/src/framed_log.rs @@ -253,7 +253,7 @@ fn open_no_follow(path: &Path) -> io::Result { options.open(path).map_err(normalize_no_follow_error) } -fn normalize_no_follow_error(error: io::Error) -> io::Error { +pub(crate) fn normalize_no_follow_error(error: io::Error) -> io::Error { #[cfg(unix)] if error.raw_os_error() == Some(libc::ELOOP) { return io::Error::new(io::ErrorKind::InvalidInput, "path is a symbolic link"); From 6088b7c5898d7ad0f6720074f85428d04f4f8ae3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:36:50 -0800 Subject: [PATCH 079/111] fix(doctor): scope sealed census to projection identities Track sealed generations by namespace, projection, and generation within the registered shard. Keep native retirement dependencies tied to their exact replay sequence. Add real journal and sealed-container regressions covering colliding generation names, namespace and shard isolation, retained dependencies, and unchanged bytes. --- .../src/doctor_kernel.rs | 45 ++++--- .../src/doctor_kernel/tests.rs | 118 ++++++++++++++++++ .../tracedecay-graph-db/src/sealed_store.rs | 107 ++++++++++++++-- .../sealed_store.rs | 2 +- 4 files changed, 244 insertions(+), 28 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel.rs b/crates/tracedecay-daemon-service/src/doctor_kernel.rs index 167442e0a0f..60f21ece85b 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel.rs @@ -42,6 +42,9 @@ use tracedecay_contracts::{ }; use tracedecay_domain::CodeGenerationId; use tracedecay_global_db::{GlobalDbNativeIntegrationStore, RegisteredGlobalDb}; +use tracedecay_graph_db::{ + GraphGenerationId, GraphNamespace, GraphProjectionId, GraphProjectionIdentity, +}; use crate::DaemonFeedbackRuntimeRegistrar; use tracedecay_maintenance::telemetry::GuardedStoreTelemetryPort; @@ -817,35 +820,45 @@ pub async fn collect_code_generation_retention_findings( storage_family_read(vec![finding]) } -/// The journaled generation ids whose sealed artifacts are still live in the -/// project graph store: each projection's verified head, every publication +/// The journaled projection/generation identities whose sealed artifacts remain +/// live in the project graph store: each projection's verified head, every publication /// newer than its head (pending), and every generation an active replay /// depends on. A sealed directory outside this set is one the ordinary /// retirement pass reclaims; `None` when the journal cannot be read (an /// absent table on a store that never published, a lock, a corrupt row). async fn live_sealed_generations( graph: &tracedecay_runtime_core::db::Database, -) -> Option> { +) -> Option> { + let shard = serde_json::to_string(&graph.registered_binding().shard_id).ok()?; let mut rows = graph .read_connection() .query( - "SELECT replay.generation + "SELECT replay.namespace, replay.projection, replay.generation FROM graph_publication_replay_v1 AS replay LEFT JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id AND head.namespace = replay.namespace AND head.projection = replay.projection - WHERE head.replay_sequence IS NULL - OR replay.sequence >= head.replay_sequence + WHERE replay.shard_id = ?1 + AND (head.replay_sequence IS NULL + OR replay.sequence >= head.replay_sequence) UNION - SELECT generation FROM graph_publication_replay_dependencies_v1", - (), + SELECT namespace, projection, generation + FROM graph_publication_replay_dependencies_v1 + WHERE shard_id = ?1", + (shard,), ) .await .ok()?; let mut heads = BTreeSet::new(); while let Some(row) = rows.next().await.ok()? { - heads.insert(row.get::(0).ok()?); + heads.insert(( + GraphProjectionIdentity::new( + GraphNamespace::new(row.get::(0).ok()?).ok()?, + GraphProjectionId::new(row.get::(1).ok()?).ok()?, + ), + GraphGenerationId::new(row.get::(2).ok()?).ok()?, + )); } Some(heads) } @@ -856,21 +869,25 @@ async fn live_sealed_generations( /// that retirement has not yet reached. `None` when the journal cannot be /// read. async fn deferred_native_retirements(graph: &tracedecay_runtime_core::db::Database) -> Option { + let shard = serde_json::to_string(&graph.registered_binding().shard_id).ok()?; let mut rows = graph .read_connection() .query( - "SELECT (SELECT COUNT(*) FROM graph_publication_replay_tombstones_v1) + "SELECT (SELECT COUNT(*) FROM graph_publication_replay_tombstones_v1 + WHERE shard_id = ?1) + (SELECT COUNT(*) FROM graph_publication_replay_v1 AS replay JOIN graph_verified_heads_v1 AS head ON head.shard_id = replay.shard_id AND head.namespace = replay.namespace AND head.projection = replay.projection - WHERE replay.sequence < head.replay_sequence - AND replay.generation NOT IN ( - SELECT generation FROM graph_publication_replay_dependencies_v1 + WHERE replay.shard_id = ?1 + AND replay.sequence < head.replay_sequence + AND NOT EXISTS ( + SELECT 1 FROM graph_publication_replay_dependencies_v1 AS dependency + WHERE dependency.dependency_replay_sequence = replay.sequence ))", - (), + (shard,), ) .await .ok()?; diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs index b25e2bf0df4..c340b696dd1 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs @@ -15,9 +15,127 @@ use tracedecay_contracts::doctor::{ ObservabilityReadV1, ObservabilityStateV1, host_integration_finding, }; use tracedecay_graph_db::GraphDbCorruptionCensusV1; +use tracedecay_runtime_core::db::{Database, DatabaseAuthority, TestDatabaseRuntimeMode}; +use tracedecay_rusqlite_runtime::repository::GRAPH_PUBLICATION_SCHEMA_V1; use super::*; +#[tokio::test] +async fn sealed_journal_liveness_and_retirement_preserve_projection_identity() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("graph.db"); + let authority = + DatabaseAuthority::acquire_test(&path, "doctor sealed journal fixture").unwrap(); + let (database, _) = + Database::publish_test_runtime(&path, &authority, TestDatabaseRuntimeMode::Initialize) + .await + .unwrap(); + database + .execute_write_batch( + "install graph publication schema", + GRAPH_PUBLICATION_SCHEMA_V1, + ) + .await + .unwrap(); + let shard = serde_json::to_string(&database.registered_binding().shard_id).unwrap(); + let mut foreign = database.registered_binding().shard_id.clone(); + foreign.profile_id = tracedecay_domain::UserProfileId::new("profile.foreign").unwrap(); + let foreign = serde_json::to_string(&foreign).unwrap(); + let digest = format!("sha256:{}", "0".repeat(64)); + for (sequence, owner, namespace, projection, generation) in [ + (1_i64, &shard, "project", "code", "generation:shared"), + (2, &shard, "project", "sessions", "generation:shared"), + (3, &shard, "project", "sessions", "generation:current"), + (4, &shard, "other-project", "code", "generation:shared"), + (5, &shard, "other-project", "code", "generation:current"), + (6, &foreign, "project", "sessions", "generation:shared"), + ] { + database + .execute_write( + "publish graph journal fixture", + "INSERT INTO graph_publication_replay_v1 + (sequence, shard_id, namespace, projection, generation, idempotency_key, + input_digest, dependency_generation_closure_digest, direct_dependency_bytes, + expected_recovered_digest, canonical_replay_source_digest, canonical_replay_source) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7, 2, ?7, ?7, X'7b7d')", + (sequence, owner.as_str(), namespace, projection, generation, + format!("publication:{sequence}"), digest.as_str()), + ) + .await + .unwrap(); + } + for sequence in [1_i64, 3, 5, 6] { + database + .execute_write( + "install graph journal head", + "INSERT INTO graph_verified_heads_v1 + (shard_id, namespace, projection, replay_sequence, recovered_digest) + SELECT shard_id, namespace, projection, sequence, expected_recovered_digest + FROM graph_publication_replay_v1 WHERE sequence = ?1", + (sequence,), + ) + .await + .unwrap(); + } + database + .execute_write( + "retain exact graph dependency", + "INSERT INTO graph_publication_replay_dependencies_v1 + (owner_replay_sequence, ordinal, dependency_replay_sequence, + shard_id, namespace, projection, generation) + SELECT 3, 0, sequence, shard_id, namespace, projection, generation + FROM graph_publication_replay_v1 WHERE sequence = 1", + (), + ) + .await + .unwrap(); + let identity = |namespace: &str, projection: &str, generation: &str| { + ( + GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new(projection).unwrap(), + ), + GraphGenerationId::new(generation).unwrap(), + ) + }; + let mut expected = BTreeSet::from([ + identity("project", "code", "generation:shared"), + identity("project", "sessions", "generation:current"), + identity("other-project", "code", "generation:current"), + ]); + assert_eq!( + live_sealed_generations(&database).await, + Some(expected.clone()) + ); + assert_eq!(deferred_native_retirements(&database).await, Some(2)); + + database + .execute_write( + "retain one superseded projection as a dependency", + "INSERT INTO graph_publication_replay_dependencies_v1 + (owner_replay_sequence, ordinal, dependency_replay_sequence, + shard_id, namespace, projection, generation) + SELECT 3, 1, sequence, shard_id, namespace, projection, generation + FROM graph_publication_replay_v1 WHERE sequence = 4", + (), + ) + .await + .unwrap(); + expected.insert(identity("other-project", "code", "generation:shared")); + assert_eq!(live_sealed_generations(&database).await, Some(expected)); + assert_eq!(deferred_native_retirements(&database).await, Some(1)); + assert_eq!( + database + .query_scalar_i64( + "preserved replays", + "SELECT COUNT(*) FROM graph_publication_replay_v1" + ) + .await + .unwrap(), + 6, + ); +} + #[test] fn configuration_read_from_pin_absent_on_cold_cache() { let missing: Result = diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 683a3e14f59..895b3298c36 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -81,10 +81,10 @@ use crate::state::{ }; use crate::{ GraphCommit, GraphDb, GraphDbError, GraphDbLocation, GraphDbOpenOptions, GraphDurability, - GraphEntity, GraphFormatVersion, GraphGenerationManifest, GraphGenerationManifestIdentity, - GraphGenerationRowSpill, GraphGenerationRows, GraphNamespace, GraphProjectionId, - GraphProjectionIdentity, GraphRelation, GraphWatermark, GraphWriteBatch, NeverCancelled, - SourceGeneration, SpilledGraphGeneration, + GraphEntity, GraphFormatVersion, GraphGenerationId, GraphGenerationManifest, + GraphGenerationManifestIdentity, GraphGenerationRowSpill, GraphGenerationRows, GraphNamespace, + GraphProjectionId, GraphProjectionIdentity, GraphRelation, GraphWatermark, GraphWriteBatch, + NeverCancelled, SourceGeneration, SpilledGraphGeneration, }; /// A sealed generation opened without the shared staging database. @@ -581,8 +581,8 @@ pub struct SealedStoreCensusV1 { pub unrecognized_count: u64, } -/// Measure the sealed root beside `database_path` against the generation ids -/// of the store's current verified heads (as journaled: `:`). +/// Measure the sealed root beside `database_path` against the exact projection +/// and generation identities that remain live in that store's journal. /// /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the @@ -597,7 +597,7 @@ enum SealedCensusEntry { pub fn census_sealed_store( database_path: &Path, - head_generations: &std::collections::BTreeSet, + head_generations: &std::collections::BTreeSet<(GraphProjectionIdentity, GraphGenerationId)>, ) -> std::io::Result { let root = sealed_store_root(database_path); let current_producer = row_spill_owner().to_string(); @@ -644,7 +644,17 @@ pub fn census_sealed_store( { return SealedCensusEntry::Unrecognized; } - if head_generations.contains(&receipt.generation) { + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace), + GraphProjectionId::new(receipt.projection), + GraphGenerationId::new(receipt.generation), + ) else { + return SealedCensusEntry::Unrecognized; + }; + if head_generations.contains(&( + GraphProjectionIdentity::new(namespace, projection), + generation, + )) { SealedCensusEntry::Head } else { SealedCensusEntry::Superseded @@ -3442,8 +3452,14 @@ mod receipt_budget_tests { let oldest = oldest.unwrap(); std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); - let census = - census_sealed_store(&database, &BTreeSet::from(["generation:g69".to_owned()])).unwrap(); + let heads = BTreeSet::from([( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + GraphGenerationId::new("generation:g69").unwrap(), + )]); + let census = census_sealed_store(&database, &heads).unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 69); assert_eq!(census.unrecognized_count, 0); @@ -3493,8 +3509,11 @@ mod receipt_budget_tests { .set_len(5 * 1024 * 1024 * 1024) .unwrap(); - let census = - census_sealed_store(&database, &BTreeSet::from([generation.to_owned()])).unwrap(); + let census = census_sealed_store( + &database, + &BTreeSet::from([(locator.projection, locator.generation)]), + ) + .unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 0); assert_eq!(census.unrecognized_count, 1); @@ -3634,6 +3653,65 @@ mod build_tests { census_with_layered_publication(1); } + #[test] + fn sealed_census_scopes_equal_generation_names_to_their_projection() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let mut heads = BTreeSet::new(); + let mut artifacts = Vec::new(); + let mut expected_head_bytes = 0; + let mut expected_superseded_bytes = 0; + for (namespace, projection, generation, live) in [ + ("project", "code", "generation:shared", true), + ("project", "sessions", "generation:shared", false), + ("project", "sessions", "generation:current", true), + ("other-project", "code", "generation:shared", false), + ("other-project", "code", "generation:current", true), + ] { + let mut manifest = manifest(2, 0); + manifest.projection = GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new(projection).unwrap(), + ); + manifest.generation = GraphGenerationId::new(generation).unwrap(); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + let directory = store.directory.clone(); + drop(store); + if live { + heads.insert((identity.projection, identity.generation)); + } + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + let bytes = std::fs::read(&path).unwrap(); + if live { + expected_head_bytes += bytes.len() as u64; + } else { + expected_superseded_bytes += bytes.len() as u64; + } + artifacts.push((path, bytes)); + } + } + + let census = census_sealed_store(&database_path, &heads).unwrap(); + assert_eq!(census.head_count, 3); + assert_eq!(census.superseded_count, 2); + assert_eq!(census.unrecognized_count, 0); + assert_eq!(census.head_bytes, expected_head_bytes); + assert_eq!(census.superseded_bytes, expected_superseded_bytes); + for (path, bytes) in artifacts { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + } + #[cfg(unix)] fn census_with_layered_publication(initial_layers: u64) { let temp = tempfile::tempdir().unwrap(); @@ -3726,7 +3804,10 @@ mod build_tests { } let census = census_sealed_store( &database_path, - &BTreeSet::from(["generation:child2".to_owned()]), + &BTreeSet::from([( + identity.projection, + GraphGenerationId::new("generation:child2").unwrap(), + )]), ) .unwrap(); assert_eq!(census.head_count, 1); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index ec402598a0f..db1b11d4309 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1445,7 +1445,7 @@ fn sealed_census_separates_heads_and_preserves_unproven_staging() { std::fs::create_dir_all(&unreadable).unwrap(); std::fs::write(unreadable.join("generation.grafeo"), b"no receipt").unwrap(); - let heads = std::collections::BTreeSet::from(["census-g2".to_owned()]); + let heads = BTreeSet::from([(identity, GraphGenerationId::new("census-g2").unwrap())]); let census = tracedecay_graph_db::census_sealed_store(&support::graph_path(temp.path()), &heads) .unwrap(); From 2eb7085a9a4b0c9ac3b442b76e3bb87627f2dc34 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:37:44 -0800 Subject: [PATCH 080/111] test(global-db): remove redundant runtime reopens Reuse the schema fixture runtime for schema-shape assertions and drop the redundant pre-cancellation reopen. Preserve the cancellation recovery and actual reopen checks. --- .../src/registered/workflow_schema_tests.rs | 6 ++-- .../src/tests/schema_install_cancellation.rs | 33 +++++++++---------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 3dd339b74f0..6db136334eb 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -980,10 +980,10 @@ mod published_artifact_ceiling { ); if state == before { cancelled_published += 1; + let (reopened, _) = converge_cancelled_at(&store.path, usize::MAX).await; + reopened.unwrap(); + assert_eq!(store.state(), completed, "reopen after poll {cancel_at}"); } - let (reopened, _) = converge_cancelled_at(&store.path, usize::MAX).await; - reopened.unwrap(); - assert_eq!(store.state(), completed, "reopen after poll {cancel_at}"); } } } diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 92cf28bd455..f40a96a4cbe 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -64,23 +64,19 @@ impl EmptyStore { .unwrap() } - async fn schema_objects(&self) -> Vec { - let (owner, _runtime, _retirement) = self.publish().await.into_parts(); - let database = owner.issue_lease().unwrap(); - let mut rows = database - .read_connection() - .query( + /// Schema assertions read SQLite directly instead of reopening the registered runtime. + fn schema_objects(&self) -> Vec { + let connection = rusqlite::Connection::open(&self.path).unwrap(); + connection + .prepare( "SELECT type || ':' || name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", - (), ) - .await - .unwrap(); - let mut objects = Vec::new(); - while let Some(row) = rows.next().await.unwrap() { - objects.push(row.get::(0).unwrap()); - } - objects + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>>() + .unwrap() } /// Runs daemon admission, cancelling its token just before the @@ -115,7 +111,7 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { let reference = EmptyStore::create(); let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); - let installed = reference.schema_objects().await; + let installed = reference.schema_objects(); assert!( installed.contains(&"table:code_projects".to_owned()), "a completed attach installs the registered schema: {installed:?}" @@ -129,9 +125,10 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { let store = EmptyStore::create(); // Publishing the runtime installs its writer ledger before admission. // Cancellation must preserve exactly that pre-admission schema. - let before_admission = store.schema_objects().await; + drop(store.publish().await); + let before_admission = store.schema_objects(); let (outcome, _) = store.attach_cancelled_at(cancel_at).await; - let objects = store.schema_objects().await; + let objects = store.schema_objects(); match outcome { Err(error) => { assert!( @@ -162,7 +159,7 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { reopened.unwrap_or_else(|error| { panic!("reopen after cancel at poll {cancel_at}/{total_polls} failed: {error:?}") }); - assert_eq!(store.schema_objects().await, installed); + assert_eq!(store.schema_objects(), installed); } assert!( cancelled.contains(&1), From 4108e7d75bb6b3d264b907be2f820d7a691142d5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:09:05 -0800 Subject: [PATCH 081/111] test(graph): remove obsolete sealed builder import The test module retained build_sealed_container after its callers moved to build_sealed_container_with_work. Both Clippy and the Linux test prebuild failed to resolve the old name. Remove that unused import from the existing test module. --- crates/tracedecay-graph-db/src/sealed_store.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index d8e79c99124..787a1e0d2fa 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3690,10 +3690,10 @@ mod build_tests { MAX_SEALED_RECEIPT_BYTES, NativeCompactEncodeBoundaryV1, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedCompactRows, SealedRowSource, SealedStagingOwnerV1, build_or_open_sealed_store, build_or_open_sealed_store_with_work, - build_sealed_container, build_sealed_container_with_work, census_sealed_store, - initial_sealed_staging_owner, persist_sealed_store_check, push_manifest_rows, - retire_sealed_directory_checked, sealed_artifact_database_options, sealed_copy_proof, - sealed_generation_directory, sealed_store_root, sweep_abandoned_sealed_staging, + build_sealed_container_with_work, census_sealed_store, initial_sealed_staging_owner, + persist_sealed_store_check, push_manifest_rows, retire_sealed_directory_checked, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, sweep_abandoned_sealed_staging, }; use tracedecay_store::runtime::{GraphNativeWorkKindV1, GraphNativeWorkV1}; From 2f05c8d074c8f2f0751086f40e8cc12ee032f8d3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:41:32 -0800 Subject: [PATCH 082/111] fix(fs): classify macOS socket receipts as nonregular --- .../tracedecay-private-fs/src/capability_dir.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index cd82d0cf55b..45808d65678 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -754,11 +754,11 @@ fn read_bounded_child_receipt( }; if descriptor < 0 { let error = io::Error::last_os_error(); - // Unix sockets may reject O_RDONLY with ENXIO before fstat can - // establish their type. Only downgrade that error when the SAME - // held directory still names a nonregular receipt. A regular file's - // I/O failure must remain a refusal of complete census coverage. - if error.raw_os_error() == Some(libc::ENXIO) + // Unix socket opens return ENXIO on Linux or EOPNOTSUPP on macOS. + // Classify that refusal as a nonregular receipt only when the held + // directory still names a nonregular entry. A regular file's I/O + // failure must remain a refusal of complete census coverage. + if matches!(error.raw_os_error(), Some(libc::ENXIO | libc::EOPNOTSUPP)) && directory .symlink_metadata(name) .is_ok_and(|metadata| !metadata.file_type().is_file()) @@ -1334,9 +1334,8 @@ mod tests { std::fs::write(unknown.join("sentinel"), b"preserve").unwrap(); let _socket = UnixListener::bind(unknown.join("owner")).unwrap(); - // On Unix, openat(O_RDONLY | O_NONBLOCK) on a socket may fail with - // ENXIO before a regular-file check. The held directory's metadata - // proves that entry is nonregular, not an unreadable owned receipt. + // openat on a socket fails with ENXIO or EOPNOTSUPP before fstat. + // The held directory's metadata proves the entry is nonregular. let receipt = read_bounded_child_receipt(&open(&unknown), OsStr::new("owner"), 64); assert_eq!(receipt.unwrap_err().kind(), io::ErrorKind::InvalidData); From 40ab61b08398e8f0fae687c4c91650cb1bcbfc00 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:41:34 -0800 Subject: [PATCH 083/111] test(doctor): register schema authority before journal fixture --- crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs index c340b696dd1..e407ca50581 100644 --- a/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs +++ b/crates/tracedecay-daemon-service/src/doctor_kernel/tests.rs @@ -22,6 +22,7 @@ use super::*; #[tokio::test] async fn sealed_journal_liveness_and_retirement_preserve_projection_identity() { + tracedecay_global_db::register_registered_schema_installer(); let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("graph.db"); let authority = From 9491d6f4ba8b5d89d1c1fbf7cf60ee276e18935e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:08:19 -0800 Subject: [PATCH 084/111] fix(graph): own sealed staging writes through held directory A staging directory replaced by a symlink after the owner marker could redirect the live producer's container, receipt, and auxiliary writes into a foreign tree. The builder now creates the staging entry exclusively through a held directory capability, writes the marker, receipt, spool, and base links relative to that handle, and resolves the one path-based container writer from the handle rather than the substitutable name. Installation compares the held directory identity before and after the no-replace rename, so a substituted entry is refused and its bytes are preserved. --- .../tracedecay-graph-db/src/sealed_layer.rs | 21 +- .../tracedecay-graph-db/src/sealed_store.rs | 247 +++++++++++++----- .../src/capability_dir.rs | 228 ++++++++++++++++ 3 files changed, 418 insertions(+), 78 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_layer.rs b/crates/tracedecay-graph-db/src/sealed_layer.rs index dd07b4806b0..33853ade58a 100644 --- a/crates/tracedecay-graph-db/src/sealed_layer.rs +++ b/crates/tracedecay-graph-db/src/sealed_layer.rs @@ -34,6 +34,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::HeldDirectory; use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ @@ -861,7 +862,9 @@ impl LayeredGraphGeneration { /// Moves the pinned base links into `staging` and records the hidden /// rows, completing a layered artifact directory beside its container. - pub(crate) fn install_base_files(&self, staging: &Path) -> Result<(), GraphDbError> { + /// Every write is relative to the held staging directory, so a replaced + /// staging pathname cannot redirect a base link or hidden-row file. + pub(crate) fn install_base_files(&self, staging: &HeldDirectory) -> Result<(), GraphDbError> { let pinned = self.delta.directory(); for file in [ LAYERED_BASE_CONTAINER_FILE, @@ -870,21 +873,23 @@ impl LayeredGraphGeneration { LAYERED_BASE_ENTITY_ROWS_FILE, LAYERED_BASE_ENTITY_ROW_OFFSETS_FILE, ] { - std::fs::hard_link(pinned.join(file), staging.join(file)) + staging + .hard_link_from(&pinned.join(file), std::ffi::OsStr::new(file)) .map_err(|error| layered_io("base link", error))?; } let container = Path::new(LAYERED_BASE_CONTAINER_FILE); - match std::fs::copy( - pinned.join(marker_path(container)), - staging.join(marker_path(container)), - ) { - Ok(_) => {} + let marker = marker_path(container); + match std::fs::read(pinned.join(&marker)) { + Ok(bytes) => staging + .write_new_file(marker.as_os_str(), &bytes) + .map_err(|error| layered_io("base marker link", error))?, Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(layered_io("base marker link", error)), } let encoded = serde_json::to_vec(&self.hidden) .map_err(|error| GraphDbError::unavailable(format!("hidden rows encode: {error}")))?; - std::fs::write(staging.join(LAYERED_HIDDEN_FILE), encoded) + staging + .write_new_file(std::ffi::OsStr::new(LAYERED_HIDDEN_FILE), &encoded) .map_err(|error| layered_io("hidden rows write", error)) } } diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 787a1e0d2fa..7c3a34e3cd8 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,7 +44,6 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; -use std::io::Write; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -55,7 +54,9 @@ use grafeo_core::graph::compact::IncrementalCompactStoreBuilder; use grafeo_engine::GrafeoDB; use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nofollow; +use tracedecay_private_fs::capability_dir::{ + HeldDirectory, install_held_child_directory_noreplace, +}; use tracedecay_store::runtime::{ GraphNativeWorkKindV1, GraphNativeWorkV1, GraphRecoveredGenerationDigestV1, }; @@ -1063,10 +1064,14 @@ struct PreparedSealedRelation { impl SealedCompactRows { /// Spools pushed column values to an anonymous file in `staging`, so - /// the build holds the row topology rather than every value. - fn new(staging: &Path) -> Result { - let spool = tempfile::tempfile_in(staging) - .map_err(|error| sealed_store_io_failure("column spool", error))?; + /// the build holds the row topology rather than every value. The spool is + /// created and unlinked through the held directory, so a replaced staging + /// pathname cannot place it in a foreign tree. + fn new(staging: &HeldDirectory) -> Result { + let spool = staging + .anonymous_file() + .map_err(|error| sealed_store_io_failure("column spool", error))? + .into_std(); Ok(Self { builder: IncrementalCompactStoreBuilder::spooling_to(spool), next_node: 0, @@ -2008,27 +2013,19 @@ pub(crate) enum SealedRowSource<'a> { /// concurrent writer can adopt the winner only after the caller reopens and /// verifies its digest; existence alone is not a new durable write. fn install_staged_sealed_store( - staging: &Path, - directory: &Path, + staging: &HeldDirectory, + root: &Path, + staging_name: &std::ffi::OsStr, + artifact_name: &std::ffi::OsStr, locator: &GenerationLocator, expected_digest: &str, ) -> Result { - let root = staging - .parent() - .ok_or_else(|| GraphDbError::unavailable("sealed staging root missing"))?; - if directory.parent() != Some(root) { - return Err(GraphDbError::unavailable("sealed install roots differ")); - } - let staging_name = staging - .file_name() - .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; - let artifact_name = directory - .file_name() - .ok_or_else(|| GraphDbError::unavailable("sealed artifact name missing"))?; - match install_child_directory_noreplace_nofollow(root, staging_name, artifact_name) { + let staging_path = root.join(staging_name); + let directory = root.join(artifact_name); + match install_held_child_directory_noreplace(root, staging, staging_name, artifact_name) { Ok(()) => Ok(true), Err(error) => { - discard_failed_sealed_staging(staging, locator, expected_digest); + discard_failed_sealed_staging(&staging_path, locator, expected_digest); if directory.exists() { // A winner may have installed the same generation. The // caller still requires a full identity/digest reopen proof. @@ -2111,13 +2108,14 @@ fn build_or_open_sealed_store_with_work( retire_sealed_directory_checked(&directory, &locator, Some(expected.as_str()))?; } } - let staging = root.join(format!( + let staging_name = std::ffi::OsString::from(format!( ".staging-{}", directory .file_name() .and_then(|name| name.to_str()) .unwrap_or("sealed") )); + let staging = root.join(&staging_name); // create_dir_all may accept a symlinked sealed root, so verify the actual // root before creating the staging entry or writing a container. std::fs::create_dir_all(&root) @@ -2133,8 +2131,10 @@ fn build_or_open_sealed_store_with_work( } // A deterministic staging name is not proof of ownership. An earlier // interrupted build or a foreign directory must not be deleted on retry. - // create_dir is exclusive, so an occupied name fails closed. - std::fs::create_dir(&staging) + // Exclusive creation refuses an occupied name, and every byte below is + // written through the held handle or the path it resolves to, so replacing + // the staging name cannot redirect a producer write into a foreign tree. + let staging_directory = HeldDirectory::create_exclusive(&root, &staging_name) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; let owner = SealedStagingOwnerV1 { producer: row_spill_owner().to_string(), @@ -2144,39 +2144,36 @@ fn build_or_open_sealed_store_with_work( .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; // Exclusive creation refuses a pre-existing marker. Flush before writing // rows so a killed producer leaves verifiable deletion authority. - let marker_path = staging.join(SEALED_STAGING_OWNER_FILE); - let mut marker = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&marker_path) - .map_err(|error| sealed_store_io_failure("staging owner create failed", error))?; - marker - .write_all(&owner_bytes) - .and_then(|()| marker.sync_all()) + staging_directory + .write_new_file( + std::ffi::OsStr::new(SEALED_STAGING_OWNER_FILE), + &owner_bytes, + ) .map_err(|error| sealed_store_io_failure("staging owner persist failed", error))?; - drop(marker); - let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check, observe) - .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; + let receipt = + stage_sealed_artifact(rows, identity, expected, &staging_directory, check, observe) + .inspect_err(|_| { + discard_failed_sealed_staging(&staging, &locator, expected.as_str()) + })?; let encoded = serde_json::to_vec_pretty(&receipt) .map_err(|error| GraphDbError::unavailable(format!("sealed receipt encode: {error}"))) .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; - let mut receipt_file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(staging.join(SEALED_STORE_RECEIPT_FILE)) - .map_err(|error| sealed_store_io_failure("receipt create failed", error)) - .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; - let receipt_write = receipt_file - .write_all(&encoded) - .and_then(|()| receipt_file.sync_all()); - drop(receipt_file); - receipt_write + staging_directory + .write_new_file(std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), &encoded) .map_err(|error| sealed_store_io_failure("receipt persist failed", error)) .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; // Only the writer that installed this directory reports a new durable // segment; losing the race still requires the winner's digest proof. - let installed_here = - install_staged_sealed_store(&staging, &directory, &locator, expected.as_str())?; + let installed_here = install_staged_sealed_store( + &staging_directory, + &root, + &staging_name, + directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact name missing"))?, + &locator, + expected.as_str(), + )?; let match_result = { let _span = tracing::trace_span!("code_index.seal.verify").entered(); open_sealed_store_checked( @@ -2237,7 +2234,7 @@ fn stage_sealed_artifact( rows: SealedRowSource<'_>, identity: &GraphGenerationManifestIdentity, expected: &GraphRecoveredGenerationDigestV1, - staging: &Path, + staging: &HeldDirectory, check: &dyn Fn() -> Result<(), GraphDbError>, observe: Option<&dyn Fn(GraphNativeWorkV1)>, ) -> Result { @@ -2275,7 +2272,8 @@ fn stage_sealed_artifact( crate::row_index::ENTITY_ROW_OFFSETS_FILE, ), ] { - std::fs::hard_link(source, staging.join(name)) + staging + .hard_link_from(source, std::ffi::OsStr::new(name)) .map_err(|error| sealed_store_io_failure("base file link failed", error))?; } } @@ -2316,7 +2314,7 @@ fn stage_sealed_artifact( fn build_sealed_container_with_work( rows: SealedRowSource<'_>, identity: &GraphGenerationManifestIdentity, - staging: &Path, + staging: &HeldDirectory, check: &dyn Fn() -> Result<(), GraphDbError>, observe: Option<&dyn Fn(GraphNativeWorkV1)>, ) -> Result<(usize, usize), GraphDbError> { @@ -2424,8 +2422,16 @@ fn build_sealed_container_with_work( check()?; { let _span = tracing::trace_span!("code_index.seal.write").entered(); - sealed.write_container(&staging.join(SEALED_STORE_DATABASE_FILE)) - }?; + // The container writer takes a pathname. Resolve the held staging + // handle to the path of the directory it owns, so a staging name + // replaced by a link or a foreign directory cannot redirect the + // container into that tree. + let container_path = staging + .path() + .map_err(|error| sealed_store_io_failure("container staging path failed", error))? + .join(SEALED_STORE_DATABASE_FILE); + sealed.write_container(&container_path)?; + } Ok((entity_count, relation_count)) } @@ -3687,16 +3693,27 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - MAX_SEALED_RECEIPT_BYTES, NativeCompactEncodeBoundaryV1, SEALED_STAGING_OWNER_FILE, - SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedCompactRows, SealedRowSource, - SealedStagingOwnerV1, build_or_open_sealed_store, build_or_open_sealed_store_with_work, - build_sealed_container_with_work, census_sealed_store, initial_sealed_staging_owner, - persist_sealed_store_check, push_manifest_rows, retire_sealed_directory_checked, - sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, - sealed_store_root, sweep_abandoned_sealed_staging, + HeldDirectory, MAX_SEALED_RECEIPT_BYTES, NativeCompactEncodeBoundaryV1, + SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, + SealedCompactRows, SealedRowSource, SealedStagingOwnerV1, build_or_open_sealed_store, + build_or_open_sealed_store_with_work, build_sealed_container_with_work, + census_sealed_store, initial_sealed_staging_owner, persist_sealed_store_check, + push_manifest_rows, retire_sealed_directory_checked, sealed_artifact_database_options, + sealed_copy_proof, sealed_generation_directory, sealed_store_root, + sweep_abandoned_sealed_staging, }; use tracedecay_store::runtime::{GraphNativeWorkKindV1, GraphNativeWorkV1}; + /// Opens a test-owned directory as the held capability the production + /// builder uses, so the tests exercise the same write authority. + fn held(directory: &std::path::Path) -> HeldDirectory { + HeldDirectory::open_nofollow( + directory.parent().expect("held directory parent"), + directory.file_name().expect("held directory name"), + ) + .expect("open the test staging directory") + } + use crate::location::PersistentGraphStoreState; use crate::{ GraphDb, GraphDbError, GraphDbLocation, GraphDbOpenOptions, GraphDbOwner, GraphDurability, @@ -4091,7 +4108,7 @@ mod build_tests { build_sealed_container_with_work( SealedRowSource::Manifest(&manifest), &identity, - staging.path(), + &held(staging.path()), check, None, ) @@ -4126,7 +4143,7 @@ mod build_tests { let pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); let counts = pool .install(|| { - let mut writer = SealedCompactRows::new(staging.path())?; + let mut writer = SealedCompactRows::new(&held(staging.path()))?; push_manifest_rows( &fixture, &identity, @@ -4172,7 +4189,7 @@ mod build_tests { let calls = AtomicUsize::new(0); let staging = tempfile::tempdir().unwrap(); let result = pool.install(|| { - let mut writer = SealedCompactRows::new(staging.path())?; + let mut writer = SealedCompactRows::new(&held(staging.path()))?; push_manifest_rows( &fixture, &identity, @@ -4211,7 +4228,7 @@ mod build_tests { build_sealed_container_with_work( SealedRowSource::Manifest(&manifest), &identity, - staging.path(), + &held(staging.path()), check, None, ) @@ -4294,7 +4311,7 @@ mod build_tests { build_sealed_container_with_work( SealedRowSource::Manifest(&manifest), &identity, - staging.path(), + &held(staging.path()), check, None, ) @@ -5007,6 +5024,86 @@ mod build_tests { assert_eq!(store.recovered_digest(), expected.as_str()); } + /// A staging directory replaced by a symlink after the owner marker is + /// written must not redirect the live producer's container write into the + /// foreign target. The builder keeps running after the substitution, so + /// this exercises writer ownership rather than deletion authority. + #[cfg(unix)] + #[test] + fn live_seal_refuses_a_staging_symlink_substituted_after_the_owner_marker() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let moved = root.join("original-stage"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"foreign").unwrap(); + let swapped = std::cell::Cell::new(false); + let substitute = || { + if !swapped.replace(true) { + assert!( + staging.join(SEALED_STAGING_OWNER_FILE).is_file(), + "the owner marker must exist before the substitution" + ); + std::fs::rename(&staging, &moved).unwrap(); + std::os::unix::fs::symlink(&foreign, &staging).unwrap(); + } + Ok(()) + }; + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &substitute, + ); + assert!( + result.is_err(), + "a staging symlink substituted mid-build must refuse the seal" + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"foreign", + "the foreign directory's own bytes must survive" + ); + assert!( + !foreign.join(SEALED_STORE_DATABASE_FILE).exists(), + "the live producer must not write its container into the foreign target" + ); + assert!( + !foreign.join(SEALED_STORE_RECEIPT_FILE).exists(), + "the live producer must not write its receipt into the foreign target" + ); + assert!( + std::fs::symlink_metadata(&staging) + .unwrap() + .file_type() + .is_symlink(), + "the substituted link is not ours to remove" + ); + assert!(!directory.exists(), "no generation may be installed"); + + std::fs::remove_file(&staging).unwrap(); + std::fs::remove_dir_all(&moved).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .expect("a genuine staging directory must still seal"); + assert_eq!(store.recovered_digest(), expected.as_str()); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves @@ -5363,9 +5460,19 @@ mod build_tests { identity.projection.clone(), identity.generation.clone(), ); - let installed_by_loser = - super::install_staged_sealed_store(&staging, &directory, &locator, expected.as_str()) - .expect("candidate exists after the install collision"); + let staging_name = staging.file_name().unwrap().to_owned(); + let artifact_name = directory.file_name().unwrap().to_owned(); + let held_staging = + HeldDirectory::open_nofollow(&root, &staging_name).expect("open the losing staging"); + let installed_by_loser = super::install_staged_sealed_store( + &held_staging, + &root, + &staging_name, + &artifact_name, + &locator, + expected.as_str(), + ) + .expect("candidate exists after the install collision"); assert!( !installed_by_loser, "the losing writer committed no new native segment" diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 45808d65678..be4c577afc3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -394,6 +394,234 @@ pub fn install_child_directory_noreplace_nofollow( rename_noreplace(&root, staging_name, &root, generation_name) } +/// A directory held open for a producer's writes. Every mutation is relative +/// to this handle, so replacing the pathname cannot redirect bytes into a +/// foreign tree. The one path-based writer resolves [`Self::path`] from the +/// handle, which names the directory this handle owns rather than the +/// substitutable entry name. +pub struct HeldDirectory { + directory: Dir, +} + +/// Durable identity of a held directory, compared across a rename. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DirectoryIdentity { + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, + #[cfg(windows)] + volume_serial_number: u32, + #[cfg(windows)] + file_index: u64, +} + +impl HeldDirectory { + /// Exclusively creates `name` beneath `root_path` and holds it. An + /// occupied name is never adopted: the caller owns only the entry it + /// created. + pub fn create_exclusive(root_path: &Path, name: &OsStr) -> io::Result { + single_component(name)?; + let root = open_root_nofollow(root_path)?; + root.create_dir(name)?; + root.open_dir_nofollow(name) + .map(|directory| Self { directory }) + } + + /// Opens an existing child of `root_path` without following a link. + pub fn open_nofollow(root_path: &Path, name: &OsStr) -> io::Result { + single_component(name)?; + let root = open_root_nofollow(root_path)?; + root.open_dir_nofollow(name) + .map(|directory| Self { directory }) + } + + /// Writes a new file exclusively, relative to this handle. + pub fn write_new_file(&self, name: &OsStr, contents: &[u8]) -> io::Result<()> { + single_component(name)?; + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + let mut file = self.directory.open_with(name, &options)?; + file.write_all(contents).and_then(|()| file.sync_all()) + } + + /// Creates an anonymous file in this directory: created exclusively and + /// unlinked immediately, so it leaves no entry behind. + pub fn anonymous_file(&self) -> io::Result { + static NEXT_SPOOL: AtomicU64 = AtomicU64::new(0); + let (name, file) = (0..8) + .find_map(|_| { + let sequence = NEXT_SPOOL.fetch_add(1, Ordering::Relaxed); + let name = OsString::from(format!( + ".spool-{}-{:016x}-{sequence:016x}", + std::process::id(), + crate::process_instance_nonce(), + )); + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + match self.directory.open_with(&name, &options) { + Ok(file) => Some(Ok((name, file))), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => None, + Err(error) => Some(Err(error)), + } + }) + .transpose()? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::AlreadyExists, + "could not exclusively stage an anonymous spool", + ) + })?; + self.directory.remove_file(&name)?; + Ok(file) + } + + /// Hard-links `source` to `name` relative to this handle. The source's + /// parent is opened without following a link, so neither side is an + /// ambient pathname. + pub fn hard_link_from(&self, source: &Path, name: &OsStr) -> io::Result<()> { + single_component(name)?; + let source_parent = source.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "hard-link source has no parent", + ) + })?; + let source_name = source.file_name().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "hard-link source has no name") + })?; + let parent = open_root_nofollow(source_parent)?; + parent.hard_link(source_name, &self.directory, name) + } + + /// Resolves this held directory to a path that names the same inode, so a + /// path-based writer can create a child without going through a + /// substitutable ancestor name. + pub fn path(&self) -> io::Result { + held_directory_path(&self.directory) + } + + /// The directory's own durable identity. + pub fn identity(&self) -> io::Result { + directory_identity(&self.directory) + } +} + +/// Install a held staging directory as the generation artifact without +/// replacing an occupied destination. The entry at `staging_name` must still +/// be the held directory, and the installed entry must remain that same +/// directory, so a replacement between the producer's writes and publication +/// is refused rather than installed. +pub fn install_held_child_directory_noreplace( + root_path: &Path, + staging: &HeldDirectory, + staging_name: &OsStr, + generation_name: &OsStr, +) -> io::Result<()> { + single_component(staging_name)?; + single_component(generation_name)?; + let root = open_root_nofollow(root_path)?; + let entry = root.open_dir_nofollow(staging_name)?; + let entry_identity = directory_identity(&entry)?; + drop(entry); + if entry_identity != staging.identity()? { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sealed staging entry is not the held producer directory", + )); + } + rename_noreplace(&root, staging_name, &root, generation_name)?; + let installed = root.open_dir_nofollow(generation_name)?; + let installed_identity = directory_identity(&installed)?; + drop(installed); + if installed_identity != staging.identity()? { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sealed generation entry is not the held producer directory", + )); + } + Ok(()) +} + +#[cfg(unix)] +fn directory_identity(directory: &Dir) -> io::Result { + use cap_std::fs::MetadataExt; + let metadata = directory.dir_metadata()?; + Ok(DirectoryIdentity { + device: metadata.dev(), + inode: metadata.ino(), + }) +} + +#[cfg(windows)] +fn directory_identity(directory: &Dir) -> io::Result { + let information = crate::windows_file::information(directory)?; + Ok(DirectoryIdentity { + volume_serial_number: information.volume_serial_number, + file_index: information.file_index, + }) +} + +#[cfg(not(any(unix, windows)))] +fn directory_identity(_directory: &Dir) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "directory identity is unavailable on this platform", + )) +} + +#[cfg(windows)] +fn held_directory_path(directory: &Dir) -> io::Result { + dir_path(directory) +} + +#[cfg(target_os = "linux")] +fn held_directory_path(directory: &Dir) -> io::Result { + use std::os::fd::AsRawFd; + Ok(std::path::PathBuf::from(format!( + "/proc/self/fd/{}", + directory.as_raw_fd() + ))) +} + +#[cfg(target_os = "macos")] +fn held_directory_path(directory: &Dir) -> io::Result { + use std::os::fd::AsRawFd; + use std::os::unix::ffi::OsStringExt; + let descriptor = directory.as_raw_fd(); + let mut buffer = vec![0_u8; libc::PATH_MAX as usize]; + // SAFETY: `buffer` is writable for `PATH_MAX` bytes and `descriptor` is an + // open directory descriptor owned by `directory`. + let result = unsafe { + libc::fcntl( + descriptor, + libc::F_GETPATH, + buffer.as_mut_ptr().cast::(), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + let length = buffer.iter().position(|byte| *byte == 0).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "held directory path was not NUL-terminated", + ) + })?; + buffer.truncate(length); + Ok(std::path::PathBuf::from(std::ffi::OsString::from_vec( + buffer, + ))) +} + +#[cfg(not(any(windows, target_os = "linux", target_os = "macos")))] +fn held_directory_path(_directory: &Dir) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "held directory path resolution is unavailable on this platform", + )) +} + /// Bound deduplication metadata independently of the size of the store. /// The Doctor caller reports partial coverage instead of trusting a census /// whose inode inventory would otherwise grow without limit. From ddb3a6ae5d0a9cf9b0efb64da8b32c7683b529f4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:15:06 -0800 Subject: [PATCH 085/111] test(fs): cover held directory producer ownership Exercise exclusive staging creation, anonymous spool cleanup, relative hard links, pathname substitution, and identity-checked installation through the held directory capability. --- .../src/capability_dir.rs | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index be4c577afc3..7b17368b1fb 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -1706,6 +1706,132 @@ mod tests { ); } + #[test] + fn held_directory_create_exclusive_refuses_an_occupied_name() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + std::fs::create_dir(&root).unwrap(); + let name = OsStr::new(".staging-generation"); + let held = HeldDirectory::create_exclusive(&root, name).unwrap(); + held.write_new_file(OsStr::new("sealed.json"), b"owned") + .unwrap(); + + let error = HeldDirectory::create_exclusive(&root, name) + .err() + .expect("an occupied staging name is never adopted"); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!( + std::fs::read(root.join(name).join("sealed.json")).unwrap(), + b"owned" + ); + } + + #[test] + fn held_directory_anonymous_file_leaves_no_entry() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + std::fs::create_dir(&root).unwrap(); + let held = HeldDirectory::create_exclusive(&root, OsStr::new("staging")).unwrap(); + + let mut spool = held.anonymous_file().unwrap(); + spool.write_all(b"column values").unwrap(); + spool.sync_all().unwrap(); + + assert_eq!( + std::fs::read_dir(root.join("staging")).unwrap().count(), + 0, + "an anonymous spool must not leave a directory entry" + ); + } + + #[test] + fn held_directory_hard_link_from_links_relative_to_the_handle() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let source = temp.path().join("source"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&source).unwrap(); + std::fs::write(source.join("payload"), b"linked").unwrap(); + let held = HeldDirectory::create_exclusive(&root, OsStr::new("staging")).unwrap(); + + held.hard_link_from(&source.join("payload"), OsStr::new("payload")) + .unwrap(); + + assert_eq!( + std::fs::read(root.join("staging").join("payload")).unwrap(), + b"linked" + ); + } + + /// A held directory keeps writing into the directory it created after the + /// entry name is replaced by a link to a foreign tree. The path-based + /// writer resolves the handle, not the substitutable name. + #[cfg(unix)] + #[test] + fn held_directory_writes_survive_a_pathname_substitution() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"foreign").unwrap(); + let name = OsStr::new(".staging-generation"); + let held = HeldDirectory::create_exclusive(&root, name).unwrap(); + held.write_new_file(OsStr::new("owner.json"), b"owned") + .unwrap(); + + let moved = root.join("moved-generation"); + std::fs::rename(root.join(name), &moved).unwrap(); + symlink(&foreign, root.join(name)).unwrap(); + + held.write_new_file(OsStr::new("sealed.json"), b"receipt") + .unwrap(); + let resolved = std::fs::canonicalize(held.path().unwrap()).unwrap(); + assert_eq!( + resolved.file_name(), + Some(OsStr::new("moved-generation")), + "the held handle must resolve to the directory it owns" + ); + assert_eq!( + std::fs::read(moved.join("sealed.json")).unwrap(), + b"receipt" + ); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"foreign"); + assert!( + !foreign.join("sealed.json").exists(), + "a substituted name must not receive the producer's bytes" + ); + } + + /// Installation compares the held identity before and after the rename, so + /// a staging entry replaced by a link is refused and the foreign tree is + /// preserved. + #[cfg(unix)] + #[test] + fn held_directory_install_refuses_a_substituted_staging_entry() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&root).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"foreign").unwrap(); + let staging_name = OsStr::new(".staging-generation"); + let generation_name = OsStr::new("generation"); + let held = HeldDirectory::create_exclusive(&root, staging_name).unwrap(); + held.write_new_file(OsStr::new("sealed.json"), b"owned") + .unwrap(); + + let moved = root.join("moved-generation"); + std::fs::rename(root.join(staging_name), &moved).unwrap(); + symlink(&foreign, root.join(staging_name)).unwrap(); + + install_held_child_directory_noreplace(&root, &held, staging_name, generation_name) + .expect_err("a substituted staging entry must not be installed"); + assert!(!root.join(generation_name).exists()); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"foreign"); + assert_eq!(std::fs::read(moved.join("sealed.json")).unwrap(), b"owned"); + } + #[test] fn rename_noreplace_moves_between_parents_and_refuses_occupied_destinations() { let root = tempfile::tempdir().expect("create rename fixture"); From 9d195fb4540cc7e9ffe062c6876d3d1f7ec3da4a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:04:59 -0800 Subject: [PATCH 086/111] fix(graph): refuse substituted sealed installs --- .../tracedecay-graph-db/src/sealed_store.rs | 67 ++++++++++++++++++- .../src/capability_dir.rs | 50 ++++---------- 2 files changed, 76 insertions(+), 41 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7c3a34e3cd8..d168116c485 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2026,9 +2026,12 @@ fn install_staged_sealed_store( Ok(()) => Ok(true), Err(error) => { discard_failed_sealed_staging(&staging_path, locator, expected_digest); - if directory.exists() { - // A winner may have installed the same generation. The - // caller still requires a full identity/digest reopen proof. + if error.kind() == std::io::ErrorKind::AlreadyExists && directory.exists() { + // Only an occupied destination is a concurrent-writer + // collision. An identity mismatch, including one detected + // after the rename, must never be reclassified as adoption. + // The caller still requires a full identity/digest reopen + // proof for a genuine collision. Ok(false) } else { Err(sealed_store_io_failure("artifact install failed", error)) @@ -5488,6 +5491,64 @@ mod build_tests { let _ = reopened.database().close(); } + #[test] + fn sealed_install_identity_mismatch_is_not_an_occupied_destination_collision() { + let temp = tempfile::tempdir().expect("persistent sealed graph root"); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(3, 2); + let identity = manifest.identity(); + let expected = manifest + .expected_recovered_digest(&|| Ok(())) + .expect("generation digest"); + let root = sealed_store_root(&database_path); + std::fs::create_dir_all(&root).expect("sealed root"); + let directory = sealed_generation_directory( + &root, + &identity.physical_namespace().expect("physical namespace"), + ); + std::fs::create_dir(&directory).expect("occupied generation"); + std::fs::write(directory.join("sentinel"), b"installed").unwrap(); + + let staging_name = std::ffi::OsString::from(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let staging = root.join(&staging_name); + let held_staging = HeldDirectory::create_exclusive(&root, &staging_name) + .expect("create the producer staging directory"); + held_staging + .write_new_file(std::ffi::OsStr::new("owned"), b"producer") + .expect("write producer evidence"); + let moved = root.join("moved-producer"); + std::fs::rename(&staging, &moved).expect("move the producer aside"); + std::fs::create_dir(&staging).expect("substitute a foreign real directory"); + std::fs::write(staging.join("sentinel"), b"foreign").unwrap(); + + let artifact_name = directory.file_name().unwrap().to_owned(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let error = super::install_staged_sealed_store( + &held_staging, + &root, + &staging_name, + &artifact_name, + &locator, + expected.as_str(), + ) + .expect_err("an identity mismatch is not a concurrent-writer collision"); + assert!(error.to_string().contains("artifact install failed")); + + assert_eq!(std::fs::read(staging.join("sentinel")).unwrap(), b"foreign"); + assert_eq!(std::fs::read(moved.join("owned")).unwrap(), b"producer"); + assert_eq!( + std::fs::read(directory.join("sentinel")).unwrap(), + b"installed" + ); + assert!(!directory.join("owned").exists()); + } + /// Two builds of the same staged rows write byte-identical section /// payloads: the build order is the recovered-digest order, and the /// compact serializer is deterministic, so every byte past the Grafeo diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 7b17368b1fb..ad0abfd895e 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -372,28 +372,6 @@ fn open_root_nofollow(root_path: &Path) -> io::Result { parent.open_dir_nofollow(root_name) } -/// Install a staged generation without replacing even an empty destination -/// directory. The final component of the sealed root and the staging entry -/// must be real directories; a link or an occupied generation is not a -/// recoverable staging collision. -/// -/// The caller still owns the responsibility for the identity of the store -/// ancestors, which this pathname API cannot establish. -pub fn install_child_directory_noreplace_nofollow( - root_path: &Path, - staging_name: &OsStr, - generation_name: &OsStr, -) -> io::Result<()> { - single_component(staging_name)?; - single_component(generation_name)?; - let root = open_root_nofollow(root_path)?; - let staging = root.open_dir_nofollow(staging_name)?; - // Reject a linked staging entry, then close the extra directory handle - // before publication for platforms that disallow renaming open children. - drop(staging); - rename_noreplace(&root, staging_name, &root, generation_name) -} - /// A directory held open for a producer's writes. Every mutation is relative /// to this handle, so replacing the pathname cannot redirect bytes into a /// foreign tree. The one path-based writer resolves [`Self::path`] from the @@ -1659,11 +1637,13 @@ mod tests { let installed_name = OsStr::new("generation"); let staged = root.join(staged_name); let installed = root.join(installed_name); - std::fs::create_dir(&staged).unwrap(); - std::fs::write(staged.join("sealed.json"), b"built").unwrap(); + let held = HeldDirectory::create_exclusive(&root, staged_name).unwrap(); + held.write_new_file(OsStr::new("sealed.json"), b"built") + .unwrap(); std::fs::create_dir(&installed).unwrap(); - let result = install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); + let result = + install_held_child_directory_noreplace(&root, &held, staged_name, installed_name); assert_eq!(result.unwrap_err().kind(), io::ErrorKind::AlreadyExists); assert!( installed.is_dir(), @@ -1672,7 +1652,7 @@ mod tests { assert_eq!(std::fs::read(staged.join("sealed.json")).unwrap(), b"built"); std::fs::remove_dir(&installed).unwrap(); - install_child_directory_noreplace_nofollow(&root, staged_name, installed_name).unwrap(); + install_held_child_directory_noreplace(&root, &held, staged_name, installed_name).unwrap(); assert!(!staged.exists()); assert_eq!( std::fs::read(installed.join("sealed.json")).unwrap(), @@ -1682,28 +1662,22 @@ mod tests { #[cfg(unix)] #[test] - fn staged_generation_install_refuses_linked_root_and_staging_entries() { + fn held_directory_create_refuses_a_linked_root() { let temp = tempfile::tempdir().unwrap(); let foreign = temp.path().join("foreign"); - let real = temp.path().join("real"); let link = temp.path().join("link"); std::fs::create_dir(&foreign).unwrap(); - std::fs::create_dir(&real).unwrap(); std::fs::write(foreign.join("sentinel"), b"preserve").unwrap(); - std::fs::create_dir(foreign.join("staged")).unwrap(); std::os::unix::fs::symlink(&foreign, &link).unwrap(); - let stage = OsStr::new("staged"); - let published = OsStr::new("installed"); - assert!(install_child_directory_noreplace_nofollow(&link, stage, published).is_err()); - - std::os::unix::fs::symlink(&foreign, real.join("staged")).unwrap(); - assert!(install_child_directory_noreplace_nofollow(&real, stage, published).is_err()); - assert!(!foreign.join("installed").exists()); - assert!(!real.join("installed").exists()); + + HeldDirectory::create_exclusive(&link, OsStr::new("staged")) + .err() + .expect("a linked sealed root grants no creation authority"); assert_eq!( std::fs::read(foreign.join("sentinel")).unwrap(), b"preserve" ); + assert!(!foreign.join("staged").exists()); } #[test] From 7e05ab8d19940ae739552134211a57e831916cfd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:08:05 -0800 Subject: [PATCH 087/111] docs(fs): bound held path ownership claims --- .../tracedecay-graph-db/src/sealed_store.rs | 4 ++- .../src/capability_dir.rs | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index d168116c485..23f42ce2d53 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2428,7 +2428,9 @@ fn build_sealed_container_with_work( // The container writer takes a pathname. Resolve the held staging // handle to the path of the directory it owns, so a staging name // replaced by a link or a foreign directory cannot redirect the - // container into that tree. + // container into that tree. Linux retains that authority through + // /proc/self/fd; macOS and Windows still require a Grafeo open-file + // writer to close substitution of the resolved pathname itself. let container_path = staging .path() .map_err(|error| sealed_store_io_failure("container staging path failed", error))? diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index ad0abfd895e..cf8944cc036 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -372,11 +372,11 @@ fn open_root_nofollow(root_path: &Path) -> io::Result { parent.open_dir_nofollow(root_name) } -/// A directory held open for a producer's writes. Every mutation is relative -/// to this handle, so replacing the pathname cannot redirect bytes into a -/// foreign tree. The one path-based writer resolves [`Self::path`] from the -/// handle, which names the directory this handle owns rather than the -/// substitutable entry name. +/// A directory held open for a producer's writes. This type's own mutations +/// are relative to the handle, so replacing the pathname cannot redirect them +/// into a foreign tree. [`Self::path`] also lets a path-based writer start +/// from the held directory rather than the substitutable entry name; its +/// platform-specific authority is documented on that method. pub struct HeldDirectory { directory: Dir, } @@ -395,9 +395,11 @@ pub struct DirectoryIdentity { } impl HeldDirectory { - /// Exclusively creates `name` beneath `root_path` and holds it. An - /// occupied name is never adopted: the caller owns only the entry it - /// created. + /// Exclusively creates `name` beneath `root_path`, refusing an entry that + /// occupied the name at creation, then opens the entry without following + /// a link and holds it. Directory creation and acquisition are separate + /// operating-system operations; a concurrent substitution between them + /// remains outside this primitive's authority. pub fn create_exclusive(root_path: &Path, name: &OsStr) -> io::Result { single_component(name)?; let root = open_root_nofollow(root_path)?; @@ -472,9 +474,14 @@ impl HeldDirectory { parent.hard_link(source_name, &self.directory, name) } - /// Resolves this held directory to a path that names the same inode, so a - /// path-based writer can create a child without going through a - /// substitutable ancestor name. + /// Resolves this held directory to a path that names the same inode at + /// resolution time, so a path-based writer need not start from the + /// substitutable entry name. On Linux the `/proc/self/fd` path continues + /// to follow the held inode. `F_GETPATH` on macOS and + /// `GetFinalPathNameByHandleW` on Windows return a pathname snapshot: + /// substituting that resolved path before the consumer opens it remains a + /// race. Those platforms need an open-file consumer API for end-to-end + /// producer authority. pub fn path(&self) -> io::Result { held_directory_path(&self.directory) } From 5c75f4f876d71aac1bd97a630cc6abec12c59e93 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:51:10 -0800 Subject: [PATCH 088/111] test(dashboard): use the production runtime scheduler --- crates/tracedecay/tests/common/mod.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay/tests/common/mod.rs b/crates/tracedecay/tests/common/mod.rs index 3cea2149c3d..cc00c809dbe 100644 --- a/crates/tracedecay/tests/common/mod.rs +++ b/crates/tracedecay/tests/common/mod.rs @@ -523,14 +523,12 @@ exit /b %ERRORLEVEL%\r\n" ) } -/// Small multi-thread runtime for `#[test]`-driven async dashboard fixtures. +/// Small runtime with production scheduling for dashboard fixtures. pub fn create_runtime() -> tokio::runtime::Runtime { - match tokio::runtime::Builder::new_multi_thread() - // Dashboard tests issue synchronous ureq calls while the in-process - // server and database handlers use this same runtime. Two workers can - // deadlock under load (one blocked in ureq, one awaiting DB work). + match tracedecay_runtime_core::native_runtime::default_builder() + // Keep the fixture's worker count while using the daemon's wake + // policy: work woken during a long poll must remain stealable. .worker_threads(4) - .enable_all() .build() { Ok(runtime) => runtime, From 87ccf4140e60ad6c603cfad43277c5abc8e305c3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:18:09 -0800 Subject: [PATCH 089/111] test(cli): retain profile daemon readiness diagnostics --- .../sweep_outcomes.rs | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs index be3435cce96..53011a9a5b6 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs @@ -429,6 +429,11 @@ impl ProfileDaemon { let mut command = cli.command(&["daemon", "run"]); command .env("TRACEDECAY_TEST_ALLOW_INCOMPLETE_HOLDER_SCAN", "1") + .env( + "RUST_LOG", + "info,tracedecay::daemon=trace,tracedecay_daemon_identity=trace,tracedecay_private_fs=trace,tracedecay_global_db=trace", + ) + .env("TRACEDECAY_SPAN_TIMINGS", "1") .stdout(Stdio::null()) .stderr(stderr_log); daemon.child = Some(command.spawn().unwrap()); @@ -439,24 +444,34 @@ impl ProfileDaemon { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); // Follow the exact isolated profile's published endpoint, retaining // child ownership and startup diagnostics if admission fails. - while !current_daemon_connection(&cli.profile).is_ok_and(|connection| { - match connection.endpoint() { - DaemonEndpoint::Unix(socket) => { - std::os::unix::net::UnixStream::connect(socket).is_ok() + loop { + let readiness_error = match current_daemon_connection(&cli.profile) { + Ok(connection) => { + let connected = match connection.endpoint() { + DaemonEndpoint::Unix(socket) => { + std::os::unix::net::UnixStream::connect(socket).map(|_| ()) + } + DaemonEndpoint::Loopback(address) => { + std::net::TcpStream::connect(address).map(|_| ()) + } + }; + match connected { + Ok(()) => break, + Err(error) => format!("connect to {}: {error}", connection.endpoint()), + } } - DaemonEndpoint::Loopback(address) => std::net::TcpStream::connect(address).is_ok(), - } - }) { + Err(error) => format!("read daemon authority: {error}"), + }; if let Some(status) = daemon.child.as_mut().unwrap().try_wait().unwrap() { panic!( - "profile daemon exited before listening for {}: {status}; stderr:\n{}", + "profile daemon exited before listening for {}: {status}; last readiness error: {readiness_error}; stderr:\n{}", cli.profile.display(), stderr() ); } assert!( std::time::Instant::now() < deadline, - "the registered daemon never listened for profile {}; stderr:\n{}", + "the registered daemon never listened for profile {}; last readiness error: {readiness_error}; stderr:\n{}", cli.profile.display(), stderr() ); From 3c7825771da0f9f6360193d4e16a21b098ddd2ba Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:33:42 -0800 Subject: [PATCH 090/111] test(dashboard): capture stalled overview phase timings --- .../tests/dashboard_api_test/api.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/tracedecay/tests/dashboard_api_test/api.rs b/crates/tracedecay/tests/dashboard_api_test/api.rs index fab6d26e171..9cf41ecc479 100644 --- a/crates/tracedecay/tests/dashboard_api_test/api.rs +++ b/crates/tracedecay/tests/dashboard_api_test/api.rs @@ -1,5 +1,11 @@ use crate::dashboard_api_support::*; use serde_json::json; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::Layer as _; +use tracing_subscriber::filter::Targets; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::layer::SubscriberExt as _; +use tracing_subscriber::util::SubscriberInitExt as _; fn absent_canonical_fact_id(existing: &FactId) -> String { let raw = existing.as_str(); @@ -488,6 +494,25 @@ fn holographic_fact_detail_returns_full_content_and_entities() { #[test] fn holographic_fact_trust_history_returns_feedback_trail_and_empty_for_unreviewed_facts() { + // Measure the existing read phases without changing this fixture's budget. + let layer = tracing_subscriber::fmt::layer() + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_ansi(false) + .with_writer(std::io::stderr) + .with_filter( + Targets::new() + .with_default(LevelFilter::WARN) + .with_target("tracedecay_dashboard_api::memory_api", LevelFilter::TRACE) + .with_target( + "tracedecay_session_memory::memory::dashboard", + LevelFilter::TRACE, + ) + .with_target( + "tracedecay_session_memory::fact_store::graph", + LevelFilter::TRACE, + ), + ); + let _ = tracing_subscriber::registry().with(layer).try_init(); let runtime = create_runtime(); runtime.block_on(async { let fixture = start_dashboard_fixture(false).await; From e79c2ae728f647833376ea127ce27b140060501c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:11:52 -0800 Subject: [PATCH 091/111] test(daemon): require completed terminal retention pages Treat maintenance Retry as pending work and require the typed completed retention phase before asserting durability. Keep the existing admission, writer, settlement, page-count, and tail assertions. --- .../src/daemon/tests/terminal_housekeeping.rs | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs index 5407b7d71a4..54dd4624338 100644 --- a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs +++ b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs @@ -412,11 +412,20 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { for ordinal in 0..33 { seed_stack_signal(&shard, &format!("signal.{ordinal:03}"), 100 + ordinal, true).await; } + let other = tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open("terminal-writer-b").await; let held = shard.begin_write_transaction().await.expect("hold A SQLite writer"); let owner = daemon.coordinator.clone(); let administration = daemon.engine.store_administration.clone(); let tick = NativeTask::spawn(async move { - terminal_tick(&owner, &administration, None).await + // A queued store job or busy background CPU returns Retry before + // its page finishes. Drive that state as the cadence owner does. + loop { + let outcome = terminal_tick(&owner, &administration, None).await; + if outcome != MaintenanceTickOutcome::Retry { + return outcome; + } + tokio::task::yield_now().await; + } }).expect("native terminal tick"); let a_scope = store_writer_scope(shard.db_path().parent().expect("A store root"), StoreWriterClass::Content); native_runtime::timeout(Duration::from_secs(10), async { @@ -425,7 +434,6 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { tokio::task::yield_now().await; } }).await.expect("maintenance admitted A"); - let other = tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open("terminal-writer-b").await; let b_scope = store_writer_scope(other.registered.db_path().parent().expect("B store root"), StoreWriterClass::Content); let progressed = native_runtime::timeout(Duration::from_secs(2), daemon.engine.store_administration.with_writer_in(b_scope, || async { @@ -438,7 +446,9 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { // settles its admitted owner and releases every fixture lease. held.rollback().await.expect("release A"); let outcome = native_runtime::timeout(Duration::from_secs(10), tick.wait()).await.expect("terminal tick settled"); - assert!(matches!(outcome, native_runtime::NativeTaskOutcome::Completed(_))); + assert!(matches!(outcome, native_runtime::NativeTaskOutcome::Completed( + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) + )), "the terminal page must finish before its durable result is asserted: {outcome:?}"); assert!(progressed.is_ok(), "B must write while A is suspended with a terminal backlog"); assert_eq!(retired_signals(&shard).await, 16, "one admission runs one bounded terminal page"); assert_eq!(retirement_frontiers(&shard).await, 1); @@ -475,7 +485,16 @@ fn terminal_housekeeping_bounds_a_mounted_store_window_and_reaches_its_tail() { retired += retired_signals(database).await; } assert_eq!(retired, 8, "one tick admits at most eight store pages"); - assert_eq!(daemon.tick().await, MaintenanceTickOutcome::Complete); + // A queued writer or busy background slot returns Retry before the + // tail is durable. Follow the owner's continuation outcomes while + // preserving the first window's exact page bound above. + native_runtime::timeout(Duration::from_secs(10), async { + while daemon.tick().await != MaintenanceTickOutcome::Complete { + tokio::task::yield_now().await; + } + }) + .await + .expect("the mounted-store tail settles"); for database in &databases { assert_eq!( retired_signals(database).await, From 1b73b1539bf0d58a24a041ecff1da92e1026cdb1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:13:28 -0800 Subject: [PATCH 092/111] test(runtime): keep restart readiness within one deadline --- .../private_route_restart_acceptance.rs | 93 ++++++++++++++----- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs index f9b2882b711..28330e94b1f 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs @@ -2,10 +2,13 @@ use crate::common; use std::path::Path; use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; use serde_json::{Value, json}; use tracedecay::daemon::call_default_tool; -use tracedecay_daemon_protocol::DaemonHandshake; +use tracedecay_contracts::code_index_freshness::{CodeIndexReadinessTargetV1, CodeIndexReadinessV1}; +use tracedecay_contracts::retrieval::StatusCodeIndexFreshnessV1; +use tracedecay_daemon_protocol::{DaemonHandshake, TOOL_REQUEST_DEADLINE_ENV}; use tracedecay_runtime_core::config::ProfileRoot; fn initialize_project(home: &Path, project: &Path, marker: &str) { @@ -169,28 +172,70 @@ async fn selected_project_source_route_survives_physical_daemon_restart() { /// arms one, so a freshness wait would perturb the very state it samples. pub(crate) fn await_published_code_index(home: &Path, project: &Path) { let project_arg = project.to_string_lossy().into_owned(); - let output = common::tracedecay_command_with_home(home) - .current_dir(project) - .args([ - "tool", - "--project", - project_arg.as_str(), - "status", - "--args", - r#"{"format":"json","wait_for":{"state":"graph_ready","timeout_ms":110000}}"#, - ]) - .stdin(Stdio::null()) - .output() - .expect("run status"); - assert!( - output.status.success(), - "status failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - let status: Value = serde_json::from_slice(&output.stdout).expect("status JSON"); - assert_eq!( - status["wait"], - json!({ "outcome": "reached" }), - "daemon never published a serving code-index generation: {status}" + let deadline = Instant::now() + Duration::from_secs(110); + // CLI admission spends part of the request budget before status runs, so + // a nested 110-second wait can exceed the remaining dispatch deadline. + // Sample the same typed predicate under one end-to-end fixture deadline. + common::poll_until( + deadline, + Duration::from_millis(100), + || { + let remaining_ms = deadline + .saturating_duration_since(Instant::now()) + .as_millis(); + assert!(remaining_ms > 0, "code-index publication deadline elapsed"); + let output = common::tracedecay_command_with_home(home) + .current_dir(project) + .env(TOOL_REQUEST_DEADLINE_ENV, remaining_ms.to_string()) + .args([ + "tool", + "--project", + project_arg.as_str(), + "status", + "--args", + r#"{"format":"json"}"#, + ]) + .stdin(Stdio::null()) + .output() + .expect("run status"); + assert!( + output.status.success(), + "status failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let status: Value = serde_json::from_slice(&output.stdout).expect("status JSON"); + assert!( + Instant::now() < deadline, + "code-index publication deadline elapsed: {status}" + ); + let freshness: StatusCodeIndexFreshnessV1 = + serde_json::from_value(status["code_index_freshness"].clone()) + .expect("typed code-index freshness"); + let worktree = match freshness { + StatusCodeIndexFreshnessV1::Current { worktree } + | StatusCodeIndexFreshnessV1::Stale { worktree } + | StatusCodeIndexFreshnessV1::Restoring { worktree } + | StatusCodeIndexFreshnessV1::Warming { worktree } + | StatusCodeIndexFreshnessV1::Parked { worktree } => worktree, + StatusCodeIndexFreshnessV1::MountFailed { .. } + | StatusCodeIndexFreshnessV1::Unavailable { .. } => { + panic!("code-index readiness is unavailable: {status}") + } + }; + match worktree.readiness(CodeIndexReadinessTargetV1::GraphReady) { + CodeIndexReadinessV1::Reached => Some(()), + CodeIndexReadinessV1::Pending => None, + CodeIndexReadinessV1::Unreachable { reason } => { + panic!("code-index readiness cannot be reached: {reason}; {status}") + } + } + }, + || { + format!( + "daemon never published a serving generation for {}", + project.display() + ) + }, ); } From 094883f64071f0109b8062d71d4aa2dad3b64b5d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:13:30 -0800 Subject: [PATCH 093/111] test(advisory): use the production runtime policy --- .../advisory_cycle_language_journey_test.rs | 117 ++++++++++-------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs index 9bd9ef19436..c569ab57932 100644 --- a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs @@ -53,63 +53,70 @@ async fn settled_advisory_cycle( } } -#[tokio::test(flavor = "multi_thread")] -async fn typescript_only_checkout_runs_the_pull_request_advisory_cycle() { - let isolation = tempfile::TempDir::new().expect("production harness isolation"); - let project = isolation.path().join("project"); - std::fs::create_dir_all(project.join("src")).expect("project source dir"); - std::fs::write( - project.join("src/index.ts"), - "export function admitPullRequest(number: number): string {\n return `pr-${number}`;\n}\n", - ) - .expect("typescript source"); - git(&project, &["init", "--quiet", "-b", "feature/ts-admission"]); - git( - &project, - &[ - "remote", - "add", - "origin", - "https://git.example.invalid/tracedecay-fixture/typescript-admission.git", - ], - ); - git(&project, &["add", "."]); - git( - &project, - &["commit", "--quiet", "-m", "seed typescript admission"], - ); +#[test] +fn typescript_only_checkout_runs_the_pull_request_advisory_cycle() { + tracedecay_runtime_core::native_runtime::default_builder() + .build() + .expect("advisory journey runtime") + .block_on(async { + let isolation = tempfile::TempDir::new().expect("production harness isolation"); + let project = isolation.path().join("project"); + std::fs::create_dir_all(project.join("src")).expect("project source dir"); + std::fs::write( + project.join("src/index.ts"), + "export function admitPullRequest(number: number): string {\n return `pr-${number}`;\n}\n", + ) + .expect("typescript source"); + git(&project, &["init", "--quiet", "-b", "feature/ts-admission"]); + git( + &project, + &[ + "remote", + "add", + "origin", + "https://git.example.invalid/tracedecay-fixture/typescript-admission.git", + ], + ); + git(&project, &["add", "."]); + git( + &project, + &["commit", "--quiet", "-m", "seed typescript admission"], + ); - let harness = ProductionProjectCompositionHarnessV1::open(isolation.path(), [project.clone()]) - .await - .expect("production composition opens a TypeScript-only checkout"); + let harness = + ProductionProjectCompositionHarnessV1::open(isolation.path(), [project.clone()]) + .await + .expect("production composition opens a TypeScript-only checkout"); - let (refused, payload) = settled_advisory_cycle(&harness, &project, "src/index.ts").await; - assert!( - !refused, - "the cycle must run over the TypeScript generation: {payload}" - ); - assert_eq!( - payload["outcome"]["outcome"], - json!("evidence"), - "{payload}" - ); - let cycle = &payload["outcome"]["value"]["payload"]["cycle"]; - let producers: Vec<&Value> = cycle["advisory_provider_states"] - .as_array() - .unwrap_or_else(|| panic!("advisory provider states: {payload}")) - .iter() - .map(|state| &state["producer"]) - .collect(); - assert_eq!( - producers, - [ - &json!("git_hub_review"), - &json!("ci_localization"), - &json!("proximity") - ], - "{payload}" - ); - harness.shutdown().await; + let (refused, payload) = + settled_advisory_cycle(&harness, &project, "src/index.ts").await; + assert!( + !refused, + "the cycle must run over the TypeScript generation: {payload}" + ); + assert_eq!( + payload["outcome"]["outcome"], + json!("evidence"), + "{payload}" + ); + let cycle = &payload["outcome"]["value"]["payload"]["cycle"]; + let producers: Vec<&Value> = cycle["advisory_provider_states"] + .as_array() + .unwrap_or_else(|| panic!("advisory provider states: {payload}")) + .iter() + .map(|state| &state["producer"]) + .collect(); + assert_eq!( + producers, + [ + &json!("git_hub_review"), + &json!("ci_localization"), + &json!("proximity") + ], + "{payload}" + ); + harness.shutdown().await; + }); } /// A reopened project already holds a ready sealed generation, so the first From d8d24f58d7ec86ea8549e2a4ea49cc5f563b1430 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:15:19 -0800 Subject: [PATCH 094/111] test(fixtures): remove temporary tracing overrides Restore normal Kiro daemon and dashboard fixture logging while retaining the exact readiness error and existing failure output. Keep every probe, assertion, cadence, and deadline unchanged. --- .../sweep_outcomes.rs | 5 ---- .../tests/dashboard_api_test/api.rs | 25 ------------------- 2 files changed, 30 deletions(-) diff --git a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs index 53011a9a5b6..8d8ddb971a1 100644 --- a/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs +++ b/crates/tracedecay-cli/tests/host_journeys_suite/host_lifecycle_cli_acceptance/sweep_outcomes.rs @@ -429,11 +429,6 @@ impl ProfileDaemon { let mut command = cli.command(&["daemon", "run"]); command .env("TRACEDECAY_TEST_ALLOW_INCOMPLETE_HOLDER_SCAN", "1") - .env( - "RUST_LOG", - "info,tracedecay::daemon=trace,tracedecay_daemon_identity=trace,tracedecay_private_fs=trace,tracedecay_global_db=trace", - ) - .env("TRACEDECAY_SPAN_TIMINGS", "1") .stdout(Stdio::null()) .stderr(stderr_log); daemon.child = Some(command.spawn().unwrap()); diff --git a/crates/tracedecay/tests/dashboard_api_test/api.rs b/crates/tracedecay/tests/dashboard_api_test/api.rs index 9cf41ecc479..fab6d26e171 100644 --- a/crates/tracedecay/tests/dashboard_api_test/api.rs +++ b/crates/tracedecay/tests/dashboard_api_test/api.rs @@ -1,11 +1,5 @@ use crate::dashboard_api_support::*; use serde_json::json; -use tracing::level_filters::LevelFilter; -use tracing_subscriber::Layer as _; -use tracing_subscriber::filter::Targets; -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::layer::SubscriberExt as _; -use tracing_subscriber::util::SubscriberInitExt as _; fn absent_canonical_fact_id(existing: &FactId) -> String { let raw = existing.as_str(); @@ -494,25 +488,6 @@ fn holographic_fact_detail_returns_full_content_and_entities() { #[test] fn holographic_fact_trust_history_returns_feedback_trail_and_empty_for_unreviewed_facts() { - // Measure the existing read phases without changing this fixture's budget. - let layer = tracing_subscriber::fmt::layer() - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .with_ansi(false) - .with_writer(std::io::stderr) - .with_filter( - Targets::new() - .with_default(LevelFilter::WARN) - .with_target("tracedecay_dashboard_api::memory_api", LevelFilter::TRACE) - .with_target( - "tracedecay_session_memory::memory::dashboard", - LevelFilter::TRACE, - ) - .with_target( - "tracedecay_session_memory::fact_store::graph", - LevelFilter::TRACE, - ), - ); - let _ = tracing_subscriber::registry().with(layer).try_init(); let runtime = create_runtime(); runtime.block_on(async { let fixture = start_dashboard_fixture(false).await; From f1a21b0f1582d6e98518d2d29bf5c679bd521fd6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:22:40 -0800 Subject: [PATCH 095/111] test(private-fs): prove batch barriers overlap directly --- .../tracedecay-private-fs/src/framed_log.rs | 91 ++++++++++++++++++- .../tracedecay-private-fs/tests/framed_log.rs | 29 ++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-private-fs/src/framed_log.rs b/crates/tracedecay-private-fs/src/framed_log.rs index 19c621b4100..4cd318a14f0 100644 --- a/crates/tracedecay-private-fs/src/framed_log.rs +++ b/crates/tracedecay-private-fs/src/framed_log.rs @@ -380,7 +380,7 @@ fn remove_owned_temp(path: &Path) { pub mod sync_latency { use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{Arc, Mutex, PoisonError}; + use std::sync::{Arc, Condvar, Mutex, PoisonError}; use std::time::Duration; struct Injection { @@ -388,6 +388,13 @@ pub mod sync_latency { root: PathBuf, delay: Duration, syncs: Arc, + held: Option>, + } + + #[derive(Default)] + struct HeldSyncState { + released: Mutex, + changed: Condvar, } static INJECTIONS: Mutex> = Mutex::new(Vec::new()); @@ -416,9 +423,72 @@ pub mod sync_latency { } } + /// Keeps matching barriers in flight until explicitly released or dropped. + pub struct HeldSyncBarriers { + registration: SyncLatencyGuard, + state: Arc, + } + + impl HeldSyncBarriers { + /// Wait for the requested number of barriers to enter without releasing them. + #[must_use] + pub fn wait_for_syncs(&self, minimum: u64, timeout: Duration) -> bool { + let released = self + .state + .released + .lock() + .unwrap_or_else(PoisonError::into_inner); + let _wait = self + .state + .changed + .wait_timeout_while(released, timeout, |_| self.syncs() < minimum) + .unwrap_or_else(PoisonError::into_inner); + self.syncs() >= minimum + } + + #[must_use] + pub fn syncs(&self) -> u64 { + self.registration.syncs() + } + + pub fn release(&self) { + *self + .state + .released + .lock() + .unwrap_or_else(PoisonError::into_inner) = true; + self.state.changed.notify_all(); + } + } + + impl Drop for HeldSyncBarriers { + fn drop(&mut self) { + self.release(); + } + } + + /// Hold every fsync below `root` so a fixture can prove concurrent admission. + #[must_use] + pub fn hold(root: &Path) -> HeldSyncBarriers { + let state = Arc::new(HeldSyncState::default()); + let registration = register(root, Duration::ZERO, Some(Arc::clone(&state))); + HeldSyncBarriers { + registration, + state, + } + } + /// Delay every fsync of `root` or a path below it by `delay`. #[must_use] pub fn inject(root: &Path, delay: Duration) -> SyncLatencyGuard { + register(root, delay, None) + } + + fn register( + root: &Path, + delay: Duration, + held: Option>, + ) -> SyncLatencyGuard { let id = NEXT_ID.fetch_add(1, Ordering::SeqCst); let syncs = Arc::new(AtomicU64::new(0)); INJECTIONS @@ -429,6 +499,7 @@ pub mod sync_latency { root: root.to_path_buf(), delay, syncs: Arc::clone(&syncs), + held, }); SyncLatencyGuard { id, syncs } } @@ -439,9 +510,23 @@ pub mod sync_latency { .unwrap_or_else(PoisonError::into_inner) .iter() .find(|injection| path.starts_with(&injection.root)) - .map(|injection| (injection.delay, Arc::clone(&injection.syncs))); - if let Some((delay, syncs)) = matched { + .map(|injection| { + ( + injection.delay, + Arc::clone(&injection.syncs), + injection.held.clone(), + ) + }); + if let Some((delay, syncs, held)) = matched { syncs.fetch_add(1, Ordering::SeqCst); + if let Some(held) = held { + let released = held.released.lock().unwrap_or_else(PoisonError::into_inner); + held.changed.notify_all(); + let _released = held + .changed + .wait_while(released, |released| !*released) + .unwrap_or_else(PoisonError::into_inner); + } std::thread::sleep(delay); } } diff --git a/crates/tracedecay-private-fs/tests/framed_log.rs b/crates/tracedecay-private-fs/tests/framed_log.rs index dea713dbe51..c16fc6a9069 100644 --- a/crates/tracedecay-private-fs/tests/framed_log.rs +++ b/crates/tracedecay-private-fs/tests/framed_log.rs @@ -308,11 +308,13 @@ fn durable_batch_sync_flushes_only_its_members() { /// batch that syncs its members one after another pays one commit per member. /// The members' barriers must be in flight together, so a seal of many /// segments waits about one barrier, not one per segment. +/// Hold all barriers until every member enters: elapsed sleep time also +/// measures unrelated host scheduling and cannot prove that overlap. #[cfg(target_os = "linux")] #[test] fn durable_batch_sync_overlaps_member_barriers_on_a_slow_disk() { const MEMBERS: u32 = 16; - const BARRIER: Duration = Duration::from_millis(500); + const COMPLETION_BUDGET: Duration = Duration::from_secs(2); let root = tempfile::tempdir().expect("batch fixture root"); let mut batch = DurableFileBatch::new(); for index in 0..MEMBERS { @@ -321,15 +323,28 @@ fn durable_batch_sync_overlaps_member_barriers_on_a_slow_disk() { member.write_all(b"sealed segment").expect("member bytes"); batch.written(&path, &member).expect("record member"); } - let slow_disk = sync_latency::inject(root.path(), BARRIER); - let started = Instant::now(); - batch.sync().expect("batch sync"); + let syncs = std::thread::scope(|scope| { + let slow_disk = sync_latency::hold(root.path()); + let syncing = scope.spawn(|| batch.sync()); + let all_entered = slow_disk.wait_for_syncs( + u64::from(MEMBERS), + COMPLETION_BUDGET.saturating_sub(started.elapsed()), + ); + // Release even on failed admission before joining any blocked helpers. + slow_disk.release(); + syncing.join().expect("batch sync worker").expect("batch sync"); + assert!( + all_entered, + "all {MEMBERS} member barriers must be in flight before any can finish" + ); + slow_disk.syncs() + }); let elapsed = started.elapsed(); - assert_eq!(slow_disk.syncs(), u64::from(MEMBERS)); + assert_eq!(syncs, u64::from(MEMBERS)); assert!( - elapsed < BARRIER * (MEMBERS / 4), - "{MEMBERS} member barriers of {BARRIER:?} each took {elapsed:?} together" + elapsed < COMPLETION_BUDGET, + "{MEMBERS} concurrent member barriers took {elapsed:?} together" ); } From d94157ba608c7bfec8abe16880ca9c559a9f92a4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:49:11 -0800 Subject: [PATCH 096/111] style(tests): apply canonical fixture formatting --- crates/tracedecay-private-fs/tests/framed_log.rs | 5 ++++- .../private_route_restart_acceptance.rs | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/tests/framed_log.rs b/crates/tracedecay-private-fs/tests/framed_log.rs index c16fc6a9069..a88f58720dc 100644 --- a/crates/tracedecay-private-fs/tests/framed_log.rs +++ b/crates/tracedecay-private-fs/tests/framed_log.rs @@ -333,7 +333,10 @@ fn durable_batch_sync_overlaps_member_barriers_on_a_slow_disk() { ); // Release even on failed admission before joining any blocked helpers. slow_disk.release(); - syncing.join().expect("batch sync worker").expect("batch sync"); + syncing + .join() + .expect("batch sync worker") + .expect("batch sync"); assert!( all_entered, "all {MEMBERS} member barriers must be in flight before any can finish" diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs index 28330e94b1f..16a7315db0f 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs @@ -6,7 +6,9 @@ use std::time::{Duration, Instant}; use serde_json::{Value, json}; use tracedecay::daemon::call_default_tool; -use tracedecay_contracts::code_index_freshness::{CodeIndexReadinessTargetV1, CodeIndexReadinessV1}; +use tracedecay_contracts::code_index_freshness::{ + CodeIndexReadinessTargetV1, CodeIndexReadinessV1, +}; use tracedecay_contracts::retrieval::StatusCodeIndexFreshnessV1; use tracedecay_daemon_protocol::{DaemonHandshake, TOOL_REQUEST_DEADLINE_ENV}; use tracedecay_runtime_core::config::ProfileRoot; From 0205e08b47f7240359c046e64788eb8985681b69 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:37:04 -0800 Subject: [PATCH 097/111] test(runtime): wait for scheduler attachment during project open --- .../private_route_restart_acceptance.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs index 16a7315db0f..2f13fb6e77e 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs @@ -220,6 +220,13 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { | StatusCodeIndexFreshnessV1::Restoring { worktree } | StatusCodeIndexFreshnessV1::Warming { worktree } | StatusCodeIndexFreshnessV1::Parked { worktree } => worktree, + StatusCodeIndexFreshnessV1::Unavailable { reason } + if reason == "code_index_scheduler_not_mounted" => + { + // The canonical readiness wait allows a scheduler to mount + // inside its budget. This status sample has no mount refusal. + return None; + } StatusCodeIndexFreshnessV1::MountFailed { .. } | StatusCodeIndexFreshnessV1::Unavailable { .. } => { panic!("code-index readiness is unavailable: {status}") From f8352b6a5e20dcb87f7558be87bd414e27aa322b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:47:58 -0800 Subject: [PATCH 098/111] chore(agent-hosts): trace paired configuration publication --- .../src/agents/text_file_transaction.rs | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs b/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs index 594bf786164..eec069be205 100644 --- a/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs +++ b/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs @@ -55,6 +55,11 @@ impl Drop for HostFileWriteLock { /// a pathological churn storm surfaces as a typed error instead of a spin. const HOST_FILE_LOCK_ACQUIRE_ATTEMPTS: usize = 64; +#[tracing::instrument( + name = "agent_hosts.agents.config.lock", + level = "trace", + skip_all +)] pub(super) fn lock_host_file_write(path: &Path) -> Result { let parent = path .parent() @@ -279,6 +284,11 @@ impl HostFileSnapshot { } } +#[tracing::instrument( + name = "agent_hosts.agents.config.snapshot", + level = "trace", + skip_all +)] fn capture_host_file_snapshot(path: &Path) -> std::io::Result { let before = match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_file() => metadata, @@ -376,6 +386,11 @@ pub(crate) fn update_text_file_transactionally( /// Hold both path locks while publishing two host documents in order. /// A failure leaves completed mutations in place. +#[tracing::instrument( + name = "agent_hosts.agents.config.update_pair", + level = "trace", + skip_all +)] pub(crate) fn update_two_config_files_locked( first_path: &Path, second_path: &Path, @@ -406,15 +421,16 @@ pub(crate) fn update_two_config_files_locked( let first_existing = snapshot_utf8(first_path, &first_snapshot)?; let second_existing = snapshot_utf8(second_path, &second_snapshot)?; let (output, first_mutation, second_mutation) = update(first_existing, second_existing)?; - apply_file_mutation(first_path, &first_snapshot, &first_mutation)?; - apply_file_mutation(second_path, &second_snapshot, &second_mutation).map_err(|error| { - TraceDecayError::Config { + tracing::trace_span!("agent_hosts.agents.config.apply_document", document = "first") + .in_scope(|| apply_file_mutation(first_path, &first_snapshot, &first_mutation))?; + tracing::trace_span!("agent_hosts.agents.config.apply_document", document = "second") + .in_scope(|| apply_file_mutation(second_path, &second_snapshot, &second_mutation)) + .map_err(|error| TraceDecayError::Config { message: format!( "{error}; earlier changes to {} remain in place", first_path.display() ), - } - })?; + })?; Ok(output) } From 5334b3715066b2d4e0cc2c15215676bdcc481b0c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:48:04 -0800 Subject: [PATCH 099/111] test(agent-hosts): capture paired config publication phases --- .../src/agents/antigravity.rs | 9 +++++++-- crates/tracedecay-runtime-core/src/logging.rs | 19 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/antigravity.rs b/crates/tracedecay-agent-hosts/src/agents/antigravity.rs index 19ef8eb4ded..695f169b305 100644 --- a/crates/tracedecay-agent-hosts/src/agents/antigravity.rs +++ b/crates/tracedecay-agent-hosts/src/agents/antigravity.rs @@ -305,6 +305,8 @@ fn uninstall_mcp_if_selected(components: &[HostComponentV1], home: &Path) -> Res mod tests { use std::sync::Arc; + use tracedecay_runtime_core::logging::FormattedTracingCapture; + use super::*; fn install_context(home: &Path, binary: &str) -> InstallContext { @@ -428,10 +430,13 @@ mod tests { let install = Arc::new(install_context(home.path(), "/tmp/tracedecay")); let worker_install = Arc::clone(&install); let worker = std::thread::spawn(move || { - AntigravityIntegration.activate_deployed_host_component_registration( + let _capture = FormattedTracingCapture::start_with_span_timings(); + let outcome = AntigravityIntegration.activate_deployed_host_component_registration( &[HostComponentV1::ContextMcp], &worker_install, - ) + ); + tracing::info!(?outcome, "paired host registration worker settled"); + outcome }); pause.wait_until_reached(); diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 94cba0b7b6c..701629d6f01 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -279,21 +279,34 @@ impl FormattedTracingCapture { /// before the first captured event. #[must_use] pub fn start() -> Self { - Self::start_with_output(false) + Self::start_with_output(false, false) } /// Also writes each line to stderr so a terminated measurement retains /// the events emitted before its deadline. #[must_use] pub fn start_with_stderr() -> Self { - Self::start_with_output(true) + Self::start_with_output(true, false) } - fn start_with_output(stderr: bool) -> Self { + /// Captures span admission and completion timings as well as events, + /// retaining live stderr if the measurement is terminated before reporting. + #[must_use] + pub fn start_with_span_timings() -> Self { + Self::start_with_output(true, true) + } + + fn start_with_output(stderr: bool, span_timings: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_ansi(false) + .with_span_events(if span_timings { + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE + } else { + tracing_subscriber::fmt::format::FmtSpan::NONE + }) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { set_tracing_capture( From 3df1e45eaf05f4017547bc7a3e6860e71785521f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:48:18 -0800 Subject: [PATCH 100/111] test(dashboard): capture automation ledger read phases --- .../tests/dashboard_api_test/automation.rs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/tracedecay/tests/dashboard_api_test/automation.rs b/crates/tracedecay/tests/dashboard_api_test/automation.rs index 7759e4537fb..95cfe6c9b92 100644 --- a/crates/tracedecay/tests/dashboard_api_test/automation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/automation.rs @@ -1,5 +1,38 @@ +use std::sync::Once; + use crate::dashboard_api_support::*; use tracedecay_automation_runtime::automation::backend::AgentTaskRetryAttempt; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::Layer as _; +use tracing_subscriber::filter::Targets; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::layer::SubscriberExt as _; +use tracing_subscriber::util::SubscriberInitExt as _; + +fn measure_automation_run_reads() { + static START: Once = Once::new(); + START.call_once(|| { + // The HTTP server and ledger reads run on separate runtime threads. + let layer = tracing_subscriber::fmt::layer() + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_thread_ids(true) + .with_ansi(false) + .with_writer(std::io::stderr) + .with_filter( + Targets::new() + .with_default(LevelFilter::WARN) + .with_target( + "tracedecay_dashboard_api::automation_run_api", + LevelFilter::TRACE, + ) + .with_target( + "tracedecay_automation_runtime::automation::run_ledger", + LevelFilter::TRACE, + ), + ); + let _ = tracing_subscriber::registry().with(layer).try_init(); + }); +} #[test] fn fact_store_curate_is_the_only_public_manual_automation_launcher() { @@ -142,6 +175,7 @@ fn fact_store_curate_is_the_only_public_manual_automation_launcher() { #[test] fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment() { + measure_automation_run_reads(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); @@ -415,6 +449,7 @@ fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment( /// admission with the run's receipt, and the run settles on the daemon. #[test] fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { + measure_automation_run_reads(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); From 4c8573292f5166520cefc69b0a18e2080d1deb63 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:23:26 -0800 Subject: [PATCH 101/111] fix(fs): classify nonregular sealed receipt opens --- .../src/capability_dir.rs | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index b832851c8b4..d8f1748316b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -1049,19 +1049,27 @@ fn read_bounded_child_receipt( }; if descriptor < 0 { let error = io::Error::last_os_error(); - // Unix socket opens return ENXIO on Linux or EOPNOTSUPP on macOS. - // Classify that refusal as a nonregular receipt only when the held - // directory still names a nonregular entry. A regular file's I/O - // failure must remain a refusal of complete census coverage. - if matches!(error.raw_os_error(), Some(libc::ENXIO | libc::EOPNOTSUPP)) - && directory - .symlink_metadata(name) - .is_ok_and(|metadata| !metadata.file_type().is_file()) - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "sealed receipt is not a regular file", - )); + // The errno an open reports for a nonregular entry is platform + // specific: Linux refuses a socket with ENXIO, Darwin with + // EOPNOTSUPP, and a FIFO differs again. Classify through the SAME + // held directory's no-follow metadata instead, so a nonregular + // receipt is unrecognized on every Unix rather than aborting the + // whole census with an unclassified refusal. A regular file's open + // failure must remain a refusal of complete coverage. + if let Ok(metadata) = directory.symlink_metadata(name) { + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "path is a symbolic link", + )); + } + if !file_type.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "sealed receipt is not a regular file", + )); + } } return Err(normalize_no_follow_error(error)); } @@ -1629,8 +1637,10 @@ mod tests { std::fs::write(unknown.join("sentinel"), b"preserve").unwrap(); let _socket = UnixListener::bind(unknown.join("owner")).unwrap(); - // openat on a socket fails with ENXIO or EOPNOTSUPP before fstat. - // The held directory's metadata proves the entry is nonregular. + // openat(O_RDONLY | O_NONBLOCK) on a socket fails before any + // regular-file check, with a platform-specific errno (ENXIO on Linux, + // EOPNOTSUPP on Darwin). The held directory's metadata proves that + // entry is nonregular, not an unreadable owned receipt. let receipt = read_bounded_child_receipt(&open(&unknown), OsStr::new("owner"), 64); assert_eq!(receipt.unwrap_err().kind(), io::ErrorKind::InvalidData); From 546c8b65766baca388947357932da4b8e0bfaec8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:32:59 -0800 Subject: [PATCH 102/111] fix(private-fs): clean failed Windows pending receipts --- .../src/capability_dir.rs | 62 +++++++++++------- .../src/capability_dir/namespace_tests.rs | 63 +++++++++++++++++++ 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index d8f1748316b..6c8eb202ba0 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -21,6 +21,8 @@ use cap_fs_ext::DirExt; use cap_fs_ext::OpenOptionsMaybeDirExt; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; pub use cap_std::fs::Dir; +#[cfg(unix)] +use cap_std::fs::MetadataExt; use cap_std::fs::{File, OpenOptions}; /// Pins the parent of a cross-directory publication. On Windows the no-delete @@ -329,35 +331,53 @@ pub fn write_exclusive_pending_receipt_nofollow( let directory = open_root_nofollow(directory_path)?; let mut options = OpenOptions::new(); options.write(true).create_new(true); - let mut marker = directory.open_with(filename, &options)?; + let marker = directory.open_with(filename, &options)?; + write_pending_receipt(directory, filename, marker, contents) +} + +/// The caller exclusively created `marker` beneath this held directory. +fn write_pending_receipt( + directory: Dir, + filename: &OsStr, + mut marker: File, + contents: &[u8], +) -> io::Result<()> { #[cfg(unix)] - let marker_identity = { - use cap_std::fs::MetadataExt; - marker - .metadata() - .ok() - .map(|metadata| (metadata.dev(), metadata.ino())) - }; + let marker_identity = marker + .metadata() + .ok() + .map(|metadata| (metadata.dev(), metadata.ino())); + #[cfg(windows)] + let marker_identity = crate::windows_file::information(&marker) + .ok() + .map(|information| (information.volume_serial_number, information.file_index)); let written = marker.write_all(contents).and_then(|()| marker.sync_all()); - drop(marker); if let Err(error) = written { // Keep the original write failure. A different marker entry is not // ours to delete, and a nonempty substituted directory is not ours // to remove. Neither cleanup operation follows an ambient pathname. #[cfg(unix)] - let same_marker = { - use cap_std::fs::MetadataExt; - marker_identity.is_some_and(|identity| { - directory - .symlink_metadata(filename) - .ok() - .is_some_and(|metadata| { - metadata.is_file() && (metadata.dev(), metadata.ino()) == identity - }) - }) - }; - #[cfg(not(unix))] + let same_marker = marker_identity.is_some_and(|identity| { + directory + .symlink_metadata(filename) + .ok() + .is_some_and(|metadata| { + metadata.is_file() && (metadata.dev(), metadata.ino()) == identity + }) + }); + #[cfg(windows)] + let same_marker = marker_identity.is_some_and(|identity| { + open_regular_nofollow(&directory, filename) + .and_then(|file| crate::windows_file::information(&file)) + .is_ok_and(|information| { + (information.volume_serial_number, information.file_index) == identity + }) + }); + #[cfg(not(any(unix, windows)))] let same_marker = false; + // Retain the original file through the comparison so its identity + // cannot be recycled, then close it before removing its name. + drop(marker); if same_marker { if let Err(cleanup) = directory.remove_file(filename) { tracing::warn!(%cleanup, "pending owner marker could not be removed"); diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs index c4b7c0c604c..64bb01ba467 100644 --- a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -1,5 +1,12 @@ +#[cfg(any(unix, windows))] +use std::io::Write; use std::{ffi::OsStr, fs, io, path::Path}; +#[cfg(any(unix, windows))] +use cap_std::fs::{File, OpenOptions}; + +#[cfg(any(unix, windows))] +use super::write_pending_receipt; use super::{Dir, rename_noreplace, replace_file}; const FAMILY: [&str; 4] = [ @@ -13,6 +20,62 @@ fn open(path: &Path) -> Dir { Dir::open_ambient_dir(path, cap_std::ambient_authority()).unwrap() } +#[cfg(any(unix, windows))] +fn read_only_pending_marker(path: &Path) -> (Dir, File) { + fs::create_dir(path).unwrap(); + let directory = open(path); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + let mut marker = directory.open_with("owner", &options).unwrap(); + marker.write_all(b"partial receipt").unwrap(); + marker.sync_all().unwrap(); + drop(marker); + // A real read-only handle makes the production write fail without + // replacing its filesystem or cleanup operations with an injected result. + let marker = directory.open("owner").unwrap(); + (directory, marker) +} + +#[cfg(any(unix, windows))] +#[test] +fn failed_pending_receipt_removes_its_marker_and_empty_directory() { + let root = tempfile::tempdir().unwrap(); + let pending = root.path().join("pending"); + let (directory, marker) = read_only_pending_marker(&pending); + + assert!( + write_pending_receipt(directory, OsStr::new("owner"), marker, b"complete receipt") + .is_err() + ); + assert!( + !pending.exists(), + "failed unpublished receipts must not leave owned staging behind" + ); +} + +#[cfg(any(unix, windows))] +#[test] +fn failed_pending_receipt_preserves_a_replaced_marker() { + let root = tempfile::tempdir().unwrap(); + let pending = root.path().join("pending"); + let (directory, marker) = read_only_pending_marker(&pending); + directory.rename("owner", &directory, "original").unwrap(); + fs::write(pending.join("owner"), b"replacement receipt").unwrap(); + + assert!( + write_pending_receipt(directory, OsStr::new("owner"), marker, b"complete receipt") + .is_err() + ); + assert_eq!( + fs::read(pending.join("owner")).unwrap(), + b"replacement receipt" + ); + assert_eq!( + fs::read(pending.join("original")).unwrap(), + b"partial receipt" + ); +} + #[test] fn publication_refuses_escaped_sources_and_destinations_without_moving_store_files() { let root = tempfile::tempdir().unwrap(); From 848016672ce87b85659330d640aeb0a68b06707e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:33:07 -0800 Subject: [PATCH 103/111] style(agent-hosts): apply hosted span formatting --- .../src/agents/text_file_transaction.rs | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs b/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs index eec069be205..1447f453a74 100644 --- a/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs +++ b/crates/tracedecay-agent-hosts/src/agents/text_file_transaction.rs @@ -55,11 +55,7 @@ impl Drop for HostFileWriteLock { /// a pathological churn storm surfaces as a typed error instead of a spin. const HOST_FILE_LOCK_ACQUIRE_ATTEMPTS: usize = 64; -#[tracing::instrument( - name = "agent_hosts.agents.config.lock", - level = "trace", - skip_all -)] +#[tracing::instrument(name = "agent_hosts.agents.config.lock", level = "trace", skip_all)] pub(super) fn lock_host_file_write(path: &Path) -> Result { let parent = path .parent() @@ -284,11 +280,7 @@ impl HostFileSnapshot { } } -#[tracing::instrument( - name = "agent_hosts.agents.config.snapshot", - level = "trace", - skip_all -)] +#[tracing::instrument(name = "agent_hosts.agents.config.snapshot", level = "trace", skip_all)] fn capture_host_file_snapshot(path: &Path) -> std::io::Result { let before = match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_file() => metadata, @@ -421,16 +413,22 @@ pub(crate) fn update_two_config_files_locked( let first_existing = snapshot_utf8(first_path, &first_snapshot)?; let second_existing = snapshot_utf8(second_path, &second_snapshot)?; let (output, first_mutation, second_mutation) = update(first_existing, second_existing)?; - tracing::trace_span!("agent_hosts.agents.config.apply_document", document = "first") - .in_scope(|| apply_file_mutation(first_path, &first_snapshot, &first_mutation))?; - tracing::trace_span!("agent_hosts.agents.config.apply_document", document = "second") - .in_scope(|| apply_file_mutation(second_path, &second_snapshot, &second_mutation)) - .map_err(|error| TraceDecayError::Config { - message: format!( - "{error}; earlier changes to {} remain in place", - first_path.display() - ), - })?; + tracing::trace_span!( + "agent_hosts.agents.config.apply_document", + document = "first" + ) + .in_scope(|| apply_file_mutation(first_path, &first_snapshot, &first_mutation))?; + tracing::trace_span!( + "agent_hosts.agents.config.apply_document", + document = "second" + ) + .in_scope(|| apply_file_mutation(second_path, &second_snapshot, &second_mutation)) + .map_err(|error| TraceDecayError::Config { + message: format!( + "{error}; earlier changes to {} remain in place", + first_path.display() + ), + })?; Ok(output) } From 75eaca8af1d899f83af443a521b13b891c1a4139 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:56:39 -0800 Subject: [PATCH 104/111] style(private-fs): format pending receipt assertions --- .../src/capability_dir/namespace_tests.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs index 64bb01ba467..12ce0b15d37 100644 --- a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -44,8 +44,7 @@ fn failed_pending_receipt_removes_its_marker_and_empty_directory() { let (directory, marker) = read_only_pending_marker(&pending); assert!( - write_pending_receipt(directory, OsStr::new("owner"), marker, b"complete receipt") - .is_err() + write_pending_receipt(directory, OsStr::new("owner"), marker, b"complete receipt").is_err() ); assert!( !pending.exists(), @@ -63,8 +62,7 @@ fn failed_pending_receipt_preserves_a_replaced_marker() { fs::write(pending.join("owner"), b"replacement receipt").unwrap(); assert!( - write_pending_receipt(directory, OsStr::new("owner"), marker, b"complete receipt") - .is_err() + write_pending_receipt(directory, OsStr::new("owner"), marker, b"complete receipt").is_err() ); assert_eq!( fs::read(pending.join("owner")).unwrap(), From 31beb52a2d2ce62bd78750158bc8fe8006f7c595 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:57:16 -0800 Subject: [PATCH 105/111] test: remove scoped host and ledger phase capture --- .../src/agents/antigravity.rs | 9 ++--- crates/tracedecay-runtime-core/src/logging.rs | 19 ++-------- .../tests/dashboard_api_test/automation.rs | 35 ------------------- 3 files changed, 5 insertions(+), 58 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/agents/antigravity.rs b/crates/tracedecay-agent-hosts/src/agents/antigravity.rs index 695f169b305..19ef8eb4ded 100644 --- a/crates/tracedecay-agent-hosts/src/agents/antigravity.rs +++ b/crates/tracedecay-agent-hosts/src/agents/antigravity.rs @@ -305,8 +305,6 @@ fn uninstall_mcp_if_selected(components: &[HostComponentV1], home: &Path) -> Res mod tests { use std::sync::Arc; - use tracedecay_runtime_core::logging::FormattedTracingCapture; - use super::*; fn install_context(home: &Path, binary: &str) -> InstallContext { @@ -430,13 +428,10 @@ mod tests { let install = Arc::new(install_context(home.path(), "/tmp/tracedecay")); let worker_install = Arc::clone(&install); let worker = std::thread::spawn(move || { - let _capture = FormattedTracingCapture::start_with_span_timings(); - let outcome = AntigravityIntegration.activate_deployed_host_component_registration( + AntigravityIntegration.activate_deployed_host_component_registration( &[HostComponentV1::ContextMcp], &worker_install, - ); - tracing::info!(?outcome, "paired host registration worker settled"); - outcome + ) }); pause.wait_until_reached(); diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 701629d6f01..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -279,34 +279,21 @@ impl FormattedTracingCapture { /// before the first captured event. #[must_use] pub fn start() -> Self { - Self::start_with_output(false, false) + Self::start_with_output(false) } /// Also writes each line to stderr so a terminated measurement retains /// the events emitted before its deadline. #[must_use] pub fn start_with_stderr() -> Self { - Self::start_with_output(true, false) + Self::start_with_output(true) } - /// Captures span admission and completion timings as well as events, - /// retaining live stderr if the measurement is terminated before reporting. - #[must_use] - pub fn start_with_span_timings() -> Self { - Self::start_with_output(true, true) - } - - fn start_with_output(stderr: bool, span_timings: bool) -> Self { + fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_ansi(false) - .with_span_events(if span_timings { - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE - } else { - tracing_subscriber::fmt::format::FmtSpan::NONE - }) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { set_tracing_capture( diff --git a/crates/tracedecay/tests/dashboard_api_test/automation.rs b/crates/tracedecay/tests/dashboard_api_test/automation.rs index 95cfe6c9b92..7759e4537fb 100644 --- a/crates/tracedecay/tests/dashboard_api_test/automation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/automation.rs @@ -1,38 +1,5 @@ -use std::sync::Once; - use crate::dashboard_api_support::*; use tracedecay_automation_runtime::automation::backend::AgentTaskRetryAttempt; -use tracing::level_filters::LevelFilter; -use tracing_subscriber::Layer as _; -use tracing_subscriber::filter::Targets; -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::layer::SubscriberExt as _; -use tracing_subscriber::util::SubscriberInitExt as _; - -fn measure_automation_run_reads() { - static START: Once = Once::new(); - START.call_once(|| { - // The HTTP server and ledger reads run on separate runtime threads. - let layer = tracing_subscriber::fmt::layer() - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .with_thread_ids(true) - .with_ansi(false) - .with_writer(std::io::stderr) - .with_filter( - Targets::new() - .with_default(LevelFilter::WARN) - .with_target( - "tracedecay_dashboard_api::automation_run_api", - LevelFilter::TRACE, - ) - .with_target( - "tracedecay_automation_runtime::automation::run_ledger", - LevelFilter::TRACE, - ), - ); - let _ = tracing_subscriber::registry().with(layer).try_init(); - }); -} #[test] fn fact_store_curate_is_the_only_public_manual_automation_launcher() { @@ -175,7 +142,6 @@ fn fact_store_curate_is_the_only_public_manual_automation_launcher() { #[test] fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment() { - measure_automation_run_reads(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); @@ -449,7 +415,6 @@ fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment( /// admission with the run's receipt, and the run settles on the daemon. #[test] fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { - measure_automation_run_reads(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); From 0af0825289899d2a0fc10e2df1bc321b1e878d05 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:38:08 -0800 Subject: [PATCH 106/111] test(global-db): reuse the published cancellation fixture Restore the exact captured published workflow state between cancellation samples after the preceding runtime has retired. Preserve every sampled cancellation, full state assertion, fresh admission and reopen while avoiding repeated unrelated canonical schema initialization. Port the independently reviewed fixture change with identical source preimage and postimage. --- .../src/registered/workflow_schema_tests.rs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 01e87f7f53c..2973d16e469 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -1074,10 +1074,36 @@ mod published_artifact_ceiling { let stride = (total_polls / 12).max(1); let mut cancelled_published = 0; + // Each convergence joins its exact runtime retirement. Reuse the + // canonically initialized store, restoring the upgrade's inputs and + // proving the entire workflow state before admitting a fresh runtime. + let store = &reference.store; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create().await; - let store = &daemon_store.store; - store.run(BULK_ROWS); + { + let mut connection = Connection::open(&store.path).unwrap(); + let transaction = connection.transaction().unwrap(); + republish(&transaction, &before.table_sql, &before.digest); + { + let mut insert = transaction + .prepare( + "INSERT INTO workflow_artifact_payloads + (rowid, payload_digest, byte_length, payload) + VALUES (?1, ?2, ?3, ?4)", + ) + .unwrap(); + for (rowid, digest, byte_length, payload) in &before.artifacts { + insert + .execute(rusqlite::params![rowid, digest, byte_length, payload]) + .unwrap(); + } + } + transaction.commit().unwrap(); + } + assert_eq!( + store.state(), + before, + "poll {cancel_at} must start from the published workflow state" + ); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; let state = store.state(); match outcome { From 3606c631db5f0a5c75ea1522634cb388e968d460 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:41:23 -0800 Subject: [PATCH 107/111] fix(graph): clean failed initial staging receipts --- .../tracedecay-graph-db/src/sealed_store.rs | 16 ++++---- .../src/capability_dir.rs | 41 ++++++++++++++++--- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ecb9cb62112..047304796c6 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2132,6 +2132,12 @@ fn build_or_open_sealed_store_with_work( "sealed root must be a real directory", )); } + let owner = SealedStagingOwnerV1 { + producer: row_spill_owner().to_string(), + receipt: initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()), + }; + let owner_bytes = serde_json::to_vec(&owner) + .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; // A deterministic staging name is not proof of ownership. An earlier // interrupted build or a foreign directory must not be deleted on retry. // Exclusive creation refuses an occupied name, and every byte below is @@ -2139,16 +2145,10 @@ fn build_or_open_sealed_store_with_work( // the staging name cannot redirect a producer write into a foreign tree. let staging_directory = HeldDirectory::create_exclusive(&root, &staging_name) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; - let owner = SealedStagingOwnerV1 { - producer: row_spill_owner().to_string(), - receipt: initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()), - }; - let owner_bytes = serde_json::to_vec(&owner) - .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; // Exclusive creation refuses a pre-existing marker. Flush before writing // rows so a killed producer leaves verifiable deletion authority. - staging_directory - .write_new_file( + let staging_directory = staging_directory + .write_initial_receipt( std::ffi::OsStr::new(SEALED_STAGING_OWNER_FILE), &owner_bytes, ) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 6c8eb202ba0..7ae156c1307 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -318,8 +318,9 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { /// Persist a new producer-owned staging receipt through a held directory. /// /// An exclusive marker creation is the only authority for removing an -/// incomplete marker. On write/sync failure, remove only that marker and -/// attempt a nonrecursive removal of the same held directory. A substituted +/// incomplete marker. On open failure, attempt only nonrecursive removal of +/// the empty held directory. On write/sync failure, remove only that marker +/// and attempt nonrecursive removal of the same held directory. A substituted /// directory containing other bytes must survive. The caller must disable /// pathname-based temporary-directory cleanup before invoking this operation. pub fn write_exclusive_pending_receipt_nofollow( @@ -329,9 +330,28 @@ pub fn write_exclusive_pending_receipt_nofollow( ) -> io::Result<()> { require_entry_name(filename)?; let directory = open_root_nofollow(directory_path)?; + write_exclusive_pending_receipt(directory, filename, contents).map(|_| ()) +} + +fn write_exclusive_pending_receipt( + directory: Dir, + filename: &OsStr, + contents: &[u8], +) -> io::Result { let mut options = OpenOptions::new(); options.write(true).create_new(true); - let marker = directory.open_with(filename, &options)?; + let opened = require_entry_name(filename).and_then(|()| directory.open_with(filename, &options)); + let marker = match opened { + Ok(marker) => marker, + Err(error) => { + // No marker was created, so only emptiness can authorize cleanup. + // Keep the original refusal if this best-effort removal fails. + if let Err(cleanup) = directory.remove_open_dir() { + tracing::warn!(%cleanup, "pending owner directory could not be removed"); + } + return Err(error); + } + }; write_pending_receipt(directory, filename, marker, contents) } @@ -341,7 +361,7 @@ fn write_pending_receipt( filename: &OsStr, mut marker: File, contents: &[u8], -) -> io::Result<()> { +) -> io::Result { #[cfg(unix)] let marker_identity = marker .metadata() @@ -387,7 +407,7 @@ fn write_pending_receipt( } return Err(error); } - Ok(()) + Ok(directory) } /// Publish a rebuildable marker through one held directory. Replacing the @@ -518,6 +538,17 @@ impl HeldDirectory { .map(|directory| Self { directory }) } + /// Persist the initial owner receipt and retain this directory on success. + /// + /// The caller must own this unpublished directory. Invalid names and + /// failed marker opens attempt only nonrecursive removal of the held + /// directory. Failed writes or syncs first remove only the identity-checked + /// marker. Cleanup is best effort and never replaces the original error. + pub fn write_initial_receipt(self, name: &OsStr, contents: &[u8]) -> io::Result { + write_exclusive_pending_receipt(self.directory, name, contents) + .map(|directory| Self { directory }) + } + /// Writes a new file exclusively, relative to this handle. pub fn write_new_file(&self, name: &OsStr, contents: &[u8]) -> io::Result<()> { require_entry_name(name)?; From f9bf22707914ef8e08c4a6b73782e5fd78f7889a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:42:11 -0800 Subject: [PATCH 108/111] test(fs): cover initial receipt failure and retry --- .../src/capability_dir/namespace_tests.rs | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs index 12ce0b15d37..3cdd146cbda 100644 --- a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -6,7 +6,7 @@ use std::{ffi::OsStr, fs, io, path::Path}; use cap_std::fs::{File, OpenOptions}; #[cfg(any(unix, windows))] -use super::write_pending_receipt; +use super::{HeldDirectory, require_entry_name, write_pending_receipt}; use super::{Dir, rename_noreplace, replace_file}; const FAMILY: [&str; 4] = [ @@ -36,6 +36,71 @@ fn read_only_pending_marker(path: &Path) -> (Dir, File) { (directory, marker) } +#[cfg(any(unix, windows))] +#[test] +fn initial_receipt_open_failure_releases_directory_for_retry() { + let root = tempfile::tempdir().unwrap(); + let name = OsStr::new("pending"); + let pending = root.path().join(name); + let directory = HeldDirectory::create_exclusive(root.path(), name).unwrap(); + let filename = "owner".repeat(128); + require_entry_name(OsStr::new(&filename)).unwrap(); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + let expected = directory + .directory + .open_with(OsStr::new(&filename), &options) + .unwrap_err(); + + let error = directory + .write_initial_receipt(OsStr::new(&filename), b"complete receipt") + .err() + .expect("an oversized single entry must fail during marker open"); + assert_eq!(error.kind(), expected.kind()); + assert_eq!(error.raw_os_error(), expected.raw_os_error()); + assert!( + !pending.exists(), + "failed receipt open must release its empty directory" + ); + + let directory = HeldDirectory::create_exclusive(root.path(), name) + .unwrap() + .write_initial_receipt(OsStr::new("owner"), b"complete receipt") + .unwrap(); + directory + .write_new_file(OsStr::new("payload"), b"sealed rows") + .unwrap(); + assert_eq!(fs::read(pending.join("owner")).unwrap(), b"complete receipt"); + assert_eq!(fs::read(pending.join("payload")).unwrap(), b"sealed rows"); +} + +#[cfg(any(unix, windows))] +#[test] +fn initial_receipt_refusal_preserves_foreign_entries() { + for (filename, kind) in [ + ("owner", io::ErrorKind::AlreadyExists), + ("../owner", io::ErrorKind::InvalidInput), + ] { + let root = tempfile::tempdir().unwrap(); + let name = OsStr::new("pending"); + let pending = root.path().join(name); + let directory = HeldDirectory::create_exclusive(root.path(), name).unwrap(); + fs::write(pending.join("owner"), b"foreign receipt").unwrap(); + fs::write(root.path().join("owner"), b"outside receipt").unwrap(); + + let error = directory + .write_initial_receipt(OsStr::new(filename), b"replacement receipt") + .err() + .expect("receipt refusal must preserve foreign entries"); + assert_eq!(error.kind(), kind); + assert_eq!(fs::read(pending.join("owner")).unwrap(), b"foreign receipt"); + assert_eq!( + fs::read(root.path().join("owner")).unwrap(), + b"outside receipt" + ); + } +} + #[cfg(any(unix, windows))] #[test] fn failed_pending_receipt_removes_its_marker_and_empty_directory() { @@ -50,6 +115,12 @@ fn failed_pending_receipt_removes_its_marker_and_empty_directory() { !pending.exists(), "failed unpublished receipts must not leave owned staging behind" ); + + HeldDirectory::create_exclusive(root.path(), OsStr::new("pending")) + .unwrap() + .write_initial_receipt(OsStr::new("owner"), b"retry receipt") + .unwrap(); + assert_eq!(fs::read(pending.join("owner")).unwrap(), b"retry receipt"); } #[cfg(any(unix, windows))] From bc21c1697d03ea2a1c661d6d37c5b1c80f8217b4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:53:46 -0800 Subject: [PATCH 109/111] style(fs): wrap initial receipt open assignment --- crates/tracedecay-private-fs/src/capability_dir.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 7ae156c1307..806f7c464a5 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -340,7 +340,8 @@ fn write_exclusive_pending_receipt( ) -> io::Result { let mut options = OpenOptions::new(); options.write(true).create_new(true); - let opened = require_entry_name(filename).and_then(|()| directory.open_with(filename, &options)); + let opened = + require_entry_name(filename).and_then(|()| directory.open_with(filename, &options)); let marker = match opened { Ok(marker) => marker, Err(error) => { From 6d8db652777a0e34c0b9ca2328df5ee47c2c3801 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:10:33 -0800 Subject: [PATCH 110/111] style(fs): format receipt retry regressions --- .../src/capability_dir/namespace_tests.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs index 3cdd146cbda..a6f8eb8d492 100644 --- a/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs +++ b/crates/tracedecay-private-fs/src/capability_dir/namespace_tests.rs @@ -5,9 +5,9 @@ use std::{ffi::OsStr, fs, io, path::Path}; #[cfg(any(unix, windows))] use cap_std::fs::{File, OpenOptions}; +use super::{Dir, rename_noreplace, replace_file}; #[cfg(any(unix, windows))] use super::{HeldDirectory, require_entry_name, write_pending_receipt}; -use super::{Dir, rename_noreplace, replace_file}; const FAMILY: [&str; 4] = [ "sessions.db", @@ -70,7 +70,10 @@ fn initial_receipt_open_failure_releases_directory_for_retry() { directory .write_new_file(OsStr::new("payload"), b"sealed rows") .unwrap(); - assert_eq!(fs::read(pending.join("owner")).unwrap(), b"complete receipt"); + assert_eq!( + fs::read(pending.join("owner")).unwrap(), + b"complete receipt" + ); assert_eq!(fs::read(pending.join("payload")).unwrap(), b"sealed rows"); } From bfc5e80d51160d33fcb8cdbfe2ebf97dd6523979 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:17:57 -0800 Subject: [PATCH 111/111] test(global-db): avoid unused schema fixture actor startup --- .../src/tests/lcm_schema/mod.rs | 7 +++- .../src/tests/lcm_schema/temporal_catalog.rs | 18 ++++------ .../lcm_schema/temporal_catalog/admission.rs | 33 +++++++------------ .../tests/lcm_schema/temporal_constraints.rs | 15 +++------ .../src/tests/lcm_schema/temporal_cursor.rs | 15 +++------ 5 files changed, 33 insertions(+), 55 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 22a7d3611c0..fbfe8b5f1f8 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -15,7 +15,7 @@ use tracedecay_runtime_core::db::{ use crate::tests::harness::open_registered_test_database_fixture; -async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { +async fn initialize_global_db(db_path: &Path) -> tracedecay_domain::errors::Result<()> { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent)?; } @@ -23,6 +23,11 @@ async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result tracedecay_domain::errors::Result { + initialize_global_db(db_path).await?; Ok(TestConnection::open(db_path)) } diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 4dcc84fa725..1d2cb2f65d2 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -63,10 +63,9 @@ async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { async fn temporal_payload_manifest_schema_is_payload_global() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -175,10 +174,9 @@ mod admission; async fn temporal_schema_query_indexes_cover_exact_lookup_shapes() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -305,10 +303,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -608,10 +605,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh async fn temporal_schema_rejects_unexpected_indexes_without_dropping_them() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -719,10 +715,9 @@ async fn temporal_schema_refuses_markerless_malformed_fts_without_mutation() { async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -791,10 +786,9 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { async fn temporal_schema_rejects_missing_graph_publication_authority_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index a5ba897a0e1..c1ec89d7d12 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -285,10 +285,9 @@ async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_cont async fn temporal_schema_rejects_transition_storage_without_mutating_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -332,10 +331,9 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -383,10 +381,9 @@ async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutati async fn final_store_missing_temporal_marker_requires_reset_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive every final authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -556,10 +553,9 @@ async fn writer_ledger_does_not_hide_a_nonempty_unmarked_store() { async fn final_schema_admission_rejects_extra_temporal_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -598,10 +594,9 @@ async fn final_schema_admission_rejects_extra_temporal_column_metadata() { async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal indexes"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -635,10 +630,9 @@ async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() async fn final_schema_admission_rejects_missing_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -670,10 +664,9 @@ async fn final_schema_admission_rejects_missing_graph_publication_table() { async fn final_schema_admission_rejects_incompatible_graph_publication_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -712,10 +705,9 @@ async fn final_schema_admission_rejects_incompatible_graph_publication_column_me async fn final_schema_admission_rejects_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -753,10 +745,9 @@ async fn final_schema_admission_rejects_extra_graph_publication_table() { async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -794,10 +785,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -835,10 +825,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index b9c1c52d30f..17c54837976 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -71,10 +71,9 @@ async fn temporal_schema_rejects_cross_session_and_generation_rows() { async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -258,10 +257,9 @@ async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -365,10 +363,9 @@ async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1160,10 +1157,9 @@ async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { async fn temporal_schema_enforces_generation_state_machine_and_durability() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1271,10 +1267,9 @@ async fn temporal_schema_enforces_generation_state_machine_and_durability() { async fn temporal_schema_keeps_append_only_authority_immutable() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index a1f6398e1b2..ad69571ce89 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -87,10 +87,9 @@ async fn temporal_schema_persists_cursor_keys_without_read_creation() { async fn temporal_schema_rejects_direct_cursor_retirement() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -121,10 +120,9 @@ async fn temporal_schema_rejects_direct_cursor_retirement() { async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -250,10 +248,9 @@ async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -304,10 +301,9 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -362,10 +358,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone();