From 835291a360a706f86a1b354c3b3f3de759556b88 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:13:22 -0800 Subject: [PATCH 01/49] perf(sessions): bound non-transcript Codex directory enumeration Refs #4756 A flat archived directory with many .zst/non-JSONL entries previously consumed its entire ReadDir iterator in one structural scan turn, outside the directory-work budget. Charge directory entries independently of child directory traversal while retaining the iterator across turns. Add a real archived-directory regression requiring the first scan to yield in the directory phase, then converge on exactly the eligible JSONL path. Partial performance repair only; durable restart continuation and affected host CPU/write-latency acceptance remain outstanding. --- .../src/runtime/hosts/codex.rs | 8 ++++ .../src/runtime/hosts/codex/tests.rs | 48 ++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 2ab2ff738ac..ab2c2ee3f14 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2077,6 +2077,10 @@ fn retained_scan_step( let directory_work_limit = work_limit.saturating_mul(usize::from(MAX_SCAN_DEPTH).saturating_add(2)); let mut directory_work = 0usize; + // Flat archived directories can contain thousands of non-transcript + // entries. Retain their ReadDir cursor across turns, just as for child + // directories, instead of enumerating the whole archive in one call. + let mut directory_entries = 0usize; let mut file_work = 0usize; let mut paths = Vec::new(); let mut selected_sources = Vec::new(); @@ -2207,8 +2211,12 @@ fn retained_scan_step( detail: "retained Codex directory iterator disappeared", }, )?; + if directory_entries >= directory_work_limit { + break; + } match listed.next() { Some(entry) => { + directory_entries += 1; let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { operation: "read Codex transcript directory entry", path: dir.clone(), diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 771f91636db..aadcc613237 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -1230,7 +1230,7 @@ mod recent_first_discovery_tests { use crate::runtime::hosts::codex::{ CodexCorpusEpoch, CodexDiscoveryDelivery, CodexDiscoveryFrontier, CodexDiscoveryHub, CodexDiscoverySourceKey, CodexDiscoveryState, CodexExactSessionPathAuthority, - CodexIndexedPath, CodexReplayIndex, EXACT_HOOK_DISCOVERY_UNITS_PER_CALL, + CodexIndexedPath, CodexReplayIndex, CodexScanPhase, EXACT_HOOK_DISCOVERY_UNITS_PER_CALL, IDLE_FULL_VALIDATION_CYCLES, MAX_EXACT_HOOK_SESSION_REQUESTS, MAX_EXACT_HOOK_SOURCE_AUTHORITIES, MAX_SCAN_DEPTH, codex_corpus_identity, codex_directory_changed, codex_directory_stat_identity, codex_directory_witness, @@ -2828,6 +2828,52 @@ mod recent_first_discovery_tests { ); } + #[test] + fn archived_non_jsonl_entries_yield_directory_enumeration_across_windows() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"compressed") + .unwrap(); + } + let admitted = archive.join("rollout-admitted.jsonl"); + std::fs::write(&admitted, b"{}\n").unwrap(); + + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + assert!(first.report.is_truncated()); + assert!(first.report.paths.is_empty()); + assert!( + matches!( + state.scan.as_ref().map(|scan| &scan.phase), + Some(CodexScanPhase::Directories { + current: Some(_), + .. + }) + ), + "non-transcript directory entries must not exhaust one discovery turn" + ); + frontier = first.next_frontier; + + let mut covered = BTreeSet::new(); + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + covered.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert_eq!(covered, BTreeSet::from([admitted])); + assert!(frontier.is_complete(), "bounded pages must converge"); + let idle = retained_pass(&source, &mut state, bounds, frontier); + assert!(idle.report.paths.is_empty()); + } + /// Sweep-complete is store-durable: a fresh source reading only admission /// parse offsets (the production persist path) idles instead of restarting /// truncated-from-zero. MemoryHostAdmission is the same offset table a From 0390a59ed2292c9a7c06c8628772e3826b631d6e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:37:18 -0800 Subject: [PATCH 02/49] perf(sessions): exclude ignored archive names from Codex probes Refs #4756 Keep unsettled directory name witnesses scoped to traversable child buckets and JSONL candidate paths. Compressed rollouts and other ignored file churn must not independently invalidate the candidate witness. Cover ignored additions and renames, candidate admission, and child bucket discovery with a real-filesystem regression. --- .../src/runtime/hosts/codex.rs | 11 +++++- .../src/runtime/hosts/codex/tests.rs | 36 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index ab2c2ee3f14..26ad74a7e77 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2693,6 +2693,15 @@ fn directory_entry_fingerprint(path: &Path) -> TranscriptIngestResult<[u8; 32]> path: entry.path(), source, })?; + // Only real subdirectories and potential JSONL candidates affect + // discovery. Ignored compressed archives must not change the name + // witness when the coarse native directory timestamp stays equal. + let name = entry.file_name(); + if !file_type.is_dir() + && Path::new(&name).extension().and_then(|ext| ext.to_str()) != Some("jsonl") + { + continue; + } let tag = if file_type.is_symlink() { 2_u8 } else if file_type.is_dir() { @@ -2700,7 +2709,7 @@ fn directory_entry_fingerprint(path: &Path) -> TranscriptIngestResult<[u8; 32]> } else { 0 }; - names.push((entry.file_name(), tag)); + names.push((name, tag)); } names.sort(); let mut hasher = Sha256::new(); diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index aadcc613237..12b9b70e1d8 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2675,6 +2675,42 @@ mod recent_first_discovery_tests { assert!(found, "the changed hot bucket must deliver the new rollout"); } + #[test] + fn ignored_codex_archive_entries_do_not_change_candidate_fingerprint() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join("archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let initial = directory_entry_fingerprint(&archive).unwrap(); + + // Non-JSONL names are in the directory but not in the candidate + // set. The previous fingerprint changed for every ignored addition. + for index in 0..128 { + std::fs::write(archive.join(format!("rollout-{index:04}.jsonl.zst")), b"zst").unwrap(); + } + assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); + + std::fs::rename( + archive.join("rollout-0000.jsonl.zst"), + archive.join("rollout-renamed.jsonl.zst"), + ) + .unwrap(); + assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); + + let eligible = archive.join("rollout-new.jsonl"); + std::fs::write(&eligible, b"{}\n").unwrap(); + assert_ne!(directory_entry_fingerprint(&archive).unwrap(), initial); + std::fs::remove_file(eligible).unwrap(); + assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); + + std::fs::create_dir(archive.join("2026")).unwrap(); + assert_ne!( + directory_entry_fingerprint(&archive).unwrap(), + initial, + "child directories must remain part of traversal authority" + ); + } + #[test] #[cfg(unix)] fn unsettled_directory_stat_does_not_change_with_each_sample() { From b71eb2bfee4e1aee8f7c22f0f74cc97cc76248ab Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:25:55 -0800 Subject: [PATCH 03/49] style(sessions): match pinned rustfmt on Codex regressions Refs #4756 Apply the two exact formatting hunks emitted by Bazel CI; preserve every behavioral test assertion. --- .../src/runtime/hosts/codex/tests.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 12b9b70e1d8..efaf61a00fb 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2686,7 +2686,11 @@ mod recent_first_discovery_tests { // Non-JSONL names are in the directory but not in the candidate // set. The previous fingerprint changed for every ignored addition. for index in 0..128 { - std::fs::write(archive.join(format!("rollout-{index:04}.jsonl.zst")), b"zst").unwrap(); + std::fs::write( + archive.join(format!("rollout-{index:04}.jsonl.zst")), + b"zst", + ) + .unwrap(); } assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); @@ -2870,8 +2874,11 @@ mod recent_first_discovery_tests { let archive = temp.path().join(".codex/archived_sessions"); std::fs::create_dir_all(&archive).unwrap(); for index in 0..64 { - std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"compressed") - .unwrap(); + std::fs::write( + archive.join(format!("rollout-{index:03}.zst")), + b"compressed", + ) + .unwrap(); } let admitted = archive.join("rollout-admitted.jsonl"); std::fs::write(&admitted, b"{}\n").unwrap(); From 3912dd4255e334b706294dfc8c9461cf78318b00 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 00:20:37 -0800 Subject: [PATCH 04/49] perf(sessions): bound Codex name witness to discovery slices Refs #4756 --- .../src/runtime/hosts/codex.rs | 150 +++++++++++++----- .../src/runtime/hosts/codex/tests.rs | 75 +++++++++ 2 files changed, 187 insertions(+), 38 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 26ad74a7e77..a4a652ace0a 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -1038,7 +1038,7 @@ struct CodexRetainedScan { enum CodexScanPhase { Directories { queued: VecDeque<(PathBuf, u8, u8, u64)>, - current: Option>, + current: Option>, }, Files { next_directory: usize, @@ -1050,7 +1050,18 @@ enum CodexScanPhase { struct CodexDirectoryResume { queued: VecDeque<(PathBuf, u8, u8, u64)>, - current: Option>, + current: Option>, +} + +/// Retain both the native iterator and its unfinished name witness across +/// budgeted discovery passes; neither can be reconstructed from an offset. +struct CodexDirectoryListing { + path: PathBuf, + depth: u8, + root_order: u8, + entries: std::fs::ReadDir, + fingerprint: Option, + directory_index: usize, } #[derive(Clone, Copy)] @@ -2187,7 +2198,14 @@ fn retained_scan_step( }, )?; } - let identity = codex_directory_witness(&path)?; + // Do not enumerate a potentially huge unsettled archive + // just to obtain its witness before the bounded iterator. + let stat = directory_stat_witness(&path)?; + let identity = stat.map(|witness| CodexDirectoryWitness { + stat: witness.stat, + entries: None, + }); + let directory_index = scan.directories.len(); scan.directories.push(CodexDirectoryIdentity { path: path.clone(), root_order, @@ -2203,10 +2221,19 @@ fn retained_scan_step( source, } })?; - *current = Some(Box::new((path, depth, root_order, listed))); + *current = Some(Box::new(CodexDirectoryListing { + path, + depth, + root_order, + entries: listed, + fingerprint: stat + .filter(|witness| !witness.settled) + .map(|_| CodexDirectoryEntryFingerprint::default()), + directory_index, + })); continue; } - let (dir, depth, root_order, listed) = current.as_deref_mut().ok_or( + let directory = current.as_deref_mut().ok_or( TranscriptIngestError::InvalidCodexDiscoveryFrontier { detail: "retained Codex directory iterator disappeared", }, @@ -2214,12 +2241,12 @@ fn retained_scan_step( if directory_entries >= directory_work_limit { break; } - match listed.next() { + match directory.entries.next() { Some(entry) => { directory_entries += 1; let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { operation: "read Codex transcript directory entry", - path: dir.clone(), + path: directory.path.clone(), source, })?; let file_type = @@ -2230,13 +2257,16 @@ fn retained_scan_step( path: entry.path(), source, })?; + if let Some(fingerprint) = directory.fingerprint.as_mut() { + fingerprint.observe(&entry.file_name(), file_type)?; + } // Rollout files are not structural work. Charging them // here spends the pass on names that are not child // directories, so the newest sessions are not emitted // until every older day has been listed. if file_type.is_dir() && !file_type.is_symlink() { directory_work += 1; - if *depth >= MAX_SCAN_DEPTH { + if directory.depth >= MAX_SCAN_DEPTH { return Err(TranscriptIngestError::ScanIo { operation: "traverse Codex transcript directory depth", path: entry.path(), @@ -2283,7 +2313,7 @@ fn retained_scan_step( detail: "Codex transcript directory depth overflowed", }, )?, - *root_order, + directory.root_order, 0, )); let resume_directories = CodexDirectoryResume { @@ -2306,13 +2336,27 @@ fn retained_scan_step( detail: "Codex transcript directory depth overflowed", }, )?, - *root_order, + directory.root_order, charge, )); } } } - None => *current = None, + None => { + if let Some(fingerprint) = directory.fingerprint.take() + && let Some(identity) = scan + .directories + .get_mut(directory.directory_index) + .filter(|item| { + item.path == directory.path + && item.root_order == directory.root_order + }) + .and_then(|item| item.identity.as_mut()) + { + identity.entries = Some(fingerprint.finish()); + } + *current = None; + } } } CodexScanPhase::Files { @@ -2611,6 +2655,7 @@ fn codex_directory_changed( Ok(directory_entry_fingerprint(path)? != entries) } +#[cfg(test)] fn codex_directory_witness(path: &Path) -> TranscriptIngestResult> { let Some(current) = directory_stat_witness(path)? else { return Ok(None); @@ -2673,13 +2718,66 @@ fn codex_directory_stat_identity( codex_corpus_identity(path, metadata, stat_change) } +/// Order-independent bounded-memory set witness. Directory entries are +/// unique names, so their separately domain-hashed names can be combined +/// without storing and sorting an unbounded filename list. +#[derive(Default)] +struct CodexDirectoryEntryFingerprint { + names: u64, + xor: [u8; 32], +} + +impl CodexDirectoryEntryFingerprint { + fn observe( + &mut self, + name: &std::ffi::OsStr, + file_type: std::fs::FileType, + ) -> TranscriptIngestResult<()> { + if !file_type.is_dir() + && Path::new(name).extension().and_then(|ext| ext.to_str()) != Some("jsonl") + { + return Ok(()); + } + let tag = if file_type.is_symlink() { + 2_u8 + } else if file_type.is_dir() { + 1 + } else { + 0 + }; + let mut hasher = Sha256::new(); + hasher.update(b"tracedecay-codex-directory-entry-v2"); + hasher.update([tag]); + hash_directory_name(&mut hasher, name); + let digest: [u8; 32] = hasher.finalize().into(); + for (accumulator, byte) in self.xor.iter_mut().zip(digest) { + *accumulator ^= byte; + } + self.names = self.names.checked_add(1).ok_or( + TranscriptIngestError::InvalidCodexDiscoveryFrontier { + detail: "Codex directory fingerprint entry count overflowed", + }, + )?; + Ok(()) + } + + fn finish(self) -> [u8; 32] { + let mut hasher = Sha256::new(); + // These witnesses are process-local, not durable corpus identities. + hasher.update(b"tracedecay-codex-directory-entries-v2"); + hasher.update(self.names.to_le_bytes()); + hasher.update(self.xor); + hasher.finalize().into() + } +} + fn directory_entry_fingerprint(path: &Path) -> TranscriptIngestResult<[u8; 32]> { let listed = std::fs::read_dir(path).map_err(|source| TranscriptIngestError::ScanIo { operation: "fingerprint Codex transcript directory", path: path.to_path_buf(), source, })?; - let mut names = Vec::new(); + let mut fingerprint = CodexDirectoryEntryFingerprint::default(); for entry in listed { let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { operation: "fingerprint Codex transcript directory entry", @@ -2693,33 +2791,9 @@ fn directory_entry_fingerprint(path: &Path) -> TranscriptIngestResult<[u8; 32]> path: entry.path(), source, })?; - // Only real subdirectories and potential JSONL candidates affect - // discovery. Ignored compressed archives must not change the name - // witness when the coarse native directory timestamp stays equal. - let name = entry.file_name(); - if !file_type.is_dir() - && Path::new(&name).extension().and_then(|ext| ext.to_str()) != Some("jsonl") - { - continue; - } - let tag = if file_type.is_symlink() { - 2_u8 - } else if file_type.is_dir() { - 1 - } else { - 0 - }; - names.push((name, tag)); - } - names.sort(); - let mut hasher = Sha256::new(); - hasher.update(b"tracedecay-codex-directory-entries-v1"); - for (name, tag) in names { - hasher.update([tag]); - hash_directory_name(&mut hasher, &name); - hasher.update([0xff]); + fingerprint.observe(&entry.file_name(), file_type)?; } - Ok(hasher.finalize().into()) + Ok(fingerprint.finish()) } fn hash_directory_name(hasher: &mut Sha256, name: &std::ffi::OsStr) { diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index efaf61a00fb..8220a29aa85 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2917,6 +2917,81 @@ mod recent_first_discovery_tests { assert!(idle.report.paths.is_empty()); } + #[test] + #[cfg(unix)] + fn unsettled_flat_archive_fingerprint_is_incremental_across_budgeted_passes() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..128 { + std::fs::write( + archive.join(format!("rollout-{index:04}.jsonl.zst")), + b"compressed", + ) + .unwrap(); + } + let eligible = archive.join("rollout-eligible.jsonl"); + std::fs::write(&eligible, b"{}\n").unwrap(); + let expected = directory_entry_fingerprint(&archive).unwrap(); + assert!( + !directory_stat_witness(&archive).unwrap().unwrap().settled, + "the fixture must exercise the coarse-clock name-witness path" + ); + + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + assert!(first.report.is_truncated()); + let scan = state.scan.as_ref().expect("unfinished structural scan"); + let CodexScanPhase::Directories { + current: Some(current), + .. + } = &scan.phase + else { + panic!("the archive must retain a live directory iterator"); + }; + assert_eq!(current.path, archive); + assert!( + scan.directories[current.directory_index] + .identity + .expect("directory stat witness") + .entries + .is_none(), + "no eager whole-archive fingerprint before bounded traversal" + ); + frontier = first.next_frontier; + + let mut covered = BTreeSet::new(); + for _ in 0..512 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + covered.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!(frontier.is_complete(), "bounded traversal must converge"); + assert_eq!(covered, BTreeSet::from([eligible])); + let idle = state.idle.as_ref().expect("completed sweep retains witnesses"); + let archive_identity = idle + .directories + .iter() + .find(|directory| directory.path == archive) + .and_then(|directory| directory.identity) + .expect("archive directory witness"); + assert_eq!( + archive_identity.entries, + Some(expected), + "the same full candidate witness must be assembled exactly once" + ); + + let stable = retained_pass(&source, &mut state, bounds, frontier); + assert!(stable.report.paths.is_empty()); + assert!(stable.next_frontier.is_complete()); + } + /// Sweep-complete is store-durable: a fresh source reading only admission /// parse offsets (the production persist path) idles instead of restarting /// truncated-from-zero. MemoryHostAdmission is the same offset table a From 2598f104eecd2bfd0b8dbe5e7a4057708424755a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:45:13 -0800 Subject: [PATCH 05/49] fix(sessions): repair Codex incremental scanner compilation Refs #4756 --- crates/tracedecay-sessions/src/runtime/hosts/codex.rs | 4 ++-- crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index a4a652ace0a..1b09fbb192a 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2308,7 +2308,7 @@ fn retained_scan_step( } queued.push_front(( child, - depth.checked_add(1).ok_or( + directory.depth.checked_add(1).ok_or( TranscriptIngestError::InvalidCodexDiscoveryFrontier { detail: "Codex transcript directory depth overflowed", }, @@ -2331,7 +2331,7 @@ fn retained_scan_step( scan.directory_bytes = charged; queued.push_front(( child, - depth.checked_add(1).ok_or( + directory.depth.checked_add(1).ok_or( TranscriptIngestError::InvalidCodexDiscoveryFrontier { detail: "Codex transcript directory depth overflowed", }, diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 8220a29aa85..16ceb55865c 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2974,7 +2974,10 @@ mod recent_first_discovery_tests { } assert!(frontier.is_complete(), "bounded traversal must converge"); assert_eq!(covered, BTreeSet::from([eligible])); - let idle = state.idle.as_ref().expect("completed sweep retains witnesses"); + let idle = state + .idle + .as_ref() + .expect("completed sweep retains witnesses"); let archive_identity = idle .directories .iter() From 4b3e4bab704de5f5fc4432b05ead7ee3d9adf83d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:23:19 -0800 Subject: [PATCH 06/49] fix(sessions): borrow Codex directory stat for witness Refs #4756 --- crates/tracedecay-sessions/src/runtime/hosts/codex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 1b09fbb192a..8d362e3a750 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2201,7 +2201,7 @@ fn retained_scan_step( // Do not enumerate a potentially huge unsettled archive // just to obtain its witness before the bounded iterator. let stat = directory_stat_witness(&path)?; - let identity = stat.map(|witness| CodexDirectoryWitness { + let identity = stat.as_ref().map(|witness| CodexDirectoryWitness { stat: witness.stat, entries: None, }); From 1fc5852b5e9e1fd9dcb727e15cbb44c5dff3ec63 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:35:02 -0800 Subject: [PATCH 07/49] fix(sessions): refuse changed Codex directory sweep coverage Refs #4756 A budgeted directory iterator may span multiple ticks while new entries arrive. Verify its opening stat witness when it is exhausted and keep coverage incomplete after an observed change, then require a fresh sweep. Add a real filesystem regression for mid-scan mutation and convergence. --- .../src/runtime/hosts/codex.rs | 29 +++++--- .../src/runtime/hosts/codex/tests.rs | 71 +++++++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 8d362e3a750..9b30085c102 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2343,17 +2343,26 @@ fn retained_scan_step( } } None => { - if let Some(fingerprint) = directory.fingerprint.take() - && let Some(identity) = scan - .directories - .get_mut(directory.directory_index) - .filter(|item| { - item.path == directory.path - && item.root_order == directory.root_order - }) - .and_then(|item| item.identity.as_mut()) + // A budgeted ReadDir can span many scheduler turns. + // If the directory changed while we held it, the + // iterator may omit new entries. Never publish a + // complete corpus frontier from that mixed snapshot. + let closing_stat = directory_stat_witness(&directory.path)?; + if let Some(identity) = scan + .directories + .get_mut(directory.directory_index) + .filter(|item| { + item.path == directory.path + && item.root_order == directory.root_order + }) + .and_then(|item| item.identity.as_mut()) { - identity.entries = Some(fingerprint.finish()); + if closing_stat.as_ref().map(|stat| stat.stat) != Some(identity.stat) { + scan.complete = false; + } + if let Some(fingerprint) = directory.fingerprint.take() { + identity.entries = Some(fingerprint.finish()); + } } *current = None; } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 16ceb55865c..06512e00a7c 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2917,6 +2917,77 @@ mod recent_first_discovery_tests { assert!(idle.report.paths.is_empty()); } + #[test] + fn a_mutated_directory_cannot_complete_a_budgeted_codex_sweep() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + let initial_stat = { + let scan = state.scan.as_ref().expect("partial structural scan"); + let CodexScanPhase::Directories { + current: Some(current), + .. + } = &scan.phase + else { + panic!("a flat archive must leave its directory reader active"); + }; + assert_eq!(current.path, archive); + scan.directories[current.directory_index] + .identity + .expect("initial directory witness") + .stat + }; + frontier = first.next_frontier; + + // The initial iterator may or may not see the new entry. Regardless, + // its opening stat no longer proves a complete, coherent sweep. + let added = archive.join("rollout-arrived-during-scan.jsonl"); + std::fs::write(&added, b"{}\n").unwrap(); + assert_ne!( + directory_stat_witness(&archive).unwrap().unwrap().stat, + initial_stat, + "the fixture must cause a real directory authority transition" + ); + + let mut ended_first_sweep = false; + for _ in 0..512 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if state.scan.is_none() { + assert!( + !frontier.is_complete(), + "a changed directory must not earn complete coverage from a stale iterator" + ); + ended_first_sweep = true; + break; + } + } + assert!(ended_first_sweep, "the first bounded sweep must terminate"); + + let mut observed = BTreeSet::new(); + for _ in 0..1024 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!(frontier.is_complete(), "a stable subsequent sweep must converge"); + assert!( + observed.contains(&added), + "the new source must be rediscovered before complete coverage" + ); + } + #[test] #[cfg(unix)] fn unsettled_flat_archive_fingerprint_is_incremental_across_budgeted_passes() { From 0978194ed07fbee1befd056b73bb748e4a133dec Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:35:43 -0800 Subject: [PATCH 08/49] test(sessions): retain sources from invalidated Codex sweep Refs #4756 The first sweep may already emit the newly arrived rollout even when its directory witness requires another pass. Assert complete source coverage across both the invalidated and stable sweeps. --- crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 06512e00a7c..9c04a133c6f 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2957,9 +2957,11 @@ mod recent_first_discovery_tests { "the fixture must cause a real directory authority transition" ); + let mut observed = BTreeSet::new(); let mut ended_first_sweep = false; for _ in 0..512 { let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); frontier = pass.next_frontier; if state.scan.is_none() { assert!( @@ -2972,7 +2974,6 @@ mod recent_first_discovery_tests { } assert!(ended_first_sweep, "the first bounded sweep must terminate"); - let mut observed = BTreeSet::new(); for _ in 0..1024 { let pass = retained_pass(&source, &mut state, bounds, frontier); observed.extend(pass.report.paths); From 3c99fb09c922935f955a95819a6c1ed7fc167633 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:36:49 -0800 Subject: [PATCH 09/49] fix(sessions): fence Codex directory identity across chunks Refs #4756 Retain the opening stat on the directory iterator itself so changes across structural memory windows still invalidate sweep completion, even when the prior chunk's directory list was released. --- .../src/runtime/hosts/codex.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 9b30085c102..3ccb1c149ae 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -1060,6 +1060,7 @@ struct CodexDirectoryListing { depth: u8, root_order: u8, entries: std::fs::ReadDir, + opening_stat: [u8; 32], fingerprint: Option, directory_index: usize, } @@ -2211,9 +2212,9 @@ fn retained_scan_step( root_order, identity, }); - if identity.is_none() { + let Some(opening_stat) = identity.map(|witness| witness.stat) else { continue; - } + }; let listed = std::fs::read_dir(&path).map_err(|source| { TranscriptIngestError::ScanIo { operation: "enumerate Codex transcript directories", @@ -2226,6 +2227,7 @@ fn retained_scan_step( depth, root_order, entries: listed, + opening_stat, fingerprint: stat .filter(|witness| !witness.settled) .map(|_| CodexDirectoryEntryFingerprint::default()), @@ -2348,6 +2350,12 @@ fn retained_scan_step( // iterator may omit new entries. Never publish a // complete corpus frontier from that mixed snapshot. let closing_stat = directory_stat_witness(&directory.path)?; + if closing_stat.as_ref().map(|stat| stat.stat) != Some(directory.opening_stat) { + scan.complete = false; + } + // Chunks can retire their directory identity while + // retaining the unfinished iterator. The opening + // witness still fences coverage in that case. if let Some(identity) = scan .directories .get_mut(directory.directory_index) @@ -2357,9 +2365,6 @@ fn retained_scan_step( }) .and_then(|item| item.identity.as_mut()) { - if closing_stat.as_ref().map(|stat| stat.stat) != Some(identity.stat) { - scan.complete = false; - } if let Some(fingerprint) = directory.fingerprint.take() { identity.entries = Some(fingerprint.finish()); } From a2cf71a461c7c93d69036d7e79c9583e4ef7064f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:38:00 -0800 Subject: [PATCH 10/49] style(sessions): format directory-sweep witness assertions Refs #4756 Match the repository's multiline Rust formatting without changing the mutation fence or the real filesystem regression. --- crates/tracedecay-sessions/src/runtime/hosts/codex.rs | 4 +++- crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 3ccb1c149ae..4529135e84a 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2350,7 +2350,9 @@ fn retained_scan_step( // iterator may omit new entries. Never publish a // complete corpus frontier from that mixed snapshot. let closing_stat = directory_stat_witness(&directory.path)?; - if closing_stat.as_ref().map(|stat| stat.stat) != Some(directory.opening_stat) { + if closing_stat.as_ref().map(|stat| stat.stat) + != Some(directory.opening_stat) + { scan.complete = false; } // Chunks can retire their directory identity while diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 9c04a133c6f..beb191d200b 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2923,7 +2923,11 @@ mod recent_first_discovery_tests { let archive = temp.path().join(".codex/archived_sessions"); std::fs::create_dir_all(&archive).unwrap(); for index in 0..64 { - std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + std::fs::write( + archive.join(format!("rollout-{index:03}.zst")), + b"ignored", + ) + .unwrap(); } let source = CodexSource::with_home(temp.path()); let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); From cfa88fae45e8bc1b3429cf655e31764cdfea88f0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:06:03 -0800 Subject: [PATCH 11/49] style(sessions): format bounded Codex scan regression --- .../src/runtime/hosts/codex/tests.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index beb191d200b..fda64fb2c68 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2923,11 +2923,7 @@ mod recent_first_discovery_tests { let archive = temp.path().join(".codex/archived_sessions"); std::fs::create_dir_all(&archive).unwrap(); for index in 0..64 { - std::fs::write( - archive.join(format!("rollout-{index:03}.zst")), - b"ignored", - ) - .unwrap(); + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); } let source = CodexSource::with_home(temp.path()); let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); @@ -2986,7 +2982,10 @@ mod recent_first_discovery_tests { break; } } - assert!(frontier.is_complete(), "a stable subsequent sweep must converge"); + assert!( + frontier.is_complete(), + "a stable subsequent sweep must converge" + ); assert!( observed.contains(&added), "the new source must be rediscovered before complete coverage" From ecc476e7ff44001190d5e5ce6f4f8d27517294d9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:16:15 -0800 Subject: [PATCH 12/49] fix(sessions): refuse incomplete coarse-clock Codex sweeps Refs #4756 An unsettled native change clock can coalesce directory mutations across a retained ReadDir walk. Matching opening and closing stat bytes are not enough to assert complete discovery. Require a settled closing witness before certifying the sweep; retain in-progress coverage until validation can establish a stable directory. Add a real bounded-read regression that aliases the native stat across a mid-scan JSONL arrival and rejects a false complete watermark. --- .../src/runtime/hosts/codex.rs | 12 +++- .../src/runtime/hosts/codex/tests.rs | 58 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 4529135e84a..9369bdddf92 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2350,9 +2350,15 @@ fn retained_scan_step( // iterator may omit new entries. Never publish a // complete corpus frontier from that mixed snapshot. let closing_stat = directory_stat_witness(&directory.path)?; - if closing_stat.as_ref().map(|stat| stat.stat) - != Some(directory.opening_stat) - { + // A coarse-clock directory can change without its + // native stat advancing. Its incremental name witness + // describes only what this iterator saw, not entries + // created between slices. Do not certify completion + // until the closing change stamp is settled and still + // names the opening directory identity. + if !closing_stat.is_some_and(|stat| { + stat.settled && stat.stat == directory.opening_stat + }) { scan.complete = false; } // Chunks can retire their directory identity while diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index fda64fb2c68..2ad6d20c88f 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2917,6 +2917,64 @@ mod recent_first_discovery_tests { assert!(idle.report.paths.is_empty()); } + #[test] + fn unsettled_directory_clock_alias_cannot_publish_complete_coverage() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + assert!(first.report.is_truncated()); + frontier = first.next_frontier; + + // Force the valid coarse-clock case: a new source arrives while the + // iterator is retained, but the native directory stat does not advance. + // Only a settled clock can vouch for the opening and closing identity. + let arrived = archive.join("rollout-added-during-scan.jsonl"); + std::fs::write(&arrived, b"{}\n").unwrap(); + let closing = directory_stat_witness(&archive).unwrap().unwrap(); + assert!( + !closing.settled, + "the regression requires an unsettled directory change stamp" + ); + let scan = state.scan.as_mut().expect("unfinished structural scan"); + let CodexScanPhase::Directories { + current: Some(directory), + .. + } = &mut scan.phase + else { + panic!("the flat archive must retain its native directory iterator"); + }; + assert_eq!(directory.path, archive); + assert!( + directory.fingerprint.is_some(), + "coarse-clock admission must retain the incremental name witness" + ); + directory.opening_stat = closing.stat; + assert!(scan.complete); + + let mut first_sweep_finished = false; + for _ in 0..512 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if state.scan.is_none() { + first_sweep_finished = true; + assert!( + !frontier.is_complete(), + "equal but unsettled directory stats cannot vouch for complete coverage" + ); + break; + } + } + assert!(first_sweep_finished, "the bounded structural sweep must finish"); + } + #[test] fn a_mutated_directory_cannot_complete_a_budgeted_codex_sweep() { let temp = TempDir::new().unwrap(); From 0d4be4e69ffeafd2e8d0781d2966129e4e9cc8e6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:18:57 -0800 Subject: [PATCH 13/49] fix(sessions): fence bounded Codex file walk coverage Refs #4756 Revert an over-conservative unsettled-stat condition that could prevent completed fresh scans until the filesystem clock settled. Instead, recheck each retained file-phase directory identity at EOF so files arriving after the structural walk cannot be silently omitted from a complete frontier. Preserve the existing per-turn entry budget. Add a real filesystem regression inserting JSONL while file-phase ReadDir is suspended; full coverage must remain in progress. --- .../src/runtime/hosts/codex.rs | 29 +++++--- .../src/runtime/hosts/codex/tests.rs | 66 +++++++++---------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 9369bdddf92..e28412eddab 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2350,15 +2350,9 @@ fn retained_scan_step( // iterator may omit new entries. Never publish a // complete corpus frontier from that mixed snapshot. let closing_stat = directory_stat_witness(&directory.path)?; - // A coarse-clock directory can change without its - // native stat advancing. Its incremental name witness - // describes only what this iterator saw, not entries - // created between slices. Do not certify completion - // until the closing change stamp is settled and still - // names the opening directory identity. - if !closing_stat.is_some_and(|stat| { - stat.settled && stat.stat == directory.opening_stat - }) { + if closing_stat.as_ref().map(|stat| stat.stat) + != Some(directory.opening_stat) + { scan.complete = false; } // Chunks can retire their directory identity while @@ -2449,7 +2443,22 @@ fn retained_scan_step( break Some((path, metadata)); } } - None => *current = None, + None => { + // This file iterator is retained across + // scheduler turns. A new directory entry can + // arrive after the structural walk but before + // file enumeration finishes. Recheck its native + // identity before granting full coverage: a + // stale ReadDir may have omitted that entry. + let directory = &scan.directories[*directory_index]; + let closing = directory_stat_witness(&directory.path)?; + if closing.as_ref().map(|stat| stat.stat) + != directory.identity.map(|identity| identity.stat) + { + scan.complete = false; + } + *current = None; + } } } }; diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 2ad6d20c88f..dea73bb4ce2 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2918,61 +2918,59 @@ mod recent_first_discovery_tests { } #[test] - fn unsettled_directory_clock_alias_cannot_publish_complete_coverage() { + fn file_walk_detects_a_new_rollout_while_its_directory_cursor_is_retained() { let temp = TempDir::new().unwrap(); let archive = temp.path().join(".codex/archived_sessions"); std::fs::create_dir_all(&archive).unwrap(); for index in 0..64 { std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); } + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let source = CodexSource::with_home(temp.path()); let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); let mut state = CodexDiscoveryState::default(); let mut frontier = CodexDiscoveryFrontier::initial(); - let first = retained_pass(&source, &mut state, bounds, frontier); - assert!(first.report.is_truncated()); - frontier = first.next_frontier; - - // Force the valid coarse-clock case: a new source arrives while the - // iterator is retained, but the native directory stat does not advance. - // Only a settled clock can vouch for the opening and closing identity. - let arrived = archive.join("rollout-added-during-scan.jsonl"); + let mut active = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_ref() + && let CodexScanPhase::Files { + current: Some(current), + .. + } = &scan.phase + { + let directory = &scan.directories[current.0]; + assert_eq!(directory.path, archive); + assert!(directory.identity.is_some()); + active = true; + break; + } + } + assert!(active, "the file-phase iterator must retain its position"); + let prior = directory_stat_witness(&archive).unwrap().unwrap().stat; + let arrived = archive.join("rollout-added-during-file-walk.jsonl"); std::fs::write(&arrived, b"{}\n").unwrap(); - let closing = directory_stat_witness(&archive).unwrap().unwrap(); - assert!( - !closing.settled, - "the regression requires an unsettled directory change stamp" - ); - let scan = state.scan.as_mut().expect("unfinished structural scan"); - let CodexScanPhase::Directories { - current: Some(directory), - .. - } = &mut scan.phase - else { - panic!("the flat archive must retain its native directory iterator"); - }; - assert_eq!(directory.path, archive); - assert!( - directory.fingerprint.is_some(), - "coarse-clock admission must retain the incremental name witness" + assert_ne!( + directory_stat_witness(&archive).unwrap().unwrap().stat, + prior, + "the mutation must change the native directory witness" ); - directory.opening_stat = closing.stat; - assert!(scan.complete); - - let mut first_sweep_finished = false; - for _ in 0..512 { + let mut finished = false; + for _ in 0..1024 { let pass = retained_pass(&source, &mut state, bounds, frontier); frontier = pass.next_frontier; if state.scan.is_none() { - first_sweep_finished = true; + finished = true; assert!( !frontier.is_complete(), - "equal but unsettled directory stats cannot vouch for complete coverage" + "a new file during the retained file walk must invalidate its coverage" ); break; } } - assert!(first_sweep_finished, "the bounded structural sweep must finish"); + assert!(finished, "the bounded file traversal must reach its frontier"); } #[test] From 32805e4907075d3982a45f88977ec5f0b76cae2d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:46:26 -0800 Subject: [PATCH 14/49] fix(sessions): verify unsettled Codex directory listings Refs #4756 After a retained file iterator finishes under a coarse directory clock, reopen its eligible-name walk in bounded chunks and compare against the structural name fingerprint before granting complete coverage. Preserve Windows convergence without assuming directories have settled timestamps. Regress a late eligible rollout with an intentionally aliased coarse stat witness; keep all file and directory per-call budgets unchanged. --- .../src/runtime/hosts/codex.rs | 77 ++++++++++++++++++- .../src/runtime/hosts/codex/tests.rs | 64 +++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index e28412eddab..7d3053db8d9 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -1043,6 +1043,7 @@ enum CodexScanPhase { Files { next_directory: usize, current: Option>, + verification: Option>, deferred: Option>, resume_directories: Option, }, @@ -2117,6 +2118,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: Some(resume_directories), }; @@ -2137,6 +2139,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: None, }; @@ -2187,6 +2190,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: Some(resume_directories), }; @@ -2325,6 +2329,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: Some(resume_directories), }; @@ -2378,6 +2383,7 @@ fn retained_scan_step( CodexScanPhase::Files { next_directory, current, + verification, deferred, resume_directories, } => { @@ -2392,6 +2398,48 @@ fn retained_scan_step( Some(*candidate) } else { loop { + // An unsettled directory's native timestamp can alias + // a new entry while a file iterator is suspended. + // Verify eligible names through a fresh, bounded walk. + if let Some((directory_index, entries, fingerprint)) = + verification.as_deref_mut() + { + if file_work >= work_limit { + break None; + } + match entries.next() { + Some(entry) => { + file_work += 1; + let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { + operation: "verify Codex transcript directory entry", + path: scan.directories[*directory_index].path.clone(), + source, + })?; + let file_type = entry.file_type().map_err(|source| { + TranscriptIngestError::ScanIo { + operation: "verify Codex transcript entry type", + path: entry.path(), + source, + } + })?; + fingerprint.observe(&entry.file_name(), file_type)?; + continue; + } + None => { + let directory = &scan.directories[*directory_index]; + let closing = directory_stat_witness(&directory.path)?; + if closing.as_ref().map(|stamp| stamp.stat) + != directory.identity.map(|identity| identity.stat) + || directory.identity.and_then(|identity| identity.entries) + != Some(std::mem::take(fingerprint).finish()) + { + scan.complete = false; + } + *verification = None; + continue; + } + } + } if current.is_none() { let Some(directory) = scan.directories.get(*next_directory) else { break None; @@ -2456,6 +2504,30 @@ fn retained_scan_step( != directory.identity.map(|identity| identity.stat) { scan.complete = false; + } else if closing.is_some_and(|stamp| !stamp.settled) { + // A fresh reader must recheck the structural + // name set before this iterator can confer + // full corpus coverage on a coarse clock. + if directory + .identity + .and_then(|identity| identity.entries) + .is_some() + { + let reopened = std::fs::read_dir(&directory.path).map_err( + |source| TranscriptIngestError::ScanIo { + operation: "verify Codex transcript bucket", + path: directory.path.clone(), + source, + }, + )?; + *verification = Some(Box::new(( + *directory_index, + reopened, + CodexDirectoryEntryFingerprint::default(), + ))); + } else { + scan.complete = false; + } } *current = None; } @@ -2463,7 +2535,10 @@ fn retained_scan_step( } }; let Some((path, metadata)) = candidate else { - if *next_directory >= scan.directories.len() && current.is_none() { + if *next_directory >= scan.directories.len() + && current.is_none() + && verification.is_none() + { if let Some(resume) = resume_directories.take() { scan.directories.clear(); scan.directory_bytes = 0; diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index dea73bb4ce2..06da3302d4c 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2973,6 +2973,70 @@ mod recent_first_discovery_tests { assert!(finished, "the bounded file traversal must reach its frontier"); } + #[test] + fn aliased_coarse_directory_stamp_cannot_certify_a_stale_file_walk() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let mut active = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_ref() + && let CodexScanPhase::Files { + current: Some(current), + .. + } = &scan.phase + && scan.directories[current.0].path == archive + { + active = true; + break; + } + } + assert!(active, "the file reader must remain suspended across turns"); + let original_names = directory_entry_fingerprint(&archive).unwrap(); + let late = archive.join("rollout-after-file-reader-open.jsonl"); + std::fs::write(&late, b"{}\n").unwrap(); + // Model two writes within one filesystem timestamp quantum: the + // directory stat aliases the scan's authority despite a new file. + let alias = directory_stat_witness(&archive).unwrap().unwrap().stat; + let scan = state.scan.as_mut().unwrap(); + let CodexScanPhase::Files { + current: Some(current), + .. + } = &scan.phase + else { + panic!("the file-phase iterator must still be retained"); + }; + let directory_index = current.0; + let identity = scan.directories[directory_index].identity.as_mut().unwrap(); + identity.stat = alias; + identity.entries = Some(original_names); + + let mut finished = false; + for _ in 0..2048 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if state.scan.is_none() { + assert!( + !frontier.is_complete(), + "an aliased directory timestamp hid a new eligible rollout" + ); + finished = true; + break; + } + } + assert!(finished, "the bounded verification must terminate"); + } + #[test] fn a_mutated_directory_cannot_complete_a_budgeted_codex_sweep() { let temp = TempDir::new().unwrap(); From 294ea91c7c370320dc8cb46119316ee90d4dd297 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:48:58 -0800 Subject: [PATCH 15/49] fix(sessions): verify originally unsettled Codex buckets Refs #4756 An initial coarse-clock name fingerprint must be independently checked when the file iterator closes, even if its clock has since settled. Do not poison a temporarily suspended structural directory batch. Keep the aliasing regression deterministic across slow test hosts. --- .../src/runtime/hosts/codex.rs | 51 ++++++++++--------- .../src/runtime/hosts/codex/tests.rs | 12 +++++ 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 7d3053db8d9..1808328b527 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2504,30 +2504,33 @@ fn retained_scan_step( != directory.identity.map(|identity| identity.stat) { scan.complete = false; - } else if closing.is_some_and(|stamp| !stamp.settled) { - // A fresh reader must recheck the structural - // name set before this iterator can confer - // full corpus coverage on a coarse clock. - if directory - .identity - .and_then(|identity| identity.entries) - .is_some() - { - let reopened = std::fs::read_dir(&directory.path).map_err( - |source| TranscriptIngestError::ScanIo { - operation: "verify Codex transcript bucket", - path: directory.path.clone(), - source, - }, - )?; - *verification = Some(Box::new(( - *directory_index, - reopened, - CodexDirectoryEntryFingerprint::default(), - ))); - } else { - scan.complete = false; - } + } else if directory + .identity + .and_then(|identity| identity.entries) + .is_some() + { + // This initially unsettled listing needs an + // independent current-name check, even if the + // filesystem clock settled during the walk. + let reopened = std::fs::read_dir(&directory.path).map_err( + |source| TranscriptIngestError::ScanIo { + operation: "verify Codex transcript bucket", + path: directory.path.clone(), + source, + }, + )?; + *verification = Some(Box::new(( + *directory_index, + reopened, + CodexDirectoryEntryFingerprint::default(), + ))); + } else if closing.is_some_and(|stamp| !stamp.settled) + && resume_directories.is_none() + { + // An unfinished structural iterator is + // resumed below; only its final pass may + // certify coverage without a name witness. + scan.complete = false; } *current = None; } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 06da3302d4c..6dcfd0d7075 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -3023,6 +3023,18 @@ mod recent_first_discovery_tests { let mut finished = false; for _ in 0..2048 { + // Preserve the intentionally aliased stat through an arbitrarily + // slow test run: elapsed wall time must not make this regression + // pass by turning the native clock into a separate detection. + let alias = directory_stat_witness(&archive).unwrap().unwrap().stat; + if let Some(scan) = state.scan.as_mut() { + let directory = scan + .directories + .iter_mut() + .find(|directory| directory.path == archive) + .expect("retained archive directory identity"); + directory.identity.as_mut().unwrap().stat = alias; + } let pass = retained_pass(&source, &mut state, bounds, frontier); frontier = pass.next_frontier; if state.scan.is_none() { From 527319cb48ec91c34d1767255830fd4bf38d2416 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:20:48 -0800 Subject: [PATCH 16/49] style(sessions): format Codex verification traversal Apply the pinned formatter layout reported by repository gates. Validation: owning Bazel formatter target passed; independent review completed. --- .../src/runtime/hosts/codex.rs | 26 ++++++++++--------- .../src/runtime/hosts/codex/tests.rs | 5 +++- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 1808328b527..c8367524cd8 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2410,11 +2410,12 @@ fn retained_scan_step( match entries.next() { Some(entry) => { file_work += 1; - let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { - operation: "verify Codex transcript directory entry", - path: scan.directories[*directory_index].path.clone(), - source, - })?; + let entry = + entry.map_err(|source| TranscriptIngestError::ScanIo { + operation: "verify Codex transcript directory entry", + path: scan.directories[*directory_index].path.clone(), + source, + })?; let file_type = entry.file_type().map_err(|source| { TranscriptIngestError::ScanIo { operation: "verify Codex transcript entry type", @@ -2512,13 +2513,14 @@ fn retained_scan_step( // This initially unsettled listing needs an // independent current-name check, even if the // filesystem clock settled during the walk. - let reopened = std::fs::read_dir(&directory.path).map_err( - |source| TranscriptIngestError::ScanIo { - operation: "verify Codex transcript bucket", - path: directory.path.clone(), - source, - }, - )?; + let reopened = + std::fs::read_dir(&directory.path).map_err(|source| { + TranscriptIngestError::ScanIo { + operation: "verify Codex transcript bucket", + path: directory.path.clone(), + source, + } + })?; *verification = Some(Box::new(( *directory_index, reopened, diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 6dcfd0d7075..0b996c257e1 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2970,7 +2970,10 @@ mod recent_first_discovery_tests { break; } } - assert!(finished, "the bounded file traversal must reach its frontier"); + assert!( + finished, + "the bounded file traversal must reach its frontier" + ); } #[test] From 659a07a258ab10acc57c352be9b06063c81aec30 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:17:46 -0800 Subject: [PATCH 17/49] fix(sessions): keep Codex sweep witnesses stable across clock settle A budgeted Codex directory sweep compares its opening and closing stat identities. The directory identity encodes whether the change clock had settled, so a directory that settled mid-sweep looked changed even though its native change time was unchanged. The sweep then discarded a complete name witness and restarted, and the idle probe re-ran the sweep. Compare a settle-stable identity for sweep coverage, commit the closing witness so the idle probe compares against the certified state, and keep the settle-sensitive identity for idle change detection. Collapse the nested fingerprint commit to satisfy clippy::collapsible_if. Validation: bazel clippy on //crates/tracedecay-sessions/..., rustfmt check, and the Codex discovery unit tests. --- .../src/runtime/hosts/codex.rs | 136 ++++++++++++++---- .../src/runtime/hosts/codex/tests.rs | 16 ++- 2 files changed, 117 insertions(+), 35 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index c8367524cd8..6c3be987610 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -1069,6 +1069,7 @@ struct CodexDirectoryListing { #[derive(Clone, Copy)] struct CodexDirectoryWitness { stat: [u8; 32], + stable_stat: [u8; 32], entries: Option<[u8; 32]>, } @@ -2205,9 +2206,10 @@ fn retained_scan_step( } // Do not enumerate a potentially huge unsettled archive // just to obtain its witness before the bounded iterator. - let stat = directory_stat_witness(&path)?; + let stat = directory_scan_witness(&path)?; let identity = stat.as_ref().map(|witness| CodexDirectoryWitness { stat: witness.stat, + stable_stat: witness.stable_stat, entries: None, }); let directory_index = scan.directories.len(); @@ -2216,7 +2218,7 @@ fn retained_scan_step( root_order, identity, }); - let Some(opening_stat) = identity.map(|witness| witness.stat) else { + let Some(opening_stat) = identity.map(|witness| witness.stable_stat) else { continue; }; let listed = std::fs::read_dir(&path).map_err(|source| { @@ -2354,8 +2356,8 @@ fn retained_scan_step( // If the directory changed while we held it, the // iterator may omit new entries. Never publish a // complete corpus frontier from that mixed snapshot. - let closing_stat = directory_stat_witness(&directory.path)?; - if closing_stat.as_ref().map(|stat| stat.stat) + let closing_stat = directory_scan_witness(&directory.path)?; + if closing_stat.as_ref().map(|stat| stat.stable_stat) != Some(directory.opening_stat) { scan.complete = false; @@ -2375,6 +2377,13 @@ fn retained_scan_step( if let Some(fingerprint) = directory.fingerprint.take() { identity.entries = Some(fingerprint.finish()); } + // Commit the closing witness so the idle probe + // compares against the state this sweep actually + // certified, not a stale unsettled sample. + if let Some(closing) = closing_stat { + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + } } *current = None; } @@ -2427,15 +2436,29 @@ fn retained_scan_step( continue; } None => { - let directory = &scan.directories[*directory_index]; - let closing = directory_stat_witness(&directory.path)?; - if closing.as_ref().map(|stamp| stamp.stat) - != directory.identity.map(|identity| identity.stat) - || directory.identity.and_then(|identity| identity.entries) + let directory_index = *directory_index; + let closing = { + let directory = &scan.directories[directory_index]; + directory_scan_witness(&directory.path)? + }; + let identity = scan.directories[directory_index].identity; + if closing.as_ref().map(|stamp| stamp.stable_stat) + != identity.map(|identity| identity.stable_stat) + || identity.and_then(|identity| identity.entries) != Some(std::mem::take(fingerprint).finish()) { scan.complete = false; } + // Commit the closing witness so the idle + // probe compares against the state this + // sweep actually certified. + if let Some(closing) = closing + && let Some(identity) = + scan.directories[directory_index].identity.as_mut() + { + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + } *verification = None; continue; } @@ -2499,34 +2522,34 @@ fn retained_scan_step( // file enumeration finishes. Recheck its native // identity before granting full coverage: a // stale ReadDir may have omitted that entry. - let directory = &scan.directories[*directory_index]; - let closing = directory_stat_witness(&directory.path)?; - if closing.as_ref().map(|stat| stat.stat) - != directory.identity.map(|identity| identity.stat) + let directory_index = *directory_index; + let closing = { + let directory = &scan.directories[directory_index]; + directory_scan_witness(&directory.path)? + }; + let identity = scan.directories[directory_index].identity; + if closing.as_ref().map(|stat| stat.stable_stat) + != identity.map(|identity| identity.stable_stat) { scan.complete = false; - } else if directory - .identity - .and_then(|identity| identity.entries) - .is_some() - { + } else if identity.and_then(|identity| identity.entries).is_some() { // This initially unsettled listing needs an // independent current-name check, even if the // filesystem clock settled during the walk. - let reopened = - std::fs::read_dir(&directory.path).map_err(|source| { - TranscriptIngestError::ScanIo { - operation: "verify Codex transcript bucket", - path: directory.path.clone(), - source, - } - })?; + let path = scan.directories[directory_index].path.clone(); + let reopened = std::fs::read_dir(&path).map_err(|source| { + TranscriptIngestError::ScanIo { + operation: "verify Codex transcript bucket", + path, + source, + } + })?; *verification = Some(Box::new(( - *directory_index, + directory_index, reopened, CodexDirectoryEntryFingerprint::default(), ))); - } else if closing.is_some_and(|stamp| !stamp.settled) + } else if closing.as_ref().is_some_and(|stamp| !stamp.settled) && resume_directories.is_none() { // An unfinished structural iterator is @@ -2534,6 +2557,16 @@ fn retained_scan_step( // certify coverage without a name witness. scan.complete = false; } + // Commit the closing witness so the idle probe + // compares against the state this sweep actually + // certified, not a stale unsettled sample. + if let Some(closing) = closing + && let Some(identity) = + scan.directories[directory_index].identity.as_mut() + { + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + } *current = None; } } @@ -2737,6 +2770,16 @@ struct DirectoryStatWitness { settled: bool, } +/// A directory witness for a budgeted sweep. `stable_stat` drops the +/// settled/unsettled encoding so a directory that settles mid-sweep is not +/// mistaken for a changed one; its name witness still describes the same +/// entries. +struct DirectoryScanWitness { + stat: [u8; 32], + stable_stat: [u8; 32], + settled: bool, +} + fn codex_directory_changed( path: &Path, stored: Option, @@ -2768,11 +2811,12 @@ fn codex_directory_changed( #[cfg(test)] fn codex_directory_witness(path: &Path) -> TranscriptIngestResult> { - let Some(current) = directory_stat_witness(path)? else { + let Some(current) = directory_scan_witness(path)? else { return Ok(None); }; Ok(Some(CodexDirectoryWitness { stat: current.stat, + stable_stat: current.stable_stat, // In the settled case, a later stat comparison proves stability. // Avoid eagerly listing and sorting all historical rollout names; // an entry-name witness is needed only for a coarse-clock window. @@ -2784,7 +2828,7 @@ fn codex_directory_witness(path: &Path) -> TranscriptIngestResult TranscriptIngestResult> { +fn stat_directory(path: &Path) -> TranscriptIngestResult> { let Some(metadata) = stat_if_present(path, "stat Codex transcript directory")? else { return Ok(None); }; @@ -2798,6 +2842,13 @@ fn directory_stat_witness(path: &Path) -> TranscriptIngestResult TranscriptIngestResult> { + let Some(metadata) = stat_directory(path)? else { + return Ok(None); + }; let changed = change_stamp(path, &metadata); Ok(Some(DirectoryStatWitness { stat: codex_directory_stat_identity(path, &metadata, changed)?, @@ -2805,6 +2856,31 @@ fn directory_stat_witness(path: &Path) -> TranscriptIngestResult TranscriptIngestResult> { + let Some(metadata) = stat_directory(path)? else { + return Ok(None); + }; + let changed = change_stamp(path, &metadata); + Ok(Some(DirectoryScanWitness { + stat: codex_directory_stat_identity(path, &metadata, changed)?, + stable_stat: codex_directory_stable_identity(path, &metadata)?, + settled: changed.is_settled(), + })) +} + +/// The directory identity without the settled/unsettled encoding. A budgeted +/// sweep compares this across slices: a directory that settles mid-sweep has +/// not changed, and its name witness still describes the same entries. +fn codex_directory_stable_identity( + path: &Path, + metadata: &std::fs::Metadata, +) -> TranscriptIngestResult<[u8; 32]> { + let native_change_time = RewriteWitness::NATIVE + .change_time_nanos(metadata) + .map_or(0, |time| u128::from_le_bytes(time.to_le_bytes())); + codex_corpus_identity(path, metadata, ChangeStamp::Unvouched(native_change_time)) +} + // An unsettled ChangeStamp has a different random sample token on *every* // stat, even when the directory has not changed. That token is appropriate // for file-content authority, but would make an unchanged hot Codex directory diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 0b996c257e1..ea50300725b 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -1234,8 +1234,9 @@ mod recent_first_discovery_tests { IDLE_FULL_VALIDATION_CYCLES, MAX_EXACT_HOOK_SESSION_REQUESTS, MAX_EXACT_HOOK_SOURCE_AUTHORITIES, MAX_SCAN_DEPTH, codex_corpus_identity, codex_directory_changed, codex_directory_stat_identity, codex_directory_witness, - directory_entry_fingerprint, directory_stat_witness, indexed_replay_pass, - replay_index_entries_visited_for_test, reset_replay_index_entries_visited_for_test, + directory_entry_fingerprint, directory_scan_witness, directory_stat_witness, + indexed_replay_pass, replay_index_entries_visited_for_test, + reset_replay_index_entries_visited_for_test, }; use crate::runtime::ingest::quarantine::SourceSweep; use crate::runtime::jsonl_observation_admission::{ @@ -3010,7 +3011,9 @@ mod recent_first_discovery_tests { std::fs::write(&late, b"{}\n").unwrap(); // Model two writes within one filesystem timestamp quantum: the // directory stat aliases the scan's authority despite a new file. - let alias = directory_stat_witness(&archive).unwrap().unwrap().stat; + let witness = directory_scan_witness(&archive).unwrap().unwrap(); + let alias = witness.stat; + let alias_stable = witness.stable_stat; let scan = state.scan.as_mut().unwrap(); let CodexScanPhase::Files { current: Some(current), @@ -3022,6 +3025,7 @@ mod recent_first_discovery_tests { let directory_index = current.0; let identity = scan.directories[directory_index].identity.as_mut().unwrap(); identity.stat = alias; + identity.stable_stat = alias_stable; identity.entries = Some(original_names); let mut finished = false; @@ -3029,14 +3033,16 @@ mod recent_first_discovery_tests { // Preserve the intentionally aliased stat through an arbitrarily // slow test run: elapsed wall time must not make this regression // pass by turning the native clock into a separate detection. - let alias = directory_stat_witness(&archive).unwrap().unwrap().stat; + let witness = directory_scan_witness(&archive).unwrap().unwrap(); if let Some(scan) = state.scan.as_mut() { let directory = scan .directories .iter_mut() .find(|directory| directory.path == archive) .expect("retained archive directory identity"); - directory.identity.as_mut().unwrap().stat = alias; + let identity = directory.identity.as_mut().unwrap(); + identity.stat = witness.stat; + identity.stable_stat = witness.stable_stat; } let pass = retained_pass(&source, &mut state, bounds, frontier); frontier = pass.next_frontier; From a41f5982007dba7a19497a6d441a8ca8a577ca2e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:58:21 -0800 Subject: [PATCH 18/49] fix(sessions): refuse incomplete chunked Codex coverage Refs #4756. Clear an unsettled directory identity when its file walk lacks an entry-name witness, including before a retained structural walk resumes, so a coarse-clock alias cannot publish complete coverage. Add a deterministic chunked-archive regression that exhausts the old file iterator before creating the rollout. --- .../src/runtime/hosts/codex.rs | 11 +- .../src/runtime/hosts/codex/tests.rs | 102 ++++++++++++++++++ 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 7867c68e7fc..8dad2236057 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2557,13 +2557,12 @@ fn retained_scan_step( reopened, CodexDirectoryEntryFingerprint::default(), ))); - } else if closing.as_ref().is_some_and(|stamp| !stamp.settled) - && resume_directories.is_none() - { - // An unfinished structural iterator is - // resumed below; only its final pass may - // certify coverage without a name witness. + } else if closing.as_ref().is_some_and(|stamp| !stamp.settled) { + // Neither an unsettled native stat nor a + // resumed structural walk can replace the + // missing entry-name witness. scan.complete = false; + scan.directories[directory_index].identity = None; } // Commit the closing witness so the idle probe // compares against the state this sweep actually diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index c94c28c8e7f..dbde79bf2d0 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -3075,6 +3075,108 @@ mod recent_first_discovery_tests { assert!(finished, "the bounded verification must terminate"); } + #[test] + #[cfg(target_os = "linux")] + fn chunked_archive_alias_cannot_complete_without_file_name_witness() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(archive.join("deferred")).unwrap(); + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let source = CodexSource::with_home(temp.path()); + let mut bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let metadata_charge = std::mem::size_of::() as u64; + let sessions = temp.path().join(".codex/sessions"); + // Retain both roots, but force the archive's child into a later + // structural chunk while the archive's file walk owns the iterator. + bounds.max_discovery_bytes = + crate::runtime::hosts::codex::candidate_charge(&sessions, metadata_charge).unwrap() + + crate::runtime::hosts::codex::candidate_charge(&archive, metadata_charge) + .unwrap(); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let mut suspended = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_mut() + && let CodexScanPhase::Files { + current: Some(current), + resume_directories: Some(_), + .. + } = &mut scan.phase + && scan.directories[current.0].path == archive + { + let identity = scan.directories[current.0].identity.as_ref().unwrap(); + assert!( + identity.entries.is_none(), + "retention must interrupt the structural name witness" + ); + // A native ReadDir may buffer every old name before the late + // creation. Exhaust it now so omission is deterministic. + for entry in current.1.by_ref() { + entry.unwrap(); + } + suspended = true; + break; + } + } + assert!(suspended, "the chunked file reader must be retained"); + + let arrived = archive.join("rollout-after-chunked-file-reader-open.jsonl"); + std::fs::write(&arrived, b"{}\n").unwrap(); + let closing = directory_scan_witness(&archive).unwrap().unwrap(); + assert!(!closing.settled, "the fixture needs the coarse clock"); + let scan = state.scan.as_mut().unwrap(); + let CodexScanPhase::Files { + current: Some(current), + resume_directories: Some(resume), + .. + } = &mut scan.phase + else { + panic!("the chunked file reader must still be retained"); + }; + // Model a directory timestamp that aliases the newly added entry in + // both the file phase and the structural iterator it will resume. + // With no structural name witness, a complete coverage claim would be + // based only on the stale file listing. + let identity = scan.directories[current.0].identity.as_mut().unwrap(); + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + let resumed = resume.current.as_deref_mut().unwrap(); + assert_eq!(resumed.path, archive); + resumed.opening_stat = closing.stable_stat; + assert!(scan.complete); + + let mut observed = BTreeSet::new(); + let mut finished = false; + for _ in 0..2048 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if state.scan.is_none() { + assert!( + observed.contains(&arrived) || !frontier.is_complete(), + "the resumed structural walk cannot vouch for rollout files" + ); + finished = true; + break; + } + } + assert!(finished, "the chunked sweep must terminate"); + + assert!(spin_until_jsonl_change_settled(&archive)); + for _ in 0..2048 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!(frontier.is_complete(), "a fresh sweep must converge"); + assert!(observed.contains(&arrived)); + } + #[test] #[cfg(unix)] fn verification_cannot_settle_over_an_aliased_new_rollout() { From 0c88051ed79ddd5edee0bf2e7078f18ae8a406c1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:06:02 -0800 Subject: [PATCH 19/49] fix(sessions): fence chunked Codex clock transitions --- .../src/runtime/hosts/codex.rs | 10 +++++++--- .../src/runtime/hosts/codex/tests.rs | 16 +++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 8dad2236057..af3e724580e 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -2557,10 +2557,14 @@ fn retained_scan_step( reopened, CodexDirectoryEntryFingerprint::default(), ))); - } else if closing.as_ref().is_some_and(|stamp| !stamp.settled) { + } else if closing.as_ref().is_some_and(|stamp| !stamp.settled) + || closing.as_ref().map(|stamp| stamp.stat) + != identity.map(|identity| identity.stat) + { // Neither an unsettled native stat nor a - // resumed structural walk can replace the - // missing entry-name witness. + // transition to settled authority can vouch + // for a retained listing without a name + // witness, even if structural work resumes. scan.complete = false; scan.directories[directory_index].identity = None; } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index dbde79bf2d0..4fc8400443b 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -3124,8 +3124,10 @@ mod recent_first_discovery_tests { let arrived = archive.join("rollout-after-chunked-file-reader-open.jsonl"); std::fs::write(&arrived, b"{}\n").unwrap(); + assert!(spin_until_jsonl_change_settled(&archive)); + let metadata = std::fs::metadata(&archive).unwrap(); let closing = directory_scan_witness(&archive).unwrap().unwrap(); - assert!(!closing.settled, "the fixture needs the coarse clock"); + assert!(closing.settled); let scan = state.scan.as_mut().unwrap(); let CodexScanPhase::Files { current: Some(current), @@ -3135,12 +3137,13 @@ mod recent_first_discovery_tests { else { panic!("the chunked file reader must still be retained"); }; - // Model a directory timestamp that aliases the newly added entry in - // both the file phase and the structural iterator it will resume. - // With no structural name witness, a complete coverage claim would be - // based only on the stale file listing. + // Model an unsettled opening sample whose native timestamp aliases + // the new entry, then closes after settlement. Derive it from the real + // directory metadata so scheduler delay cannot choose the clock branch. + // The resumed structural walk cannot vouch for the stale file listing. let identity = scan.directories[current.0].identity.as_mut().unwrap(); - identity.stat = closing.stat; + identity.stat = + codex_directory_stat_identity(&archive, &metadata, ChangeStamp::Unvouched(0)).unwrap(); identity.stable_stat = closing.stable_stat; let resumed = resume.current.as_deref_mut().unwrap(); assert_eq!(resumed.path, archive); @@ -3164,7 +3167,6 @@ mod recent_first_discovery_tests { } assert!(finished, "the chunked sweep must terminate"); - assert!(spin_until_jsonl_change_settled(&archive)); for _ in 0..2048 { let pass = retained_pass(&source, &mut state, bounds, frontier); observed.extend(pass.report.paths); From e891f50f646a4043656fb3dd69f8990bdec6dec8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:31:09 -0800 Subject: [PATCH 20/49] test(global-db): retire cancellation fixture before reopening Join the fixture runtime retirement before reopening SQLite for schema assertions. Capture the admission baseline on the same published runtime and retain cancellation rollback and successful reopen coverage. --- .../src/tests/schema_install_cancellation.rs | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index f40a96a4cbe..d0bde77fbe1 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -12,6 +12,9 @@ use tracedecay_runtime_core::db::{ DaemonDatabaseScope, Database, DatabaseAuthority, RegisteredTestRuntimeFixtureV1, TestDatabaseRuntimeMode, TestDatabaseRuntimeScope, }; +use tracedecay_runtime_core::shard_runtime::registry::{ + StoreRuntimeRetirementOutcome, StoreRuntimeRetirementResult, +}; use super::harness::TEST_RUNTIME_NONCE; use crate::RegisteredGlobalDbOwnerV1; @@ -83,8 +86,14 @@ impl EmptyStore { /// `cancel_at`-th poll, so sweeping `cancel_at` samples cancellation /// points across admission. Returns the outcome and how many polls /// admission took. - async fn attach_cancelled_at(&self, cancel_at: usize) -> (Result<(), TraceDecayError>, usize) { - let (owner, _runtime, _retirement) = self.publish().await.into_parts(); + async fn attach_cancelled_at( + &self, + cancel_at: usize, + ) -> (Result<(), TraceDecayError>, usize, Vec) { + let (owner, runtime, retirement) = self.publish().await.into_parts(); + // Publishing installs the writer ledger. Record the actual admission + // baseline on this runtime, rather than opening and dropping another. + let before_admission = self.schema_objects(); let cancellation = CancellationToken::new(); let mut admission = Box::pin(RegisteredGlobalDbOwnerV1::admit_and_attach_for_daemon( owner, @@ -102,14 +111,40 @@ impl EmptyStore { } }) .await; - (outcome, polls) + drop(admission); + drop(runtime); + // A reopen must follow a joined shutdown, not race the previous + // fixture's writer and readers on the same SQLite path. + let mut reservation = match retirement + .registry() + .reserve_retirement_batch(vec![retirement.retirement_target()]) + { + StoreRuntimeRetirementResult::Reserved(reservation) => reservation, + StoreRuntimeRetirementResult::Blocked(refusal) => { + panic!("completed admission retained runtime clients: {refusal:?}") + } + }; + let committed = + tracedecay_runtime_core::native_runtime::spawn_blocking(move || reservation.commit()) + .await + .unwrap() + .unwrap(); + assert!( + matches!( + committed.outcomes(), + [StoreRuntimeRetirementOutcome::Closed { .. }] + ), + "fixture runtime must close before reopening: {:?}", + committed.outcomes() + ); + (outcome, polls, before_admission) } } #[asupersync::test(flavor = "multi_thread")] async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { let reference = EmptyStore::create(); - let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; + let (outcome, total_polls, _) = reference.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); let installed = reference.schema_objects(); assert!( @@ -123,11 +158,7 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { let mut completed = Vec::new(); for cancel_at in (1..total_polls).step_by(stride).chain([total_polls + 10]) { let store = EmptyStore::create(); - // Publishing the runtime installs its writer ledger before admission. - // Cancellation must preserve exactly that pre-admission schema. - drop(store.publish().await); - let before_admission = store.schema_objects(); - let (outcome, _) = store.attach_cancelled_at(cancel_at).await; + let (outcome, _, before_admission) = store.attach_cancelled_at(cancel_at).await; let objects = store.schema_objects(); match outcome { Err(error) => { @@ -155,7 +186,7 @@ async fn cancelled_daemon_attach_rolls_back_schema_and_reopens() { completed.push(cancel_at); } } - let (reopened, _) = store.attach_cancelled_at(usize::MAX).await; + let (reopened, _, _) = store.attach_cancelled_at(usize::MAX).await; reopened.unwrap_or_else(|error| { panic!("reopen after cancel at poll {cancel_at}/{total_polls} failed: {error:?}") }); From b2f65b1a7a13cac467b815c4aa7377405db22dff Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:22:28 -0800 Subject: [PATCH 21/49] test(global-db): batch overlap reconciliation fixture writes Preserve all 10,001 overlap rows and paging assertions while using the canonical batch transaction for setup. The identical repair measured 93.160s to 3.269s in isolation and passed the base 501-test suite in 145.90s within the unchanged deadline. Exact-head CI remains required. --- .../src/observation_projection/rebuild.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index beb861a9d9a..af96c31baf2 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2693,18 +2693,16 @@ mod activation_tests { let harness = RegisteredGlobalDbHarness::open("session-overlap-paging").await; let mut staged_rows = Vec::with_capacity(OVERLAPS_PAST_TRANSPORT_ROW_CAP as usize); for index in 0..OVERLAPS_PAST_TRANSPORT_ROW_CAP { - let session_id = format!("session.{index:05}"); - let active = SessionRecord { - session_id: session_id.clone(), - ..session(None, None) - }; - assert!(harness.registered.upsert_session(&active).await); staged_rows.push(SessionRecord { - session_id, - title: Some(format!("composer {index:05}")), + session_id: format!("session.{index:05}"), ..session(None, None) }); } + // Fixture setup needs one durable batch, not a commit per overlap. + assert!(harness.registered.upsert_sessions(&staged_rows).await); + for (index, staged) in staged_rows.iter_mut().enumerate() { + staged.title = Some(format!("composer {index:05}")); + } let transaction = harness.registered.begin_write_transaction().await.unwrap(); transaction .execute( From da7b91fe3c41b8b1d9dcb84466298a418b8195ca Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:35:00 -0800 Subject: [PATCH 22/49] fix(sessions): deduplicate Codex witness test imports --- crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 5650a3781bd..c5d928399ff 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -1238,8 +1238,6 @@ mod recent_first_discovery_tests { directory_scan_witness, directory_stat_witness, indexed_replay_pass, replay_index_entries_visited_for_test, reset_replay_index_entries_visited_for_test, }; - #[cfg(unix)] - use crate::runtime::hosts::codex::{DirectoryStatWitness, codex_directory_changed_since}; use crate::runtime::ingest::quarantine::SourceSweep; use crate::runtime::jsonl_observation_admission::{ SHARED_JSONL_PIN_GATES, install_test_shared_jsonl_preparation_authority, From cb684f75ddf43d17d7b0ef50cc7501fcba666eef Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:56:25 -0800 Subject: [PATCH 23/49] test(sessions): make coarse fingerprint regression deterministic --- .../src/runtime/hosts/codex/tests.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index c5d928399ff..b3dd015c0e2 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -3393,10 +3393,6 @@ mod recent_first_discovery_tests { let eligible = archive.join("rollout-eligible.jsonl"); std::fs::write(&eligible, b"{}\n").unwrap(); let expected = directory_entry_fingerprint(&archive).unwrap(); - assert!( - !directory_stat_witness(&archive).unwrap().unwrap().settled, - "the fixture must exercise the coarse-clock name-witness path" - ); let source = CodexSource::with_home(temp.path()); let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); @@ -3404,11 +3400,11 @@ mod recent_first_discovery_tests { let mut frontier = CodexDiscoveryFrontier::initial(); let first = retained_pass(&source, &mut state, bounds, frontier); assert!(first.report.is_truncated()); - let scan = state.scan.as_ref().expect("unfinished structural scan"); + let scan = state.scan.as_mut().expect("unfinished structural scan"); let CodexScanPhase::Directories { current: Some(current), .. - } = &scan.phase + } = &mut scan.phase else { panic!("the archive must retain a live directory iterator"); }; @@ -3421,6 +3417,10 @@ mod recent_first_discovery_tests { .is_none(), "no eager whole-archive fingerprint before bounded traversal" ); + // Model an initially unsettled native witness after the structural + // pass so scheduler delay cannot select the settled no-name-fingerprint + // path. The bounded iterator assembles the same complete fingerprint. + current.fingerprint.get_or_insert_with(Default::default); frontier = first.next_frontier; let mut covered = BTreeSet::new(); From 7a612623f842c68b2de531fe034b020fe48c4085 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:24:21 -0800 Subject: [PATCH 24/49] test(dashboard): bind cancellation checks to the owned listener --- .../src/server_tests.rs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-dashboard-api/src/server_tests.rs b/crates/tracedecay-dashboard-api/src/server_tests.rs index 3fe16fe23c6..b999feae9f3 100644 --- a/crates/tracedecay-dashboard-api/src/server_tests.rs +++ b/crates/tracedecay-dashboard-api/src/server_tests.rs @@ -81,7 +81,7 @@ fn loopback_server_drains_in_flight_request_after_shutdown() { /// Cancelling the listener must abort the connection tasks it owns, including /// a handler that cannot finish gracefully. -#[tokio::test] +#[tokio::test(flavor = "current_thread")] async fn cancelling_the_loopback_server_drops_its_open_connections() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -122,13 +122,30 @@ async fn cancelling_the_loopback_server_drops_its_open_connections() { .await .expect("the connection must admit the held handler"); + // Connect without yielding this current-thread runtime, so the socket + // remains in this listener's accept backlog when cancellation lands. + // Reconnecting after shutdown could reach another test that reused the + // ephemeral port and would not identify the listener being checked. + let queued_peer = TcpStream::connect_timeout(&address, bound).unwrap(); + queued_peer.set_nonblocking(true).unwrap(); + let mut queued_peer = tokio::net::TcpStream::from_std(queued_peer).unwrap(); server.abort(); assert!(server.await.unwrap_err().is_cancelled()); tokio::time::timeout(bound, dropped.notified()) .await .expect("listener cancellation must drop its active handler"); - assert!(tokio::net::TcpStream::connect(address).await.is_err()); let mut byte = [0]; + let queued_received = tokio::time::timeout(bound, queued_peer.read(&mut byte)) + .await + .expect("listener cancellation must close its unaccepted socket"); + assert!( + matches!(queued_received, Ok(0)) + || matches!(queued_received, Err(ref error) if matches!( + error.kind(), + std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::ConnectionAborted + )), + "the cancelled listener must close its backlog: {queued_received:?}" + ); let received = tokio::time::timeout(bound, peer.read(&mut byte)) .await .expect("listener cancellation must close its accepted socket"); From 1c4845965d22e71034b53a0fa841125066ab2b93 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:37:36 -0800 Subject: [PATCH 25/49] test(db): reuse published schema for cancellation sweep --- .../src/tests/schema_install_cancellation.rs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 64b90ebeb1e..46e055a7a75 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -96,7 +96,7 @@ impl RegisteredStore { /// Inspect a completed family with an ordinary independent read-only SQLite /// connection. Reopening the registered runtime for every observation would /// perform another full attachment lifecycle and obscure cancellation timing. - fn external_schema_objects(&self) -> Vec { + fn external_schema_objects(&self) -> Vec<(String, Option)> { let connection = rusqlite::Connection::open_with_flags( &self.path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, @@ -104,12 +104,12 @@ impl RegisteredStore { .unwrap(); let mut statement = connection .prepare( - "SELECT type || ':' || name FROM sqlite_schema + "SELECT type || ':' || name, sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", ) .unwrap(); statement - .query_map([], |row| row.get::<_, String>(0)) + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() .map(Result::unwrap) .collect() @@ -185,12 +185,14 @@ async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { #[tokio::test(flavor = "multi_thread")] async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { - let reference = RegisteredStore::create(); - let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; + let store = RegisteredStore::create(); + let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); - let installed = reference.external_schema_objects(); + let installed = store.external_schema_objects(); assert!( - installed.contains(&"table:code_projects".to_owned()), + installed + .iter() + .any(|(object, _)| object == "table:code_projects"), "a completed attach installs the registered schema: {installed:?}" ); @@ -201,12 +203,11 @@ async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { .chain((2..total_polls).step_by(stride)) .chain([total_polls + 10]) { - let store = RegisteredStore::create(); - // Cancellation cannot modify the already-published schema or - // prevent the next admission from reopening the same family. - let (owner, runtime, retirement) = store.publish().await.into_parts(); - drop((owner, runtime)); - retire_runtime(retirement).await; + // The reference admission, then each successful reopen below, leave + // this family closed with the exact baseline schema. Creating another + // family would repeat initialization without exercising another + // cancellation point. Keep real runtime retirement and reopening + // around every sampled admission. let before_admission = store.external_schema_objects(); assert_eq!(before_admission, installed); let (outcome, _) = store.attach_cancelled_at(cancel_at).await; From 6fcad5ef2e368351a7f5058f167511ef90a8491c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:50:13 -0800 Subject: [PATCH 26/49] test(sessions): capture compaction publication diagnostics --- .../codex_compaction.rs | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs index 1abbd6d64e6..c06eefe1a47 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs @@ -20,7 +20,7 @@ use tracedecay_runtime_core::test_executable::write_executable_script; use tracedecay_sessions::runtime::{SessionProvider, with_transcript_source_profile}; #[cfg(unix)] -use crate::common::{spawn_tracedecay_daemon, tracedecay_command_with_home}; +use crate::common::{spawn_tracedecay_daemon_logged, tracedecay_command_with_home}; #[cfg(unix)] use crate::restart_atomicity::mark_test_project; use crate::support::setup; @@ -182,19 +182,27 @@ done ) .unwrap(); let project_id = mark_test_project(&project); + let daemon_log = tmp.path().join("codex-compaction-daemon.log"); + let daemon = spawn_tracedecay_daemon_logged(&home, &daemon_log, |command| { + command + .env( + "RUST_LOG", + "info,tracedecay_session_runtime::session_temporal_refresh_scheduler=trace,tracedecay_mcp::handlers::hook_runtime=trace", + ) + .env("TRACEDECAY_SPAN_TIMINGS", "1"); + }); // The hook resolves the project root through the initialized-store gate, // exactly like production installs: `init` creates the project store and // admits the canonical daemon-owned project route. crate::common::initialize_tracedecay_cli_project(&home, &project); // The daemon launches a summarizer only through the project's // `lcm.summarizer_executables.v1` binding, so the fake `codex` is bound - // through the production configuration surface while the managed daemon - // from init is still serving the project. + // through the production configuration surface while the same daemon + // is still serving the project. configure_codex_summarizer(&home, &project, &project_id, &codex_bin); - // Keep that production owner through the hook. An in-process enrollment + // Keep the logged production owner through the hook. An in-process enrollment // adds no fixture data and would require retiring and cold-opening the // same stores again just before the hook's bounded publication join. - let daemon = spawn_tracedecay_daemon(&home); write_codex_rollout_with_compaction(&home, &project, "codex-compact"); let event = serde_json::json!({ @@ -216,15 +224,17 @@ done .unwrap(); writeln!(child.stdin.take().unwrap(), "{event}").unwrap(); let output = child.wait_with_output().unwrap(); + let daemon_diagnostics = std::fs::read_to_string(&daemon_log) + .unwrap_or_else(|error| format!("daemon log unavailable: {error}")); assert!( output.status.success(), - "Codex PostCompact hook failed: {}", - String::from_utf8_lossy(&output.stderr) + "Codex PostCompact hook failed: {}; daemon log: {daemon_diagnostics}", + String::from_utf8_lossy(&output.stderr), ); assert!( !String::from_utf8_lossy(&output.stderr).contains("PostCompact daemon call failed"), - "the PostCompact hook must not fail open around the daemon effect: {}", - String::from_utf8_lossy(&output.stderr) + "the PostCompact hook must not fail open around the daemon effect: {}; daemon log: {daemon_diagnostics}", + String::from_utf8_lossy(&output.stderr), ); assert_eq!( std::fs::read_to_string(codex_bin.with_extension("launches")).unwrap(), From d749b161e4a0fb9bf3f97cb87c8be0a54fbf2e45 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:50:22 -0800 Subject: [PATCH 27/49] test(lsp): capture receipt admission phases on ACK failure --- .../src/invocation/lsp_delivery.rs | 20 ++++++++++++++++++- .../runtime_surface_acceptance.rs | 13 ++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs index 83be58f6b98..fc4e5bfd6b8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs +++ b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs @@ -114,10 +114,28 @@ impl LspDeliverySettlementClaim { self, project_lease: ProjectRuntimeRequestLeaseV1, ) -> impl Future + Send { + let owner_event_id = self.settlement.attempt.owner_event_id.clone(); + tracing::trace!( + %owner_event_id, + phase = "submit", + "LSP delivery receipt admission" + ); let admission = tracedecay_runtime_core::native_runtime::on_blocking_pool( async move { let _project_lease = project_lease; - self.admit_inline() + tracing::trace!( + %owner_event_id, + phase = "started", + "LSP delivery receipt admission" + ); + let outcome = self.admit_inline(); + tracing::trace!( + %owner_event_id, + phase = "completed", + ?outcome, + "LSP delivery receipt admission" + ); + outcome } .with_current_subscriber(), ); diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 8d0be3c8460..3737ab21b4f 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -136,9 +136,13 @@ impl RuntimeFixture { } } - fn daemon_log_tail(&self) -> String { + fn daemon_log(&self) -> String { std::fs::read_to_string(self._environment.scratch().join("runtime-daemon.log")) .expect("read isolated runtime daemon log") + } + + fn daemon_log_tail(&self) -> String { + self.daemon_log() .lines() .rev() .take(160) @@ -224,7 +228,8 @@ async fn lsp_runtime_fixture() -> RuntimeFixture { command .env( "RUST_LOG", - "info,tracedecay_daemon_service::invocation=trace", + "info,tracedecay_daemon_service::invocation=trace,\ + tracedecay_private_fs::framed_log=trace", ) .env("TRACEDECAY_SPAN_TIMINGS", "1") .stderr(daemon_log); @@ -439,10 +444,10 @@ async fn poll_lsp_response( .unwrap_or_else(|error| { panic!( "acknowledge daemon LSP frame while awaiting response {response_id}: \ - {error:?}; frame id={:?}, method={:?}; daemon log tail:\n{}", + {error:?}; frame id={:?}, method={:?}; daemon log:\n{}", value.get("id"), value.get("method"), - fixture.daemon_log_tail(), + fixture.daemon_log(), ) }); if value.get("id").and_then(Value::as_u64) == Some(response_id) { From cf5fad3b24720ed1c98415d3cd943b4a1c1392de Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:21:26 -0800 Subject: [PATCH 28/49] test(code-index): use native runtime for generation cursor journey Run the persistent generation cursor journey on the canonical native runtime while retaining its two workers. The native graph scheduler and its retained settlement tasks inherit the ambient runtime, so this fixture must use the daemon's wake policy. Keep the async journey body, twenty-second readiness bounds, cursor identity checks, and joined owner shutdown unchanged. --- .../tests/code_index_graph_activation.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs b/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs index ed51beb1bd1..0a9f3941425 100644 --- a/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs +++ b/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs @@ -470,8 +470,17 @@ async fn persistent_graph_activation_publishes_a_small_generation() { .expect("join graph reconciliation tasks"); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn persistent_callers_cursor_keeps_generation_a_without_repointing_generation_b() { +#[test] +fn persistent_callers_cursor_keeps_generation_a_without_repointing_generation_b() { + // Native tasks inherit this runtime, including the daemon's wake policy. + tracedecay_runtime_core::native_runtime::default_builder() + .worker_threads(2) + .build() + .expect("persistent graph cursor runtime") + .block_on(persistent_callers_cursor_across_generations()); +} + +async fn persistent_callers_cursor_across_generations() { let fixture = GitFixture::new(&[( "src/lib.rs", "pub fn hub() {}\npub fn caller_a() { hub(); }\npub fn caller_b() { hub(); }\n", From ce341e5483f28411dbaa023741698808adb042c1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:25:06 -0800 Subject: [PATCH 29/49] test(global-db): use native runtime for cancellation sweep The registered-runtime fixture inherits the ambient Tokio handle for native settlement and retirement. Build that owned runtime through the maintained native policy instead of a raw Tokio test macro. Preserve the default worker count, runtime lifetime, cancellation samples, explicit retirement joins, exact schema checks, and suite deadline. --- .../src/tests/schema_install_cancellation.rs | 121 +++++++++--------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 46e055a7a75..b7e3aabe99a 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -183,64 +183,71 @@ async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { assert_eq!(std::fs::read(&wal).unwrap(), foreign_wal); } -#[tokio::test(flavor = "multi_thread")] -async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { - let store = RegisteredStore::create(); - let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; - outcome.unwrap(); - let installed = store.external_schema_objects(); - assert!( - installed - .iter() - .any(|(object, _)| object == "table:code_projects"), - "a completed attach installs the registered schema: {installed:?}" - ); +#[test] +fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { + tracedecay_runtime_core::native_runtime::default_builder() + .build() + .expect("schema cancellation runtime") + .block_on(async { + let store = RegisteredStore::create(); + let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; + outcome.unwrap(); + let installed = store.external_schema_objects(); + assert!( + installed + .iter() + .any(|(object, _)| object == "table:code_projects"), + "a completed attach installs the registered schema: {installed:?}" + ); - let stride = (total_polls / 20).max(1); - let mut cancelled = Vec::new(); - let mut completed = Vec::new(); - for cancel_at in std::iter::once(1) - .chain((2..total_polls).step_by(stride)) - .chain([total_polls + 10]) - { - // The reference admission, then each successful reopen below, leave - // this family closed with the exact baseline schema. Creating another - // family would repeat initialization without exercising another - // cancellation point. Keep real runtime retirement and reopening - // around every sampled admission. - let before_admission = store.external_schema_objects(); - assert_eq!(before_admission, installed); - let (outcome, _) = store.attach_cancelled_at(cancel_at).await; - let objects = store.external_schema_objects(); - match outcome { - Err(error) => { - assert!( - error.is_store_open_cancelled(), - "cancel at poll {cancel_at}/{total_polls} must fail typed: {error:?}" - ); - assert_eq!( - objects, before_admission, - "cancel at poll {cancel_at}/{total_polls} must preserve the published schema" - ); - cancelled.push(cancel_at); - } - Ok(()) => { - assert_eq!( - objects, installed, - "an admission completed before cancel at poll {cancel_at}/{total_polls} preserves the published schema" - ); - completed.push(cancel_at); + let stride = (total_polls / 20).max(1); + let mut cancelled = Vec::new(); + let mut completed = Vec::new(); + for cancel_at in std::iter::once(1) + .chain((2..total_polls).step_by(stride)) + .chain([total_polls + 10]) + { + // The reference admission, then each successful reopen below, leave + // this family closed with the exact baseline schema. Creating another + // family would repeat initialization without exercising another + // cancellation point. Keep real runtime retirement and reopening + // around every sampled admission. + let before_admission = store.external_schema_objects(); + assert_eq!(before_admission, installed); + let (outcome, _) = store.attach_cancelled_at(cancel_at).await; + let objects = store.external_schema_objects(); + match outcome { + Err(error) => { + assert!( + error.is_store_open_cancelled(), + "cancel at poll {cancel_at}/{total_polls} must fail typed: {error:?}" + ); + assert_eq!( + objects, before_admission, + "cancel at poll {cancel_at}/{total_polls} must preserve the published schema" + ); + cancelled.push(cancel_at); + } + Ok(()) => { + assert_eq!( + objects, installed, + "an admission completed before cancel at poll {cancel_at}/{total_polls} preserves the published schema" + ); + completed.push(cancel_at); + } + } + let (reopened, _) = store.attach_cancelled_at(usize::MAX).await; + reopened.unwrap_or_else(|error| { + panic!( + "reopen after cancel at poll {cancel_at}/{total_polls} failed: {error:?}" + ) + }); + assert_eq!(store.external_schema_objects(), installed); } - } - let (reopened, _) = store.attach_cancelled_at(usize::MAX).await; - reopened.unwrap_or_else(|error| { - panic!("reopen after cancel at poll {cancel_at}/{total_polls} failed: {error:?}") + assert!( + cancelled.contains(&1), + "a cancel before admission starts must stop it; cancelled at {cancelled:?}" + ); + assert!(completed.contains(&(total_polls + 10))); }); - assert_eq!(store.external_schema_objects(), installed); - } - assert!( - cancelled.contains(&1), - "a cancel before admission starts must stop it; cancelled at {cancelled:?}" - ); - assert!(completed.contains(&(total_polls + 10))); } From b61d9d9c086cdb07f754c31e6dd384878816e76d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:25:26 -0800 Subject: [PATCH 30/49] test(sessions): remove temporary acceptance tracing Remove the investigation's daemon tracing overrides after the compaction and runtime acceptance suites passed freshly. Retain their logged daemon owners, complete failure output and all behavioral assertions. Restore the LSP fixture's pre-existing invocation trace configuration; remove only the added private-filesystem filter. Operator receipt phase events remain available through the maintained tracing surface. --- .../runtime_surface_acceptance.rs | 3 +-- .../tests/transcript_ingest_suite/codex_compaction.rs | 9 +-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 3737ab21b4f..1fea2c8882a 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -228,8 +228,7 @@ async fn lsp_runtime_fixture() -> RuntimeFixture { command .env( "RUST_LOG", - "info,tracedecay_daemon_service::invocation=trace,\ - tracedecay_private_fs::framed_log=trace", + "info,tracedecay_daemon_service::invocation=trace", ) .env("TRACEDECAY_SPAN_TIMINGS", "1") .stderr(daemon_log); diff --git a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs index c06eefe1a47..45c6f603565 100644 --- a/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs +++ b/crates/tracedecay/tests/transcript_ingest_suite/codex_compaction.rs @@ -183,14 +183,7 @@ done .unwrap(); let project_id = mark_test_project(&project); let daemon_log = tmp.path().join("codex-compaction-daemon.log"); - let daemon = spawn_tracedecay_daemon_logged(&home, &daemon_log, |command| { - command - .env( - "RUST_LOG", - "info,tracedecay_session_runtime::session_temporal_refresh_scheduler=trace,tracedecay_mcp::handlers::hook_runtime=trace", - ) - .env("TRACEDECAY_SPAN_TIMINGS", "1"); - }); + let daemon = spawn_tracedecay_daemon_logged(&home, &daemon_log, |_| {}); // The hook resolves the project root through the initialized-store gate, // exactly like production installs: `init` creates the project store and // admits the canonical daemon-owned project route. From 408a89e2e5ca1d61ab6e57af327a6b587f3b3d5a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:35 -0800 Subject: [PATCH 31/49] test(sessions): start archive fingerprint with its iterator --- .../src/runtime/hosts/codex/tests.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index b3dd015c0e2..50d61cf17b5 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -3392,6 +3392,7 @@ mod recent_first_discovery_tests { } let eligible = archive.join("rollout-eligible.jsonl"); std::fs::write(&eligible, b"{}\n").unwrap(); + spin_until_jsonl_change_settled(&eligible); let expected = directory_entry_fingerprint(&archive).unwrap(); let source = CodexSource::with_home(temp.path()); @@ -3417,10 +3418,11 @@ mod recent_first_discovery_tests { .is_none(), "no eager whole-archive fingerprint before bounded traversal" ); - // Model an initially unsettled native witness after the structural - // pass so scheduler delay cannot select the settled no-name-fingerprint - // path. The bounded iterator assembles the same complete fingerprint. - current.fingerprint.get_or_insert_with(Default::default); + // Model an initially unsettled directory witness independently of the + // native clock. Restart its iterator with the empty fingerprint so + // entries consumed by the first pass cannot be omitted from the witness. + current.entries = std::fs::read_dir(&archive).unwrap(); + current.fingerprint = Some(Default::default()); frontier = first.next_frontier; let mut covered = BTreeSet::new(); From a50b826749ced99e006a8b7f5b342f59132d2ade Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:27:25 -0800 Subject: [PATCH 32/49] test(mcp): keep routed serve sessions off the async worker --- .../tests/mcp_suite/mcp_cli_serve_test.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs index a4e66eaca79..cd5de053c19 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs @@ -1077,13 +1077,16 @@ async fn no_explicit_path_prefers_initialize_roots_over_global_fallback() { register_global_project(home.path(), stale.path()).await; let _daemon = common::spawn_tracedecay_daemon(home.path()); - let output = run_serve_runtime_with_initialize_root( - home.path(), - cwd.path(), - None, - file_uri(active.path()), - "active", - ); + // The stdio session blocks while the initialized projects still have + // runtime-owned work, so keep the current-thread worker available. + let home_path = home.path().to_path_buf(); + let cwd_path = cwd.path().to_path_buf(); + let root_uri = file_uri(active.path()); + let output = tokio::task::spawn_blocking(move || { + run_serve_runtime_with_initialize_root(&home_path, &cwd_path, None, root_uri, "active") + }) + .await + .expect("serve runtime host session"); assert_eq!( canonical_path_string(Path::new(&runtime_project_root(&output.stdout, 2))), From 6a8f155d4dfdec81d46506812b9988e22beab0b7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:51:32 -0800 Subject: [PATCH 33/49] test(cli): wait for typed diagnostics discovery --- .../tool_surface_transport_test.rs | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-cli/tests/core_cli_suite/tool_surface_transport_test.rs b/crates/tracedecay-cli/tests/core_cli_suite/tool_surface_transport_test.rs index ff131bb0e16..8276b22963a 100644 --- a/crates/tracedecay-cli/tests/core_cli_suite/tool_surface_transport_test.rs +++ b/crates/tracedecay-cli/tests/core_cli_suite/tool_surface_transport_test.rs @@ -457,32 +457,39 @@ fn init_typescript_project(home: &Path, project: &Path, write_fixture: impl FnOn const DIAGNOSTICS_FILE_ARGS: &str = r#"{"scope":"file","path":"src/index.ts","format":"json"}"#; -/// Polls `tracedecay tool diagnostics` until the daemon's producer has -/// published for the current generation; pending is the only state worth -/// waiting through. -fn await_published_cli_diagnostics(home: &Path, project: &Path, args: &str) -> serde_json::Value { +/// Producer discovery can still be pending after project initialization. +/// Wait through only pending or stale states before asserting its current result. +fn await_cli_diagnostics_discovery(home: &Path, project: &Path, args: &str) -> SurfaceOutcome { let started = Instant::now(); loop { let outcome = run_surface_tool_from(home, project, "diagnostics", args); match outcome.problem_code().as_deref() { - None => break outcome.answer(), Some("application.diagnostics.pending" | "application.diagnostics.stale") => { assert!( started.elapsed() < SURFACE_TIMEOUT, - "the TypeScript producer did not publish within {:?}\nstdout:\n{}", + "the TypeScript producer did not finish discovery within {:?}\nstdout:\n{}", started.elapsed(), outcome.stdout ); std::thread::sleep(Duration::from_millis(250)); } - Some(code) => panic!( - "the producer reported a terminal state instead of publishing: {code}\nstdout:\n{}\nstderr:\n{}", - outcome.stdout, outcome.stderr - ), + _ => break outcome, } } } +/// Polls until the producer publishes, preserving terminal refusal failures. +fn await_published_cli_diagnostics(home: &Path, project: &Path, args: &str) -> serde_json::Value { + let outcome = await_cli_diagnostics_discovery(home, project, args); + match outcome.problem_code().as_deref() { + None => outcome.answer(), + Some(code) => panic!( + "the producer reported a terminal state instead of publishing: {code}\nstdout:\n{}\nstderr:\n{}", + outcome.stdout, outcome.stderr + ), + } +} + /// The CLI fallback the MCP error text names must answer exactly like the /// MCP tool: a fresh TypeScript project with its own `tsc` yields the /// compiler's `TS4023` on the file once the daemon's producer has published. @@ -547,12 +554,7 @@ fn tool_diagnostics_names_the_install_command_without_a_compiler() { }); let _daemon = spawn_tracedecay_daemon(&home_path); - let outcome = run_surface_tool_from( - &home_path, - &project_path, - "diagnostics", - DIAGNOSTICS_FILE_ARGS, - ); + let outcome = await_cli_diagnostics_discovery(&home_path, &project_path, DIAGNOSTICS_FILE_ARGS); assert_eq!( outcome.problem_code().as_deref(), Some("application.diagnostics.producer-missing"), @@ -700,12 +702,8 @@ fn tool_diagnostics_names_pnpm_install_for_an_uninstalled_monorepo() { }); let _daemon = spawn_tracedecay_daemon(&home_path); - let outcome = run_surface_tool_from( - &home_path, - &project_path, - "diagnostics", - MONOREPO_APP_FILE_ARGS, - ); + let outcome = + await_cli_diagnostics_discovery(&home_path, &project_path, MONOREPO_APP_FILE_ARGS); assert_eq!( outcome.problem_code().as_deref(), Some("application.diagnostics.producer-missing"), From df5c1ed5c5d93f4c83c744bda5e240ce8b8a0263 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:54:38 -0800 Subject: [PATCH 34/49] test(sessions): settle archive before bounded traversal --- crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 50d61cf17b5..550da23b247 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -2908,6 +2908,8 @@ mod recent_first_discovery_tests { } let admitted = archive.join("rollout-admitted.jsonl"); std::fs::write(&admitted, b"{}\n").unwrap(); + // Isolate bounded enumeration from the file's coarse-clock transition. + spin_until_jsonl_change_settled(&admitted); let source = CodexSource::with_home(temp.path()); let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); From 37dd630efe513abbaf20ab8f98b1ae9d088c083f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:01:11 -0800 Subject: [PATCH 35/49] test(daemon): observe reconciled text for edit hooks --- .../tracedecay/src/daemon/tests/rmcp_route.rs | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index de5c8297ef2..8120fa2fcd7 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -662,12 +662,16 @@ async fn unix_fire_and_forget_hook(fixture: &RmcpRouteFixture, event: DaemonHook } #[cfg(unix)] -async fn wait_for_new_generation( +async fn wait_for_new_text_generation( fixture: &RmcpRouteFixture, project: &std::path::Path, prior: Option<&CodeGenerationId>, + expected_paths: &[&str], step: &str, ) -> CodeGenerationId { + // Queryable text is the edit's reconciliation receipt. The complete graph + // slot can still name its predecessor while optional graph activation runs. + // Match the source paths too: an unrelated generation change proves no edit. // Well inside the 30 s query-admission backstop, so only this step's // own hook wake can publish in time. tokio::time::timeout(PHASE_TIMEOUT, async { @@ -676,17 +680,26 @@ async fn wait_for_new_generation( .engine .invocation .code_index_schedulers - .latest_generation_id(project) + .latest_text_serving_for_root(project) .await - && Some(¤t) != prior { - return current; + let metadata = current.metadata(); + if Some(&metadata.manifest().generation_id) != prior + && metadata + .snapshot() + .files + .iter() + .map(|file| file.logical_path.as_str()) + .eq(expected_paths.iter().copied()) + { + return metadata.manifest().generation_id.clone(); + } } tokio::time::sleep(Duration::from_millis(10)).await; } }) .await - .unwrap_or_else(|_| panic!("{step} hook did not wake a reconcile pass")) + .unwrap_or_else(|_| panic!("{step} hook did not reconcile source paths {expected_paths:?}")) } /// Save, rename, and delete each arrive as a fire-and-forget after-edit @@ -713,20 +726,48 @@ async fn fire_and_forget_edit_hooks_each_wake_a_reconcile() { ) }; - let initial = wait_for_new_generation(&fixture, &project, None, "initial mount").await; + let initial = wait_for_new_text_generation( + &fixture, + &project, + None, + &["src/main.rs"], + "initial mount", + ) + .await; std::fs::write(project.join("src/saved.rs"), "pub fn saved() {}\n").expect("save source"); unix_fire_and_forget_hook(&fixture, edit(&["src/saved.rs"])).await; - let saved = wait_for_new_generation(&fixture, &project, Some(&initial), "save").await; + let saved = wait_for_new_text_generation( + &fixture, + &project, + Some(&initial), + &["src/main.rs", "src/saved.rs"], + "save", + ) + .await; std::fs::rename(project.join("src/saved.rs"), project.join("src/renamed.rs")) .expect("rename source"); unix_fire_and_forget_hook(&fixture, edit(&["src/saved.rs", "src/renamed.rs"])).await; - let renamed = wait_for_new_generation(&fixture, &project, Some(&saved), "rename").await; + let renamed = wait_for_new_text_generation( + &fixture, + &project, + Some(&saved), + &["src/main.rs", "src/renamed.rs"], + "rename", + ) + .await; std::fs::remove_file(project.join("src/renamed.rs")).expect("delete source"); unix_fire_and_forget_hook(&fixture, edit(&["src/renamed.rs"])).await; - wait_for_new_generation(&fixture, &project, Some(&renamed), "delete").await; + wait_for_new_text_generation( + &fixture, + &project, + Some(&renamed), + &["src/main.rs"], + "delete", + ) + .await; } async fn portable_one_request(fixture: &RmcpRouteFixture, request: &Value) -> Vec { From bac6a374132cd29823c900e0d1591c6499b6f084 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:07:51 -0800 Subject: [PATCH 36/49] chore(feedback): trace observation writes and terminal drain --- .../src/feedback/concrete.rs | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-application/src/feedback/concrete.rs b/crates/tracedecay-application/src/feedback/concrete.rs index 2a9f5a7d02a..a6fff6542b6 100644 --- a/crates/tracedecay-application/src/feedback/concrete.rs +++ b/crates/tracedecay-application/src/feedback/concrete.rs @@ -47,6 +47,7 @@ use tracedecay_domain::{ }; use tracedecay_store::DiagnosticStore; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; +use tracing::Instrument; use super::concrete_evidence::{complete, cursor_refused, interruption, unavailable}; use super::observations::{ @@ -294,11 +295,24 @@ impl ProjectFeedbackObservationSinkV1 { Ok(()) } - #[tracing::instrument(name = "usecases.feedback.close_drain", level = "trace", skip_all)] + #[tracing::instrument( + name = "usecases.feedback.close_drain", + level = "trace", + skip_all, + fields(producer_boot_id = %self.boot_id) + )] async fn join_worker(&self) -> Result<(), FeedbackRuntimeError> { - let mut worker = self.worker.lock().await; + let mut worker = self + .worker + .lock() + .instrument(tracing::trace_span!("usecases.feedback.close_drain.lock")) + .await; let outcome = match worker.as_ref() { - Some(task) => match task.wait().await { + Some(task) => match task + .wait() + .instrument(tracing::trace_span!("usecases.feedback.close_drain.worker")) + .await + { NativeTaskOutcome::Completed(Ok(())) => Ok(()), NativeTaskOutcome::Completed(Err(())) => Err(FeedbackRuntimeError::Store), NativeTaskOutcome::Cancelled | NativeTaskOutcome::Panicked(_) => { @@ -1783,7 +1797,11 @@ fn interruption_outcome(context: &RequestContext, observed_at: UtcMicros) -> Fee #[tracing::instrument( name = "usecases.feedback.persist_observation", level = "trace", - skip_all + skip_all, + fields( + producer_boot_id = ?envelope.producer_boot_id, + producer_sequence = ?envelope.producer_sequence + ) )] async fn persist_feedback_observation( database: &Database, @@ -1794,9 +1812,12 @@ async fn persist_feedback_observation( } let transaction = database .begin_write_transaction("record feedback observation") + .instrument(tracing::trace_span!("usecases.feedback.writer_admission")) .await .map_err(|_| FeedbackRuntimeError::Store)?; - let mut ledger = load_observation_ledger(&transaction).await?; + let mut ledger = load_observation_ledger(&transaction) + .instrument(tracing::trace_span!("usecases.feedback.load_ledger")) + .await?; if ledger .observations .iter() @@ -1850,18 +1871,27 @@ async fn persist_feedback_observation( } ledger.observations.push(envelope); ledger.schema_version = OBSERVATION_LEDGER_SCHEMA_VERSION; - let encoded = encode_bounded_observation_ledger( - &mut ledger, - MAX_STORED_OBSERVATIONS, - MAX_STORED_OBSERVATION_BOOTS, - MAX_OBSERVATION_LEDGER_BYTES, - )?; + let encoded = tracing::trace_span!( + "usecases.feedback.encode_ledger", + observations = ledger.observations.len(), + boots = ledger.producer_boots.len() + ) + .in_scope(|| { + encode_bounded_observation_ledger( + &mut ledger, + MAX_STORED_OBSERVATIONS, + MAX_STORED_OBSERVATION_BOOTS, + MAX_OBSERVATION_LEDGER_BYTES, + ) + })?; database .set_metadata_unguarded(&transaction, OBSERVATION_LEDGER_METADATA_KEY, &encoded) + .instrument(tracing::trace_span!("usecases.feedback.persist_ledger")) .await .map_err(|_| FeedbackRuntimeError::Store)?; transaction .commit() + .instrument(tracing::trace_span!("usecases.feedback.commit")) .await .map_err(|_| FeedbackRuntimeError::Store) } From c42606ceb6d49baa97aacafe6406f8a832e30628 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:08:15 -0800 Subject: [PATCH 37/49] test(daemon): capture deadline route shutdown phases --- .../src/feedback/concrete.rs | 81 +++++---- .../src/project_runtime/shutdown.rs | 58 +++--- .../src/shutdown/orchestration.rs | 31 ++-- .../src/shutdown_coordination.rs | 12 +- .../src/daemon/project_open_admission.rs | 28 +-- .../tracedecay/src/daemon/tests/rmcp_route.rs | 168 ++++++++++-------- 6 files changed, 216 insertions(+), 162 deletions(-) diff --git a/crates/tracedecay-application/src/feedback/concrete.rs b/crates/tracedecay-application/src/feedback/concrete.rs index a6fff6542b6..ee627ec6a1c 100644 --- a/crates/tracedecay-application/src/feedback/concrete.rs +++ b/crates/tracedecay-application/src/feedback/concrete.rs @@ -47,7 +47,7 @@ use tracedecay_domain::{ }; use tracedecay_store::DiagnosticStore; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; -use tracing::Instrument; +use tracing::{Instrument, instrument::WithSubscriber}; use super::concrete_evidence::{complete, cursor_refused, interruption, unavailable}; use super::observations::{ @@ -212,49 +212,52 @@ impl ProjectFeedbackObservationSinkV1 { mpsc::channel::(1); let dropped_count = Arc::new(AtomicU64::new(0)); let worker_dropped_count = Arc::clone(&dropped_count); - let worker = NativeTask::spawn(async move { - let mut data_open = true; - let mut control_open = true; - while data_open || control_open { - let envelope = tokio::select! { - biased; - envelope = receiver.recv(), if data_open => { - if envelope.is_none() { - data_open = false; - } - envelope - }, - envelope = control_receiver.recv(), if control_open => { - if envelope.is_none() { - control_open = false; + let worker = NativeTask::spawn( + async move { + let mut data_open = true; + let mut control_open = true; + while data_open || control_open { + let envelope = tokio::select! { + biased; + envelope = receiver.recv(), if data_open => { + if envelope.is_none() { + data_open = false; + } + envelope + }, + envelope = control_receiver.recv(), if control_open => { + if envelope.is_none() { + control_open = false; + } + envelope + }, + }; + let Some(mut envelope) = envelope else { + continue; + }; + let terminal = matches!( + envelope.source_event.as_ref(), + Some(FeedbackSourceEventV1::TelemetryDropObserved { terminal: true, .. }) + ); + let reported_drops = + apply_pending_worker_drops(&mut envelope, &worker_dropped_count); + if persist_feedback_observation(&database, envelope) + .await + .is_err() + { + saturating_add(&worker_dropped_count, reported_drops.saturating_add(1)); + if terminal { + return Err(()); } - envelope - }, - }; - let Some(mut envelope) = envelope else { - continue; - }; - let terminal = matches!( - envelope.source_event.as_ref(), - Some(FeedbackSourceEventV1::TelemetryDropObserved { terminal: true, .. }) - ); - let reported_drops = - apply_pending_worker_drops(&mut envelope, &worker_dropped_count); - if persist_feedback_observation(&database, envelope) - .await - .is_err() - { - saturating_add(&worker_dropped_count, reported_drops.saturating_add(1)); + } if terminal { - return Err(()); + return Ok(()); } } - if terminal { - return Ok(()); - } + Ok(()) } - Ok(()) - }) + .with_current_subscriber(), + ) .map_err(|_| FeedbackRuntimeError::ObservationWorker)?; Ok(Self { sender, diff --git a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs index 9300993c921..3d9fe57cd3c 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, atomic::Ordering}; use std::time::Instant; use tracedecay_runtime_core::native_runtime::{self, NativeTask, NativeTaskOutcome}; +use tracing::{Instrument, instrument::WithSubscriber}; use super::{ProjectRuntime, ProjectRuntimeRegistryV1}; use crate::invocation::UnavailableFeedbackCycleRuntimeV1; @@ -200,29 +201,32 @@ impl ProjectRuntimeRegistryV1 { if !self.shutdown_started.swap(true, Ordering::AcqRel) { let registry = self.clone(); self.shutdown_complete.send_replace(ShutdownState::Pending); - let task = NativeTask::spawn(async move { - let drain_registry = registry.clone(); - let drained = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_registry.take_all_blocking() - })) - }) - .await; - tracedecay_runtime_core::logging::log_daemon_event( - "daemon_shutdown", - &[("outcome", "project_runtimes_drained".to_string())], - ); - let stop = match drained { - Ok(Ok(runtimes)) => registry.finish_drained_runtimes(runtimes, deadline).await, - _ => { - registry.shutdown_started.store(false, Ordering::Release); - ProjectRuntimesStop::DidNotStop - } - }; - registry - .shutdown_complete - .send_replace(ShutdownState::Finished(stop)); - }); + let task = NativeTask::spawn( + async move { + let drain_registry = registry.clone(); + let drained = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + drain_registry.take_all_blocking() + })) + }) + .await; + tracedecay_runtime_core::logging::log_daemon_event( + "daemon_shutdown", + &[("outcome", "project_runtimes_drained".to_string())], + ); + let stop = match drained { + Ok(Ok(runtimes)) => registry.finish_drained_runtimes(runtimes, deadline).await, + _ => { + registry.shutdown_started.store(false, Ordering::Release); + ProjectRuntimesStop::DidNotStop + } + }; + registry + .shutdown_complete + .send_replace(ShutdownState::Finished(stop)); + } + .with_current_subscriber(), + ); match task { Ok(task) => *shutdown_task = Some(task), Err(error) => { @@ -384,7 +388,13 @@ async fn shut_down_feedback(runtimes: &BTreeMap) -> Vec .collect(); let mut lost_work = Vec::new(); for (project_root, drain) in drains { - if let Err(error) = drain.await { + if let Err(error) = drain + .instrument(tracing::trace_span!( + "daemon.service.project_runtime.feedback_drain", + project = %project_root.display() + )) + .await + { lost_work.push(format!( "{}: feedback observations: {error}", project_root.display() diff --git a/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs b/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs index 9a5be4cc7ab..8b76ac7bc8d 100644 --- a/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs +++ b/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs @@ -17,6 +17,7 @@ use super::owners::{ use tracedecay_runtime_core::logging::log_daemon_event; use tracedecay_runtime_core::native_runtime::{self, TimeoutError}; use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt}; +use tracing::instrument::WithSubscriber; type ProjectServerShutdownFuture = Pin + Send + 'static>>; @@ -337,17 +338,20 @@ where let coordinator_attempt = Arc::clone(&attempt); let mut coordinator_failures = failures.clone(); let coordinator = runner_runtime.map(|runtime| async move { - let mut runner = runtime.spawn(async move { - let prepare_started = Instant::now(); - let plan = prepare.await; - log_shutdown_phase( - "plan_prepare", - "complete", - prepare_started, - shutdown_deadline, - ); - run_daemon_shutdown(runner_lifecycle, plan, shutdown_deadline).await - }); + let mut runner = runtime.spawn( + async move { + let prepare_started = Instant::now(); + let plan = prepare.await; + log_shutdown_phase( + "plan_prepare", + "complete", + prepare_started, + shutdown_deadline, + ); + run_daemon_shutdown(runner_lifecycle, plan, shutdown_deadline).await + } + .with_current_subscriber(), + ); let (mut receipt, runner_needs_join) = tokio::select! { biased; result = &mut runner => { @@ -399,7 +403,10 @@ where let spawned = coordinator .ok_or_else(|| "daemon shutdown runner found no Tokio runtime".to_owned()) .and_then(|coordinator| { - lifecycle.spawn_shutdown_coordinator(&attempt, coordinator) + lifecycle.spawn_shutdown_coordinator( + &attempt, + coordinator.with_current_subscriber(), + ) }); if let Err(error) = spawned { let receipt = Arc::new(DaemonShutdownReceipt::coordinator_failed( diff --git a/crates/tracedecay-daemon-service/src/shutdown_coordination.rs b/crates/tracedecay-daemon-service/src/shutdown_coordination.rs index b98b9e5c3fe..466f08dea83 100644 --- a/crates/tracedecay-daemon-service/src/shutdown_coordination.rs +++ b/crates/tracedecay-daemon-service/src/shutdown_coordination.rs @@ -5,6 +5,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use tracedecay_runtime_core::native_runtime::{NativeTask, NativeTaskOutcome}; use tracedecay_store_runtime::ShutdownStatus; +use tracing::instrument::WithSubscriber; pub struct ShutdownCoordinatorV1 { state: Arc, @@ -91,10 +92,13 @@ impl ShutdownCoordinatorV1 { .ok_or_else(|| "MCP shutdown coordinator lost its work future".to_owned()) .and_then(|work| { let state = Arc::clone(&self.state); - NativeTask::spawn(async move { - let _completion = ShutdownCoordinatorCompletion(Arc::clone(&state)); - state.finish(work.await); - }) + NativeTask::spawn( + async move { + let _completion = ShutdownCoordinatorCompletion(Arc::clone(&state)); + state.finish(work.await); + } + .with_current_subscriber(), + ) .map_err(|error| format!("MCP shutdown coordinator was not started: {error}")) }); match started { diff --git a/crates/tracedecay/src/daemon/project_open_admission.rs b/crates/tracedecay/src/daemon/project_open_admission.rs index a2c38f94bc2..b87bae1a080 100644 --- a/crates/tracedecay/src/daemon/project_open_admission.rs +++ b/crates/tracedecay/src/daemon/project_open_admission.rs @@ -38,6 +38,7 @@ use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::git_discovery::REPOSITORY_DISCOVERY_DEFERRED_REASON_CODE; use tracedecay_runtime_core::native_runtime::{self, TimeoutError}; use tracedecay_session_runtime::StoreOwnerKey; +use tracing::instrument::WithSubscriber; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(super) struct ProjectServerKey { @@ -671,18 +672,21 @@ impl ProjectOpenTasks { // completion fires once the task and the blocking work it owns end. let completion_owner: Arc = Arc::new(ProjectOpenTaskCompletionFinalizer(task_completion)); - let task = runtime.spawn(tracing::Instrument::instrument( - native_runtime::BLOCKING_WORK_OWNER.scope(completion_owner, async move { - let state = match open(task_cancellation).await { - Ok(()) => ProjectOpenTaskState::Ready, - Err(error) => ProjectOpenTaskState::Failed( - ProjectOpenFailure::recorded_for_route(&error, &failure_route), - ), - }; - updates.send_replace(state); - }), - tracing::trace_span!("daemon.project.admit.task"), - )); + let task = runtime.spawn( + tracing::Instrument::instrument( + native_runtime::BLOCKING_WORK_OWNER.scope(completion_owner, async move { + let state = match open(task_cancellation).await { + Ok(()) => ProjectOpenTaskState::Ready, + Err(error) => ProjectOpenTaskState::Failed( + ProjectOpenFailure::recorded_for_route(&error, &failure_route), + ), + }; + updates.send_replace(state); + }), + tracing::trace_span!("daemon.project.admit.task"), + ) + .with_current_subscriber(), + ); registry.routes.insert( route, ProjectOpenTaskEntry { diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index 8120fa2fcd7..7e9ef318157 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -19,6 +19,10 @@ use tracedecay_hooks::core_events::DaemonHookEvent; use tracedecay_runtime_core::cancellation::CancellationToken; #[cfg(unix)] use tracedecay_runtime_core::path_safety::canonical_existing_identity; +#[cfg(unix)] +use tracing::instrument::WithSubscriber; +#[cfg(unix)] +use tracing_subscriber::{EnvFilter, fmt::format::FmtSpan}; use super::*; @@ -1462,77 +1466,99 @@ async fn serve_proxy_cancels_running_and_queued_requests_without_host_disconnect #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn deadline_reply_does_not_wait_for_a_worker_that_ignores_cancellation() { - let fixture = rmcp_route_fixture("rmcp-deadline-uncooperative").await; - let (executor, controlled_server) = mount_cancellation_executor(&fixture).await; - let (reader, mut writer, server_task) = - super::handshake_authenticated_test_client(fixture.engine.clone(), &fixture.handshake) - .await; - let mut reader = tokio::io::BufReader::new(reader); - write_line(&mut writer, &initialize_request()).await; - assert_eq!( - read_value(&mut reader, "initialize deadline route").await["id"], - json!(1) - ); + let dispatch = { + let subscriber = tracing_subscriber::fmt() + .with_env_filter(EnvFilter::new(concat!( + "tracedecay_application::feedback::concrete=trace,", + "tracedecay_daemon_service::project_runtime::shutdown=trace,", + "tracedecay_daemon_service::shutdown=trace,", + "tracedecay::mcp::server::connection=trace,", + "tracedecay_runtime_core::db::engine=trace" + ))) + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_timer(tracing_subscriber::fmt::time::uptime()) + .with_thread_names(true) + .with_ansi(false) + .with_writer(std::io::stderr) + .finish(); + let _capture = tracedecay_runtime_core::logging::set_tracing_capture(subscriber); + tracing::dispatcher::get_default(Clone::clone) + }; + async { + let fixture = rmcp_route_fixture("rmcp-deadline-uncooperative").await; + let (executor, controlled_server) = mount_cancellation_executor(&fixture).await; + let (reader, mut writer, server_task) = + super::handshake_authenticated_test_client(fixture.engine.clone(), &fixture.handshake) + .await; + let mut reader = tokio::io::BufReader::new(reader); + write_line(&mut writer, &initialize_request()).await; + assert_eq!( + read_value(&mut reader, "initialize deadline route").await["id"], + json!(1) + ); - let budget = Duration::from_secs(5); - let mut request = blocked_tool_request(10); - request["params"]["_meta"] = - tracedecay_mcp::tool_call_deadline_meta(tracedecay_domain::UtcMicros( - tracedecay_contracts::clock::now_micros().0 - + i64::try_from(budget.as_micros()).expect("budget micros"), - )); - let sent = std::time::Instant::now(); - write_line(&mut writer, &request).await; - wait_for_count( - &executor.started, - 1, - "deadline request never reached executor", - ) - .await; - let response = read_value( - &mut reader, - "deadline reply waited for a worker that ignores cancellation", - ) - .await; - let elapsed = sent.elapsed(); - assert_eq!(response["id"], json!(10), "{response}"); - wait_for_count( - &executor.cancellation_observed, - 1, - "the deadline must cancel the admitted worker", - ) - .await; - assert_eq!( - executor.completed.load(Ordering::SeqCst), - 0, - "the reply must not wait for the worker to exit" - ); - assert!( - elapsed < budget + Duration::from_secs(2), - "deadline reply took {elapsed:?} for a {budget:?} budget" - ); - assert!( - response - .to_string() - .contains("tool_dispatch_deadline_exceeded"), - "expected the typed dispatch deadline terminal: {response}" - ); + let budget = Duration::from_secs(5); + let mut request = blocked_tool_request(10); + request["params"]["_meta"] = + tracedecay_mcp::tool_call_deadline_meta(tracedecay_domain::UtcMicros( + tracedecay_contracts::clock::now_micros().0 + + i64::try_from(budget.as_micros()).expect("budget micros"), + )); + let sent = std::time::Instant::now(); + write_line(&mut writer, &request).await; + wait_for_count( + &executor.started, + 1, + "deadline request never reached executor", + ) + .await; + let response = read_value( + &mut reader, + "deadline reply waited for a worker that ignores cancellation", + ) + .await; + let elapsed = sent.elapsed(); + assert_eq!(response["id"], json!(10), "{response}"); + wait_for_count( + &executor.cancellation_observed, + 1, + "the deadline must cancel the admitted worker", + ) + .await; + assert_eq!( + executor.completed.load(Ordering::SeqCst), + 0, + "the reply must not wait for the worker to exit" + ); + assert!( + elapsed < budget + Duration::from_secs(2), + "deadline reply took {elapsed:?} for a {budget:?} budget" + ); + assert!( + response + .to_string() + .contains("tool_dispatch_deadline_exceeded"), + "expected the typed dispatch deadline terminal: {response}" + ); - executor.release_first.store(true, Ordering::SeqCst); - wait_for_count(&executor.completed, 1, "released worker did not exit").await; - writer.shutdown().await.expect("shutdown deadline client"); - drop(writer); - drop(reader); - tokio::time::timeout(PHASE_TIMEOUT, server_task) - .await - .expect("deadline RMCP connection did not close") - .expect("join deadline RMCP task") - .expect("serve deadline RMCP task"); - fixture.server.shutdown().await; - drop((controlled_server, fixture.server)); - let shutdown = fixture.engine.shutdown_all().await; - assert!( - shutdown.project_servers.is_clean(), - "deadline route shutdown: {shutdown:?}" - ); + executor.release_first.store(true, Ordering::SeqCst); + wait_for_count(&executor.completed, 1, "released worker did not exit").await; + writer.shutdown().await.expect("shutdown deadline client"); + drop(writer); + drop(reader); + tokio::time::timeout(PHASE_TIMEOUT, server_task) + .await + .expect("deadline RMCP connection did not close") + .expect("join deadline RMCP task") + .expect("serve deadline RMCP task"); + fixture.server.shutdown().await; + drop((controlled_server, fixture.server)); + let shutdown = fixture.engine.shutdown_all().await; + assert!( + shutdown.project_servers.is_clean(), + "deadline route shutdown: {shutdown:?}" + ); + } + .with_subscriber(dispatch) + .await; } From 3f745b64a1c85d40642182205b4d14c82cf4e8e1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:10:03 -0800 Subject: [PATCH 38/49] style(daemon): wrap shutdown drain result --- .../tracedecay-daemon-service/src/project_runtime/shutdown.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs index 3d9fe57cd3c..5750d54d347 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs @@ -215,7 +215,9 @@ impl ProjectRuntimeRegistryV1 { &[("outcome", "project_runtimes_drained".to_string())], ); let stop = match drained { - Ok(Ok(runtimes)) => registry.finish_drained_runtimes(runtimes, deadline).await, + Ok(Ok(runtimes)) => { + registry.finish_drained_runtimes(runtimes, deadline).await + } _ => { registry.shutdown_started.store(false, Ordering::Release); ProjectRuntimesStop::DidNotStop From 613aabb32292acc6e09971b80a8bec61c6642251 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:21:37 -0800 Subject: [PATCH 39/49] style: format shutdown measurement and hook readiness --- .../src/project_runtime/shutdown.rs | 13 +++++++------ .../src/shutdown/orchestration.rs | 6 ++---- crates/tracedecay/src/daemon/tests/rmcp_route.rs | 11 +++-------- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs index 5750d54d347..be867208bbb 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs @@ -204,12 +204,13 @@ impl ProjectRuntimeRegistryV1 { let task = NativeTask::spawn( async move { let drain_registry = registry.clone(); - let drained = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_registry.take_all_blocking() - })) - }) - .await; + let drained = + tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + drain_registry.take_all_blocking() + })) + }) + .await; tracedecay_runtime_core::logging::log_daemon_event( "daemon_shutdown", &[("outcome", "project_runtimes_drained".to_string())], diff --git a/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs b/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs index 8b76ac7bc8d..0d1807b4280 100644 --- a/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs +++ b/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs @@ -403,10 +403,8 @@ where let spawned = coordinator .ok_or_else(|| "daemon shutdown runner found no Tokio runtime".to_owned()) .and_then(|coordinator| { - lifecycle.spawn_shutdown_coordinator( - &attempt, - coordinator.with_current_subscriber(), - ) + lifecycle + .spawn_shutdown_coordinator(&attempt, coordinator.with_current_subscriber()) }); if let Err(error) = spawned { let receipt = Arc::new(DaemonShutdownReceipt::coordinator_failed( diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index 7e9ef318157..80c78fd2b32 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -730,14 +730,9 @@ async fn fire_and_forget_edit_hooks_each_wake_a_reconcile() { ) }; - let initial = wait_for_new_text_generation( - &fixture, - &project, - None, - &["src/main.rs"], - "initial mount", - ) - .await; + let initial = + wait_for_new_text_generation(&fixture, &project, None, &["src/main.rs"], "initial mount") + .await; std::fs::write(project.join("src/saved.rs"), "pub fn saved() {}\n").expect("save source"); unix_fire_and_forget_hook(&fixture, edit(&["src/saved.rs"])).await; From dab11b791193dfe8b84c686e8d0aa40c966ba511 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:08:55 -0800 Subject: [PATCH 40/49] test(daemon): use static filters for shutdown measurement --- .../tracedecay/src/daemon/tests/rmcp_route.rs | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index 80c78fd2b32..7026fa75b23 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -22,7 +22,7 @@ use tracedecay_runtime_core::path_safety::canonical_existing_identity; #[cfg(unix)] use tracing::instrument::WithSubscriber; #[cfg(unix)] -use tracing_subscriber::{EnvFilter, fmt::format::FmtSpan}; +use tracing_subscriber::{filter::Targets, fmt::format::FmtSpan, layer::SubscriberExt}; use super::*; @@ -1462,20 +1462,23 @@ async fn serve_proxy_cancels_running_and_queued_requests_without_host_disconnect #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn deadline_reply_does_not_wait_for_a_worker_that_ignores_cancellation() { let dispatch = { - let subscriber = tracing_subscriber::fmt() - .with_env_filter(EnvFilter::new(concat!( - "tracedecay_application::feedback::concrete=trace,", - "tracedecay_daemon_service::project_runtime::shutdown=trace,", - "tracedecay_daemon_service::shutdown=trace,", - "tracedecay::mcp::server::connection=trace,", - "tracedecay_runtime_core::db::engine=trace" - ))) - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .with_timer(tracing_subscriber::fmt::time::uptime()) - .with_thread_names(true) - .with_ansi(false) - .with_writer(std::io::stderr) - .finish(); + let targets = Targets::new().with_targets([ + ("tracedecay_application::feedback::concrete", tracing::Level::TRACE), + ("tracedecay_daemon_service::project_runtime::shutdown", tracing::Level::TRACE), + ("tracedecay_daemon_service::shutdown", tracing::Level::TRACE), + ("tracedecay::mcp::server::connection", tracing::Level::TRACE), + ("tracedecay_runtime_core::db::engine", tracing::Level::TRACE), + ]); + let subscriber = tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_timer(tracing_subscriber::fmt::time::uptime()) + .with_thread_names(true) + .with_ansi(false) + .with_writer(std::io::stderr), + ) + .with(targets); let _capture = tracedecay_runtime_core::logging::set_tracing_capture(subscriber); tracing::dispatcher::get_default(Clone::clone) }; From 0ecb01b5cd2ad69350a895b061ea4c32ec203af9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:25:40 -0800 Subject: [PATCH 41/49] style: format shutdown capture targets --- crates/tracedecay/src/daemon/tests/rmcp_route.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index 7026fa75b23..1bbe530742f 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -1463,8 +1463,14 @@ async fn serve_proxy_cancels_running_and_queued_requests_without_host_disconnect async fn deadline_reply_does_not_wait_for_a_worker_that_ignores_cancellation() { let dispatch = { let targets = Targets::new().with_targets([ - ("tracedecay_application::feedback::concrete", tracing::Level::TRACE), - ("tracedecay_daemon_service::project_runtime::shutdown", tracing::Level::TRACE), + ( + "tracedecay_application::feedback::concrete", + tracing::Level::TRACE, + ), + ( + "tracedecay_daemon_service::project_runtime::shutdown", + tracing::Level::TRACE, + ), ("tracedecay_daemon_service::shutdown", tracing::Level::TRACE), ("tracedecay::mcp::server::connection", tracing::Level::TRACE), ("tracedecay_runtime_core::db::engine", tracing::Level::TRACE), From cf1d97acfe8271a645c8341be8ed6f8e68bd23a3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:25:41 -0800 Subject: [PATCH 42/49] test(daemon): remove shutdown capture after measurement --- .../src/feedback/concrete.rs | 81 ++++---- .../src/project_runtime/shutdown.rs | 61 +++--- .../src/shutdown/orchestration.rs | 29 ++- .../src/shutdown_coordination.rs | 12 +- .../src/daemon/project_open_admission.rs | 28 ++- .../tracedecay/src/daemon/tests/rmcp_route.rs | 177 +++++++----------- 6 files changed, 162 insertions(+), 226 deletions(-) diff --git a/crates/tracedecay-application/src/feedback/concrete.rs b/crates/tracedecay-application/src/feedback/concrete.rs index ee627ec6a1c..a6fff6542b6 100644 --- a/crates/tracedecay-application/src/feedback/concrete.rs +++ b/crates/tracedecay-application/src/feedback/concrete.rs @@ -47,7 +47,7 @@ use tracedecay_domain::{ }; use tracedecay_store::DiagnosticStore; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; -use tracing::{Instrument, instrument::WithSubscriber}; +use tracing::Instrument; use super::concrete_evidence::{complete, cursor_refused, interruption, unavailable}; use super::observations::{ @@ -212,52 +212,49 @@ impl ProjectFeedbackObservationSinkV1 { mpsc::channel::(1); let dropped_count = Arc::new(AtomicU64::new(0)); let worker_dropped_count = Arc::clone(&dropped_count); - let worker = NativeTask::spawn( - async move { - let mut data_open = true; - let mut control_open = true; - while data_open || control_open { - let envelope = tokio::select! { - biased; - envelope = receiver.recv(), if data_open => { - if envelope.is_none() { - data_open = false; - } - envelope - }, - envelope = control_receiver.recv(), if control_open => { - if envelope.is_none() { - control_open = false; - } - envelope - }, - }; - let Some(mut envelope) = envelope else { - continue; - }; - let terminal = matches!( - envelope.source_event.as_ref(), - Some(FeedbackSourceEventV1::TelemetryDropObserved { terminal: true, .. }) - ); - let reported_drops = - apply_pending_worker_drops(&mut envelope, &worker_dropped_count); - if persist_feedback_observation(&database, envelope) - .await - .is_err() - { - saturating_add(&worker_dropped_count, reported_drops.saturating_add(1)); - if terminal { - return Err(()); + let worker = NativeTask::spawn(async move { + let mut data_open = true; + let mut control_open = true; + while data_open || control_open { + let envelope = tokio::select! { + biased; + envelope = receiver.recv(), if data_open => { + if envelope.is_none() { + data_open = false; } - } + envelope + }, + envelope = control_receiver.recv(), if control_open => { + if envelope.is_none() { + control_open = false; + } + envelope + }, + }; + let Some(mut envelope) = envelope else { + continue; + }; + let terminal = matches!( + envelope.source_event.as_ref(), + Some(FeedbackSourceEventV1::TelemetryDropObserved { terminal: true, .. }) + ); + let reported_drops = + apply_pending_worker_drops(&mut envelope, &worker_dropped_count); + if persist_feedback_observation(&database, envelope) + .await + .is_err() + { + saturating_add(&worker_dropped_count, reported_drops.saturating_add(1)); if terminal { - return Ok(()); + return Err(()); } } - Ok(()) + if terminal { + return Ok(()); + } } - .with_current_subscriber(), - ) + Ok(()) + }) .map_err(|_| FeedbackRuntimeError::ObservationWorker)?; Ok(Self { sender, diff --git a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs index be867208bbb..9300993c921 100644 --- a/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs +++ b/crates/tracedecay-daemon-service/src/project_runtime/shutdown.rs @@ -6,7 +6,6 @@ use std::sync::{Arc, atomic::Ordering}; use std::time::Instant; use tracedecay_runtime_core::native_runtime::{self, NativeTask, NativeTaskOutcome}; -use tracing::{Instrument, instrument::WithSubscriber}; use super::{ProjectRuntime, ProjectRuntimeRegistryV1}; use crate::invocation::UnavailableFeedbackCycleRuntimeV1; @@ -201,35 +200,29 @@ impl ProjectRuntimeRegistryV1 { if !self.shutdown_started.swap(true, Ordering::AcqRel) { let registry = self.clone(); self.shutdown_complete.send_replace(ShutdownState::Pending); - let task = NativeTask::spawn( - async move { - let drain_registry = registry.clone(); - let drained = - tracedecay_runtime_core::native_runtime::spawn_blocking(move || { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_registry.take_all_blocking() - })) - }) - .await; - tracedecay_runtime_core::logging::log_daemon_event( - "daemon_shutdown", - &[("outcome", "project_runtimes_drained".to_string())], - ); - let stop = match drained { - Ok(Ok(runtimes)) => { - registry.finish_drained_runtimes(runtimes, deadline).await - } - _ => { - registry.shutdown_started.store(false, Ordering::Release); - ProjectRuntimesStop::DidNotStop - } - }; - registry - .shutdown_complete - .send_replace(ShutdownState::Finished(stop)); - } - .with_current_subscriber(), - ); + let task = NativeTask::spawn(async move { + let drain_registry = registry.clone(); + let drained = tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + drain_registry.take_all_blocking() + })) + }) + .await; + tracedecay_runtime_core::logging::log_daemon_event( + "daemon_shutdown", + &[("outcome", "project_runtimes_drained".to_string())], + ); + let stop = match drained { + Ok(Ok(runtimes)) => registry.finish_drained_runtimes(runtimes, deadline).await, + _ => { + registry.shutdown_started.store(false, Ordering::Release); + ProjectRuntimesStop::DidNotStop + } + }; + registry + .shutdown_complete + .send_replace(ShutdownState::Finished(stop)); + }); match task { Ok(task) => *shutdown_task = Some(task), Err(error) => { @@ -391,13 +384,7 @@ async fn shut_down_feedback(runtimes: &BTreeMap) -> Vec .collect(); let mut lost_work = Vec::new(); for (project_root, drain) in drains { - if let Err(error) = drain - .instrument(tracing::trace_span!( - "daemon.service.project_runtime.feedback_drain", - project = %project_root.display() - )) - .await - { + if let Err(error) = drain.await { lost_work.push(format!( "{}: feedback observations: {error}", project_root.display() diff --git a/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs b/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs index 0d1807b4280..9a5be4cc7ab 100644 --- a/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs +++ b/crates/tracedecay-daemon-service/src/shutdown/orchestration.rs @@ -17,7 +17,6 @@ use super::owners::{ use tracedecay_runtime_core::logging::log_daemon_event; use tracedecay_runtime_core::native_runtime::{self, TimeoutError}; use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt}; -use tracing::instrument::WithSubscriber; type ProjectServerShutdownFuture = Pin + Send + 'static>>; @@ -338,20 +337,17 @@ where let coordinator_attempt = Arc::clone(&attempt); let mut coordinator_failures = failures.clone(); let coordinator = runner_runtime.map(|runtime| async move { - let mut runner = runtime.spawn( - async move { - let prepare_started = Instant::now(); - let plan = prepare.await; - log_shutdown_phase( - "plan_prepare", - "complete", - prepare_started, - shutdown_deadline, - ); - run_daemon_shutdown(runner_lifecycle, plan, shutdown_deadline).await - } - .with_current_subscriber(), - ); + let mut runner = runtime.spawn(async move { + let prepare_started = Instant::now(); + let plan = prepare.await; + log_shutdown_phase( + "plan_prepare", + "complete", + prepare_started, + shutdown_deadline, + ); + run_daemon_shutdown(runner_lifecycle, plan, shutdown_deadline).await + }); let (mut receipt, runner_needs_join) = tokio::select! { biased; result = &mut runner => { @@ -403,8 +399,7 @@ where let spawned = coordinator .ok_or_else(|| "daemon shutdown runner found no Tokio runtime".to_owned()) .and_then(|coordinator| { - lifecycle - .spawn_shutdown_coordinator(&attempt, coordinator.with_current_subscriber()) + lifecycle.spawn_shutdown_coordinator(&attempt, coordinator) }); if let Err(error) = spawned { let receipt = Arc::new(DaemonShutdownReceipt::coordinator_failed( diff --git a/crates/tracedecay-daemon-service/src/shutdown_coordination.rs b/crates/tracedecay-daemon-service/src/shutdown_coordination.rs index 466f08dea83..b98b9e5c3fe 100644 --- a/crates/tracedecay-daemon-service/src/shutdown_coordination.rs +++ b/crates/tracedecay-daemon-service/src/shutdown_coordination.rs @@ -5,7 +5,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use tracedecay_runtime_core::native_runtime::{NativeTask, NativeTaskOutcome}; use tracedecay_store_runtime::ShutdownStatus; -use tracing::instrument::WithSubscriber; pub struct ShutdownCoordinatorV1 { state: Arc, @@ -92,13 +91,10 @@ impl ShutdownCoordinatorV1 { .ok_or_else(|| "MCP shutdown coordinator lost its work future".to_owned()) .and_then(|work| { let state = Arc::clone(&self.state); - NativeTask::spawn( - async move { - let _completion = ShutdownCoordinatorCompletion(Arc::clone(&state)); - state.finish(work.await); - } - .with_current_subscriber(), - ) + NativeTask::spawn(async move { + let _completion = ShutdownCoordinatorCompletion(Arc::clone(&state)); + state.finish(work.await); + }) .map_err(|error| format!("MCP shutdown coordinator was not started: {error}")) }); match started { diff --git a/crates/tracedecay/src/daemon/project_open_admission.rs b/crates/tracedecay/src/daemon/project_open_admission.rs index b87bae1a080..a2c38f94bc2 100644 --- a/crates/tracedecay/src/daemon/project_open_admission.rs +++ b/crates/tracedecay/src/daemon/project_open_admission.rs @@ -38,7 +38,6 @@ use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_runtime_core::git_discovery::REPOSITORY_DISCOVERY_DEFERRED_REASON_CODE; use tracedecay_runtime_core::native_runtime::{self, TimeoutError}; use tracedecay_session_runtime::StoreOwnerKey; -use tracing::instrument::WithSubscriber; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(super) struct ProjectServerKey { @@ -672,21 +671,18 @@ impl ProjectOpenTasks { // completion fires once the task and the blocking work it owns end. let completion_owner: Arc = Arc::new(ProjectOpenTaskCompletionFinalizer(task_completion)); - let task = runtime.spawn( - tracing::Instrument::instrument( - native_runtime::BLOCKING_WORK_OWNER.scope(completion_owner, async move { - let state = match open(task_cancellation).await { - Ok(()) => ProjectOpenTaskState::Ready, - Err(error) => ProjectOpenTaskState::Failed( - ProjectOpenFailure::recorded_for_route(&error, &failure_route), - ), - }; - updates.send_replace(state); - }), - tracing::trace_span!("daemon.project.admit.task"), - ) - .with_current_subscriber(), - ); + let task = runtime.spawn(tracing::Instrument::instrument( + native_runtime::BLOCKING_WORK_OWNER.scope(completion_owner, async move { + let state = match open(task_cancellation).await { + Ok(()) => ProjectOpenTaskState::Ready, + Err(error) => ProjectOpenTaskState::Failed( + ProjectOpenFailure::recorded_for_route(&error, &failure_route), + ), + }; + updates.send_replace(state); + }), + tracing::trace_span!("daemon.project.admit.task"), + )); registry.routes.insert( route, ProjectOpenTaskEntry { diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index 1bbe530742f..bcddc8fa755 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -19,10 +19,6 @@ use tracedecay_hooks::core_events::DaemonHookEvent; use tracedecay_runtime_core::cancellation::CancellationToken; #[cfg(unix)] use tracedecay_runtime_core::path_safety::canonical_existing_identity; -#[cfg(unix)] -use tracing::instrument::WithSubscriber; -#[cfg(unix)] -use tracing_subscriber::{filter::Targets, fmt::format::FmtSpan, layer::SubscriberExt}; use super::*; @@ -1461,108 +1457,77 @@ async fn serve_proxy_cancels_running_and_queued_requests_without_host_disconnect #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn deadline_reply_does_not_wait_for_a_worker_that_ignores_cancellation() { - let dispatch = { - let targets = Targets::new().with_targets([ - ( - "tracedecay_application::feedback::concrete", - tracing::Level::TRACE, - ), - ( - "tracedecay_daemon_service::project_runtime::shutdown", - tracing::Level::TRACE, - ), - ("tracedecay_daemon_service::shutdown", tracing::Level::TRACE), - ("tracedecay::mcp::server::connection", tracing::Level::TRACE), - ("tracedecay_runtime_core::db::engine", tracing::Level::TRACE), - ]); - let subscriber = tracing_subscriber::registry() - .with( - tracing_subscriber::fmt::layer() - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .with_timer(tracing_subscriber::fmt::time::uptime()) - .with_thread_names(true) - .with_ansi(false) - .with_writer(std::io::stderr), - ) - .with(targets); - let _capture = tracedecay_runtime_core::logging::set_tracing_capture(subscriber); - tracing::dispatcher::get_default(Clone::clone) - }; - async { - let fixture = rmcp_route_fixture("rmcp-deadline-uncooperative").await; - let (executor, controlled_server) = mount_cancellation_executor(&fixture).await; - let (reader, mut writer, server_task) = - super::handshake_authenticated_test_client(fixture.engine.clone(), &fixture.handshake) - .await; - let mut reader = tokio::io::BufReader::new(reader); - write_line(&mut writer, &initialize_request()).await; - assert_eq!( - read_value(&mut reader, "initialize deadline route").await["id"], - json!(1) - ); - - let budget = Duration::from_secs(5); - let mut request = blocked_tool_request(10); - request["params"]["_meta"] = - tracedecay_mcp::tool_call_deadline_meta(tracedecay_domain::UtcMicros( - tracedecay_contracts::clock::now_micros().0 - + i64::try_from(budget.as_micros()).expect("budget micros"), - )); - let sent = std::time::Instant::now(); - write_line(&mut writer, &request).await; - wait_for_count( - &executor.started, - 1, - "deadline request never reached executor", - ) - .await; - let response = read_value( - &mut reader, - "deadline reply waited for a worker that ignores cancellation", - ) - .await; - let elapsed = sent.elapsed(); - assert_eq!(response["id"], json!(10), "{response}"); - wait_for_count( - &executor.cancellation_observed, - 1, - "the deadline must cancel the admitted worker", - ) - .await; - assert_eq!( - executor.completed.load(Ordering::SeqCst), - 0, - "the reply must not wait for the worker to exit" - ); - assert!( - elapsed < budget + Duration::from_secs(2), - "deadline reply took {elapsed:?} for a {budget:?} budget" - ); - assert!( - response - .to_string() - .contains("tool_dispatch_deadline_exceeded"), - "expected the typed dispatch deadline terminal: {response}" - ); + let fixture = rmcp_route_fixture("rmcp-deadline-uncooperative").await; + let (executor, controlled_server) = mount_cancellation_executor(&fixture).await; + let (reader, mut writer, server_task) = + super::handshake_authenticated_test_client(fixture.engine.clone(), &fixture.handshake) + .await; + let mut reader = tokio::io::BufReader::new(reader); + write_line(&mut writer, &initialize_request()).await; + assert_eq!( + read_value(&mut reader, "initialize deadline route").await["id"], + json!(1) + ); - executor.release_first.store(true, Ordering::SeqCst); - wait_for_count(&executor.completed, 1, "released worker did not exit").await; - writer.shutdown().await.expect("shutdown deadline client"); - drop(writer); - drop(reader); - tokio::time::timeout(PHASE_TIMEOUT, server_task) - .await - .expect("deadline RMCP connection did not close") - .expect("join deadline RMCP task") - .expect("serve deadline RMCP task"); - fixture.server.shutdown().await; - drop((controlled_server, fixture.server)); - let shutdown = fixture.engine.shutdown_all().await; - assert!( - shutdown.project_servers.is_clean(), - "deadline route shutdown: {shutdown:?}" - ); - } - .with_subscriber(dispatch) + let budget = Duration::from_secs(5); + let mut request = blocked_tool_request(10); + request["params"]["_meta"] = + tracedecay_mcp::tool_call_deadline_meta(tracedecay_domain::UtcMicros( + tracedecay_contracts::clock::now_micros().0 + + i64::try_from(budget.as_micros()).expect("budget micros"), + )); + let sent = std::time::Instant::now(); + write_line(&mut writer, &request).await; + wait_for_count( + &executor.started, + 1, + "deadline request never reached executor", + ) .await; + let response = read_value( + &mut reader, + "deadline reply waited for a worker that ignores cancellation", + ) + .await; + let elapsed = sent.elapsed(); + assert_eq!(response["id"], json!(10), "{response}"); + wait_for_count( + &executor.cancellation_observed, + 1, + "the deadline must cancel the admitted worker", + ) + .await; + assert_eq!( + executor.completed.load(Ordering::SeqCst), + 0, + "the reply must not wait for the worker to exit" + ); + assert!( + elapsed < budget + Duration::from_secs(2), + "deadline reply took {elapsed:?} for a {budget:?} budget" + ); + assert!( + response + .to_string() + .contains("tool_dispatch_deadline_exceeded"), + "expected the typed dispatch deadline terminal: {response}" + ); + + executor.release_first.store(true, Ordering::SeqCst); + wait_for_count(&executor.completed, 1, "released worker did not exit").await; + writer.shutdown().await.expect("shutdown deadline client"); + drop(writer); + drop(reader); + tokio::time::timeout(PHASE_TIMEOUT, server_task) + .await + .expect("deadline RMCP connection did not close") + .expect("join deadline RMCP task") + .expect("serve deadline RMCP task"); + fixture.server.shutdown().await; + drop((controlled_server, fixture.server)); + let shutdown = fixture.engine.shutdown_all().await; + assert!( + shutdown.project_servers.is_clean(), + "deadline route shutdown: {shutdown:?}" + ); } From df62f257e6c73db7e3a832f9fab1f92db90e022e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:17:23 -0800 Subject: [PATCH 43/49] test(daemon): use native runtime for capacity journeys --- .../project_server_capacity_journey_test.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs index 1feffacc327..602cbe959a5 100644 --- a/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs @@ -190,8 +190,16 @@ fn assert_generation_contains_probe(latest: &LatestCompleteCodeIndexV1, probe: & ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn project_full_publication_precedes_registered_schema_convergence() { +#[test] +fn project_full_publication_precedes_registered_schema_convergence() { + tracedecay_runtime_core::native_runtime::default_builder() + .worker_threads(2) + .build() + .expect("native project composition runtime") + .block_on(verify_project_full_publication_precedes_registered_schema_convergence()); +} + +async fn verify_project_full_publication_precedes_registered_schema_convergence() { let isolation = TempDir::new().expect("production harness isolation"); let bootstrap_project = isolation.path().join("bootstrap-project"); let target_project = isolation.path().join("target-project"); @@ -265,8 +273,16 @@ async fn project_full_publication_precedes_registered_schema_convergence() { harness.shutdown().await; } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn twelve_project_journey_retires_idle_owners_without_empty_graphs() { +#[test] +fn twelve_project_journey_retires_idle_owners_without_empty_graphs() { + tracedecay_runtime_core::native_runtime::default_builder() + .worker_threads(2) + .build() + .expect("native project composition runtime") + .block_on(verify_twelve_project_journey_retires_idle_owners_without_empty_graphs()); +} + +async fn verify_twelve_project_journey_retires_idle_owners_without_empty_graphs() { let isolation = TempDir::new().expect("production harness isolation"); let mut projects = Vec::new(); for ordinal in 0..12 { From 09ef40a51367858edf6a6b6d29d95f2db3fe6dcd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:23:06 -0800 Subject: [PATCH 44/49] test(daemon): compare status within one paused attempt --- .../daemon_suite/indexing_lifecycle_test.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/tests/daemon_suite/indexing_lifecycle_test.rs b/crates/tracedecay/tests/daemon_suite/indexing_lifecycle_test.rs index e001fa5798e..ea0a1675902 100644 --- a/crates/tracedecay/tests/daemon_suite/indexing_lifecycle_test.rs +++ b/crates/tracedecay/tests/daemon_suite/indexing_lifecycle_test.rs @@ -344,11 +344,30 @@ async fn hard_killed_cold_preseal_attempt_recovers_a_verified_native_graph() { let attempt = &receipt["code_index_freshness"]["worktree"]["preseal_attempt"]; if attempt["active"] == true && attempt["phase"] == "parse_evidence" - && attempt["completed_items"] - .as_u64() - .is_some_and(|count| count > 0) + && attempt["completed_items"].as_u64().is_some_and(|count| { + count > 0 && attempt["total_items"].as_u64() == Some(count) + }) { - return receipt; + let repeat = status(&socket, &handshake).await; + let repeated_attempt = + &repeat["code_index_freshness"]["worktree"]["preseal_attempt"]; + // Init may still deliver its queued reconcile after a cold + // attempt enters the hold. The arrival marker survives that + // attempt's cancellation; compare observations of one owner. + if [ + "daemon_incarnation", + "producer_incarnation", + "attempt_sequence", + ] + .into_iter() + .all(|field| repeated_attempt[field] == attempt[field]) + { + assert_eq!( + repeated_attempt, attempt, + "status polling under a paused producer must not invent new work" + ); + return receipt; + } } tokio::time::sleep(Duration::from_millis(25)).await; } @@ -380,12 +399,6 @@ async fn hard_killed_cold_preseal_attempt_recovers_a_verified_native_graph() { .is_some_and(|epoch| epoch > 0), "the active attempt must have an observed work epoch: {in_flight}" ); - let repeat = status(&socket, &handshake).await; - assert_eq!( - repeat["code_index_freshness"]["worktree"]["preseal_attempt"], *attempt, - "status polling under a paused producer must not invent new work" - ); - let killed = daemon.kill_and_wait().expect("hard-kill the cold daemon"); assert!( !killed.success(), From 544a7e9aa9f8c36336a7c3520b66c03e45ad601e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:58:16 -0800 Subject: [PATCH 45/49] test: capture automation ledger append subphases --- .../automation/effect_runtime/settlement.rs | 9 ++++++ .../run_ledger/exact_publication.rs | 29 +++++++++++++------ crates/tracedecay-runtime-core/src/logging.rs | 19 ++++++++++-- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs index 65adf5e1f84..660a467bd01 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs @@ -1717,6 +1717,15 @@ fn settle_bound_owner_with_budget( mut state: RetainedBoundSettlement, budget: Duration, ) -> Result> { + #[cfg(test)] + let _capture = state.phase_hook.as_ref().map(|_| { + tracedecay_runtime_core::logging::FormattedTracingCapture::start_with_span_timings() + }); + #[cfg(test)] + let _capture_span = state.phase_hook.as_ref().map(|_| { + tracing::info_span!("daemon.automation.retained_owner", run_id = %state.ledger.run_id) + .entered() + }); let started = std::time::Instant::now(); let mut delay = Duration::from_millis(25); loop { diff --git a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs index 15100632cf1..389b652602e 100644 --- a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs +++ b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs @@ -764,11 +764,17 @@ fn publish_under_ledger_lock( bytes = publication.payload_len ) .entered(); - ledger - .seek(SeekFrom::Start(pre_append_eof)) - .map_err(TraceDecayError::from)?; + { + let _span = tracing::trace_span!("hosts.automation.run_ledger.seek_output").entered(); + ledger + .seek(SeekFrom::Start(pre_append_eof)) + .map_err(TraceDecayError::from)?; + } copy_exact_payload(&mut spool, &mut ledger, publication.payload_len)?; - ledger.write_all(b"\n").map_err(TraceDecayError::from)?; + { + let _span = tracing::trace_span!("hosts.automation.run_ledger.newline").entered(); + ledger.write_all(b"\n").map_err(TraceDecayError::from)?; + } } sync_run_ledger_file_and_parent(ledger_path, &ledger)?; verify_published_range(&mut ledger, &mut spool, &intent)?; @@ -928,11 +934,16 @@ fn copy_exact_payload( destination: &mut std::fs::File, payload_len: u64, ) -> Result<()> { - source - .seek(SeekFrom::Start(0)) - .map_err(TraceDecayError::from)?; - let copied = - std::io::copy(&mut source.take(payload_len), destination).map_err(TraceDecayError::from)?; + { + let _span = tracing::trace_span!("hosts.automation.run_ledger.seek_input").entered(); + source + .seek(SeekFrom::Start(0)) + .map_err(TraceDecayError::from)?; + } + let copied = { + let _span = tracing::trace_span!("hosts.automation.run_ledger.copy_payload").entered(); + std::io::copy(&mut source.take(payload_len), destination).map_err(TraceDecayError::from)? + }; if copied != payload_len { return Err(config_error( "automation run spool ended before its bound payload length", diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 94cba0b7b6c..701629d6f01 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -279,21 +279,34 @@ impl FormattedTracingCapture { /// before the first captured event. #[must_use] pub fn start() -> Self { - Self::start_with_output(false) + Self::start_with_output(false, false) } /// Also writes each line to stderr so a terminated measurement retains /// the events emitted before its deadline. #[must_use] pub fn start_with_stderr() -> Self { - Self::start_with_output(true) + Self::start_with_output(true, false) } - fn start_with_output(stderr: bool) -> Self { + /// Captures span admission and completion timings as well as events, + /// retaining live stderr if the measurement is terminated before reporting. + #[must_use] + pub fn start_with_span_timings() -> Self { + Self::start_with_output(true, true) + } + + fn start_with_output(stderr: bool, span_timings: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_ansi(false) + .with_span_events(if span_timings { + tracing_subscriber::fmt::format::FmtSpan::NEW + | tracing_subscriber::fmt::format::FmtSpan::CLOSE + } else { + tracing_subscriber::fmt::format::FmtSpan::NONE + }) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { set_tracing_capture( From 8a48a71e6db712d8eb926c31ca536b4fed76dcd8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:58:26 -0800 Subject: [PATCH 46/49] test: remove completed automation append measurement --- .../automation/effect_runtime/settlement.rs | 9 ------ .../run_ledger/exact_publication.rs | 29 ++++++------------- crates/tracedecay-runtime-core/src/logging.rs | 19 ++---------- 3 files changed, 12 insertions(+), 45 deletions(-) diff --git a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs index 660a467bd01..65adf5e1f84 100644 --- a/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs +++ b/crates/tracedecay-automation-runtime/src/automation/effect_runtime/settlement.rs @@ -1717,15 +1717,6 @@ fn settle_bound_owner_with_budget( mut state: RetainedBoundSettlement, budget: Duration, ) -> Result> { - #[cfg(test)] - let _capture = state.phase_hook.as_ref().map(|_| { - tracedecay_runtime_core::logging::FormattedTracingCapture::start_with_span_timings() - }); - #[cfg(test)] - let _capture_span = state.phase_hook.as_ref().map(|_| { - tracing::info_span!("daemon.automation.retained_owner", run_id = %state.ledger.run_id) - .entered() - }); let started = std::time::Instant::now(); let mut delay = Duration::from_millis(25); loop { diff --git a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs index 389b652602e..15100632cf1 100644 --- a/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs +++ b/crates/tracedecay-automation-runtime/src/automation/run_ledger/exact_publication.rs @@ -764,17 +764,11 @@ fn publish_under_ledger_lock( bytes = publication.payload_len ) .entered(); - { - let _span = tracing::trace_span!("hosts.automation.run_ledger.seek_output").entered(); - ledger - .seek(SeekFrom::Start(pre_append_eof)) - .map_err(TraceDecayError::from)?; - } + ledger + .seek(SeekFrom::Start(pre_append_eof)) + .map_err(TraceDecayError::from)?; copy_exact_payload(&mut spool, &mut ledger, publication.payload_len)?; - { - let _span = tracing::trace_span!("hosts.automation.run_ledger.newline").entered(); - ledger.write_all(b"\n").map_err(TraceDecayError::from)?; - } + ledger.write_all(b"\n").map_err(TraceDecayError::from)?; } sync_run_ledger_file_and_parent(ledger_path, &ledger)?; verify_published_range(&mut ledger, &mut spool, &intent)?; @@ -934,16 +928,11 @@ fn copy_exact_payload( destination: &mut std::fs::File, payload_len: u64, ) -> Result<()> { - { - let _span = tracing::trace_span!("hosts.automation.run_ledger.seek_input").entered(); - source - .seek(SeekFrom::Start(0)) - .map_err(TraceDecayError::from)?; - } - let copied = { - let _span = tracing::trace_span!("hosts.automation.run_ledger.copy_payload").entered(); - std::io::copy(&mut source.take(payload_len), destination).map_err(TraceDecayError::from)? - }; + source + .seek(SeekFrom::Start(0)) + .map_err(TraceDecayError::from)?; + let copied = + std::io::copy(&mut source.take(payload_len), destination).map_err(TraceDecayError::from)?; if copied != payload_len { return Err(config_error( "automation run spool ended before its bound payload length", diff --git a/crates/tracedecay-runtime-core/src/logging.rs b/crates/tracedecay-runtime-core/src/logging.rs index 701629d6f01..94cba0b7b6c 100644 --- a/crates/tracedecay-runtime-core/src/logging.rs +++ b/crates/tracedecay-runtime-core/src/logging.rs @@ -279,34 +279,21 @@ impl FormattedTracingCapture { /// before the first captured event. #[must_use] pub fn start() -> Self { - Self::start_with_output(false, false) + Self::start_with_output(false) } /// Also writes each line to stderr so a terminated measurement retains /// the events emitted before its deadline. #[must_use] pub fn start_with_stderr() -> Self { - Self::start_with_output(true, false) + Self::start_with_output(true) } - /// Captures span admission and completion timings as well as events, - /// retaining live stderr if the measurement is terminated before reporting. - #[must_use] - pub fn start_with_span_timings() -> Self { - Self::start_with_output(true, true) - } - - fn start_with_output(stderr: bool, span_timings: bool) -> Self { + fn start_with_output(stderr: bool) -> Self { let bytes = std::sync::Arc::new(Mutex::new(Vec::new())); let subscriber = tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_ansi(false) - .with_span_events(if span_timings { - tracing_subscriber::fmt::format::FmtSpan::NEW - | tracing_subscriber::fmt::format::FmtSpan::CLOSE - } else { - tracing_subscriber::fmt::format::FmtSpan::NONE - }) .with_writer(CaptureBuffer(std::sync::Arc::clone(&bytes), stderr)); let guard = if stderr { set_tracing_capture( From cc50a0f645aea6e55ca3a0f1ec2d5b0465777e15 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:29:32 -0800 Subject: [PATCH 47/49] test(runtime): retain the last restart readiness observation A readiness deadline can expire between status probes, losing the only state needed to distinguish an unmounted route from stalled publication. Retain the last completed pending response and include it in either deadline assertion without another request or a changed wait budget. --- .../private_route_restart_acceptance.rs | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs index 2f13fb6e77e..1c753f532ed 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs @@ -1,5 +1,6 @@ use crate::common; +use std::cell::RefCell; use std::path::Path; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; @@ -175,6 +176,9 @@ async fn selected_project_source_route_survives_physical_daemon_restart() { pub(crate) fn await_published_code_index(home: &Path, project: &Path) { let project_arg = project.to_string_lossy().into_owned(); let deadline = Instant::now() + Duration::from_secs(110); + // Keep the last completed observation when the deadline falls between + // probes; the timeout itself must not issue another status request. + let last_status = RefCell::new(None::); // CLI admission spends part of the request budget before status runs, so // a nested 110-second wait can exceed the remaining dispatch deadline. // Sample the same typed predicate under one end-to-end fixture deadline. @@ -185,7 +189,12 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { let remaining_ms = deadline .saturating_duration_since(Instant::now()) .as_millis(); - assert!(remaining_ms > 0, "code-index publication deadline elapsed"); + assert!( + remaining_ms > 0, + "code-index publication deadline elapsed for {}; last status: {}", + project.display(), + last_status.borrow().as_ref().unwrap_or(&Value::Null), + ); let output = common::tracedecay_command_with_home(home) .current_dir(project) .env(TOOL_REQUEST_DEADLINE_ENV, remaining_ms.to_string()) @@ -225,6 +234,7 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { { // The canonical readiness wait allows a scheduler to mount // inside its budget. This status sample has no mount refusal. + *last_status.borrow_mut() = Some(status); return None; } StatusCodeIndexFreshnessV1::MountFailed { .. } @@ -234,7 +244,10 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { }; match worktree.readiness(CodeIndexReadinessTargetV1::GraphReady) { CodeIndexReadinessV1::Reached => Some(()), - CodeIndexReadinessV1::Pending => None, + CodeIndexReadinessV1::Pending => { + *last_status.borrow_mut() = Some(status); + None + } CodeIndexReadinessV1::Unreachable { reason } => { panic!("code-index readiness cannot be reached: {reason}; {status}") } @@ -242,8 +255,9 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { }, || { format!( - "daemon never published a serving generation for {}", - project.display() + "daemon never published a serving generation for {}; last status: {}", + project.display(), + last_status.borrow().as_ref().unwrap_or(&Value::Null), ) }, ); From 92b2a78747fb1d98520753a54d88bcb41c12da3a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:11:46 -0800 Subject: [PATCH 48/49] test(global-db): avoid unused raw fixture connections --- .../tests/lcm_schema/lcm_schema_contract.rs | 7 +- .../src/tests/lcm_schema/mod.rs | 5 -- .../src/tests/lcm_schema/temporal_catalog.rs | 11 ++- .../lcm_schema/temporal_catalog/admission.rs | 73 ++++++++----------- .../tests/lcm_schema/temporal_constraints.rs | 3 +- .../src/tests/lcm_schema/temporal_cursor.rs | 22 ++---- 6 files changed, 49 insertions(+), 72 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs index 574f15286b1..192ab5fdf7c 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs @@ -7,7 +7,7 @@ async fn legacy_profile_requires_reset_without_carrying_session_content_forward( let db_path = tmp.path().join(".tracedecay").join("sessions.db"); create_legacy_sessions_db(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Err(error) => error, Ok(_) => panic!("legacy session content must require an explicit profile reset"), }; @@ -31,8 +31,9 @@ async fn stale_or_future_lcm_marker_requires_reset_without_rewriting_marker() { ] { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.expect("fresh global db"); - drop(db); + initialize_global_db(&db_path) + .await + .expect("fresh global db"); set_migration_version(&db_path, found_version).await; let (lease, owner) = open_registered_test_database_fixture( diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index fbfe8b5f1f8..88ef9206b40 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -26,11 +26,6 @@ async fn initialize_global_db(db_path: &Path) -> tracedecay_domain::errors::Resu Ok(()) } -async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { - initialize_global_db(db_path).await?; - Ok(TestConnection::open(db_path)) -} - async fn open_read_only_global_db( db_path: &Path, ) -> tracedecay_domain::errors::Result> { diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 1d2cb2f65d2..6606fbdb0bc 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -5,10 +5,9 @@ async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "lcm_raw_messages").await, "fresh initialization must compose the final temporal and LCM schemas" @@ -622,7 +621,7 @@ async fn temporal_schema_rejects_unexpected_indexes_without_dropping_them() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("unexpected temporal indexes must require reset"), Err(error) => error, }; @@ -675,7 +674,7 @@ async fn temporal_schema_refuses_markerless_malformed_fts_without_mutation() { drop(raw_db); let before_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a markerless malformed FTS store must require reset"), Err(error) => error, }; @@ -743,7 +742,7 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("missing temporal FTS objects must require reset"), Err(error) => error, }; @@ -798,7 +797,7 @@ async fn temporal_schema_rejects_missing_graph_publication_authority_without_reb drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("missing graph publication authority must require reset"), Err(error) => error, }; diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index c1ec89d7d12..699e17c46a1 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -74,7 +74,7 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a partial temporal schema must require reset"), Err(error) => error, }; @@ -105,7 +105,7 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { // Removing an application table does not make a previously opened SQLite // namespace freshly owned. Existing sidecars remain unclaimed. - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("unpublished existing SQLite must not be reinitialized"), Err(error) => error, }; @@ -119,19 +119,17 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { ); let fresh_path = tmp.path().join(".tracedecay").join("fresh.db"); - let db = open_global_db(&fresh_path) + initialize_global_db(&fresh_path) .await .expect("a new prospective store should receive the final temporal schema"); - drop(db); let initial_catalog = temporal_schema_object_catalog(&fresh_path).await; let initial_version = temporal_schema_version(&fresh_path).await; let restart_path = tmp.path().join(".tracedecay").join("restart.db"); copy_database_for_temporal_restart(&fresh_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("idempotent temporal reopen should succeed"); - drop(reopened); assert_eq!( temporal_schema_version(&restart_path).await, initial_version @@ -163,7 +161,7 @@ async fn temporal_schema_malformed_marker_requires_reset_without_mutation() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a malformed temporal marker must require reset"), Err(error) => error, }; @@ -187,10 +185,9 @@ async fn temporal_schema_malformed_marker_requires_reset_without_mutation() { async fn temporal_schema_lower_marker_requires_reset_without_repairing_guards() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let previous = SESSION_TEMPORAL_SCHEMA_VERSION - 1; let raw_db = TestConnection::open(&db_path); @@ -226,21 +223,17 @@ async fn temporal_schema_lower_marker_requires_reset_without_repairing_guards() async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_contract() { let tmp = TempDir::new().unwrap(); let fresh_path = tmp.path().join("fresh").join("sessions.db"); - drop( - open_global_db(&fresh_path) - .await - .expect("fresh store should receive the final temporal schema"), - ); + initialize_global_db(&fresh_path) + .await + .expect("fresh store should receive the final temporal schema"); let final_catalog = temporal_schema_object_catalog(&fresh_path).await; let final_guard = normalized_trigger_sql(&fresh_path, "session_refresh_progress_insert_guard_v1").await; let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - drop( - open_global_db(&db_path) - .await - .expect("temporal schema initialization should not error"), - ); + initialize_global_db(&db_path) + .await + .expect("temporal schema initialization should not error"); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); conn.execute_batch(&format!( @@ -262,11 +255,9 @@ async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_cont "the earlier shape lacks the introduced-row index" ); - drop( - open_global_db(&db_path) - .await - .expect("an earlier authority holding no rows is replaced, not refused"), - ); + initialize_global_db(&db_path) + .await + .expect("an earlier authority holding no rows is replaced, not refused"); assert_eq!( temporal_schema_version(&db_path).await, SESSION_TEMPORAL_SCHEMA_VERSION @@ -303,7 +294,7 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { let before_catalog = temporal_schema_object_catalog(&db_path).await; let before_version = temporal_schema_version(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("branch-local transition storage must require reset"), Err(error) => error, }; @@ -348,7 +339,7 @@ async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutati let before_catalog = temporal_schema_object_catalog(&db_path).await; let before_version = temporal_schema_version(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("foreign summary-namespace storage must require reset"), Err(error) => error, }; @@ -399,7 +390,7 @@ async fn final_store_missing_temporal_marker_requires_reset_without_mutation() { drop(raw_db); let before_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a final LCM store without temporal identity must require reset"), Err(error) => error, }; @@ -452,7 +443,7 @@ async fn temporal_schema_is_not_installed_into_a_nonempty_store() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a nonempty store without temporal identity must require reset"), Err(error) => error, }; @@ -482,7 +473,7 @@ async fn unpublished_writer_ledger_is_preserved_and_fresh_publication_still_work "the seeded store must hold writer ledger tables" ); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an unpublished writer-ledger database is not a fresh store"), Err(error) => error, }; @@ -500,10 +491,9 @@ async fn unpublished_writer_ledger_is_preserved_and_fresh_publication_still_work ); let fresh_path = tmp.path().join("fresh").join("sessions.db"); - let db = open_global_db(&fresh_path) + initialize_global_db(&fresh_path) .await .expect("new prospective publication must install the final schema and writer ledger"); - drop(db); assert!( table_exists(&fresh_path, "tracedecay_schema_version").await && table_exists(&fresh_path, "td_runtime_writer_idempotency_v2").await, @@ -525,7 +515,7 @@ async fn writer_ledger_does_not_hide_a_nonempty_unmarked_store() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an application table without a temporal marker must require reset"), Err(error) => error, }; @@ -568,7 +558,7 @@ async fn final_schema_admission_rejects_extra_temporal_column_metadata() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("extra persisted temporal column metadata must require reset"), Err(error) => error, }; @@ -607,7 +597,7 @@ async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() drop(raw_db); let rejected_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a missing required temporal query index must require reset"), Err(error) => error, }; @@ -642,7 +632,7 @@ async fn final_schema_admission_rejects_missing_graph_publication_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a missing graph publication table must require reset"), Err(error) => error, }; @@ -679,7 +669,7 @@ async fn final_schema_admission_rejects_incompatible_graph_publication_column_me drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("incompatible graph publication column metadata must require reset"), Err(error) => error, }; @@ -722,7 +712,7 @@ async fn final_schema_admission_rejects_extra_graph_publication_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an extra graph publication table must require reset"), Err(error) => error, }; @@ -762,7 +752,7 @@ async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a mixed-case extra temporal table must require reset"), Err(error) => error, }; @@ -802,7 +792,7 @@ async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a mixed-case extra graph publication table must require reset"), Err(error) => error, }; @@ -840,7 +830,7 @@ async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an extra index on a canonical graph table must require reset"), Err(error) => error, }; @@ -862,10 +852,9 @@ async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { async fn temporal_schema_refuses_future_version_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "tracedecay_schema_version").await, "the temporal schema must install a version marker before a future version is tested" diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index 17c54837976..fec4a8e7215 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -4,10 +4,9 @@ use super::*; async fn temporal_schema_rejects_cross_session_and_generation_rows() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "session_temporal_generations").await, "the temporal generation owner table must exist before ownership checks" diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index ad69571ce89..898537d7158 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -24,10 +24,9 @@ fn assert_cursor_rotation_verdict(error: TraceDecayError, context: &str) { async fn temporal_schema_persists_cursor_keys_without_read_creation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "session_query_cursor_keys").await, "the temporal schema must create the cursor-key authority table" @@ -50,10 +49,9 @@ async fn temporal_schema_persists_cursor_keys_without_read_creation() { let restart_path = tmp.path().join(".tracedecay").join("restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("writer reopen should preserve a persisted cursor key"); - drop(reopened); assert_eq!( row_count(&restart_path, "session_query_cursor_keys").await, 1 @@ -234,10 +232,9 @@ async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { let restart_path = tmp.path().join(".tracedecay").join("cursor-restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("rotated cursor key authority must pass restart validation"); - drop(reopened); assert_eq!( row_count(&restart_path, "session_query_cursor_keys").await, 2 @@ -281,7 +278,7 @@ async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { let restart_path = tmp.path().join(".tracedecay").join("cursor-audit.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; assert_cursor_rotation_verdict( - open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .err() .expect("restart audit refuses the database"), @@ -335,7 +332,7 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { .join("restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; assert_cursor_rotation_verdict( - open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .err() .expect("restart audit refuses the database"), @@ -385,10 +382,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { .join(".tracedecay") .join("valid-restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("valid immediate-successor chain must pass restart audit"); - drop(reopened); assert_valid_cursor_chain(&cursor_key_history(&restart_path).await); } } @@ -397,10 +393,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { async fn temporal_schema_concurrent_cursor_rotations_serialize_safely() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let seed_db = TestConnection::open(&db_path); let initial = (*seed_db).clone(); @@ -540,9 +535,8 @@ async fn temporal_schema_concurrent_cursor_rotations_serialize_safely() { let restart_path = tmp.path().join(".tracedecay").join("concurrent-restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("serialized concurrent rotations must pass restart audit"); - drop(reopened); assert_valid_cursor_chain(&cursor_key_history(&restart_path).await); } From 74f4a86d2d5c478ea67f88a7ea582ec79e4863e5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:33:28 -0800 Subject: [PATCH 49/49] test(global-db): assert initialization refusal directly --- .../src/tests/lcm_schema/temporal_cursor.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index 898537d7158..848337f9235 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -280,8 +280,7 @@ async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { assert_cursor_rotation_verdict( initialize_global_db(&restart_path) .await - .err() - .expect("restart audit refuses the database"), + .expect_err("restart audit refuses the database"), "restart audit must reject an active key that is not the monotonic maximum", ); } @@ -334,8 +333,7 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { assert_cursor_rotation_verdict( initialize_global_db(&restart_path) .await - .err() - .expect("restart audit refuses the database"), + .expect_err("restart audit refuses the database"), &format!( "{fixture}: a later key must not satisfy a skipped immediate-successor retirement" ),